File size: 4,024 Bytes
152c60b 3129d49 4314c58 3e28aad 7cdc7d0 d08fbc6 3e28aad 50db311 75cf782 058c80a a387724 50db311 0c55b4f 9d5b4c0 785b9b9 3e28aad 0c578b5 3129d49 3e28aad 058c80a 36e41c0 d08fbc6 4314c58 3e28aad 4314c58 3129d49 058c80a d9c13ca 3e28aad a387724 3e28aad be4a716 3e28aad 0c55b4f 3e28aad 7cdc7d0 058c80a be4a716 3e28aad 3129d49 0c578b5 3e28aad 100c2eb 3128771 be4a716 3e28aad 64458da 7cdc7d0 058c80a be4a716 3129d49 a64dc20 b5afa71 3e28aad d9c13ca be4a716 d9c13ca 152c60b e4e068f 3e28aad be4a716 3e28aad be4a716 36e41c0 be4a716 058c80a 152c60b d9c13ca 152c60b d9c13ca 152c60b 3e28aad 1083665 3e28aad 1083665 0c55b4f 1083665 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 |
import os
import datasets
from .api import __file__ as _
from .artifact import __file__ as _
from .augmentors import __file__ as _
from .benchmark import __file__ as _
from .blocks import __file__ as _
from .card import __file__ as _
from .catalog import __file__ as _
from .collections import __file__ as _
from .collections_operators import __file__ as _
from .dataclass import __file__ as _
from .dataset_utils import __file__ as _
from .dataset_utils import get_dataset_artifact
from .deprecation_utils import __file__ as _
from .dialog_operators import __file__ as _
from .dict_utils import __file__ as _
from .error_utils import __file__ as _
from .eval_utils import __file__ as _
from .file_utils import __file__ as _
from .formats import __file__ as _
from .fusion import __file__ as _
from .generator_utils import __file__ as _
from .hf_utils import __file__ as _
from .hf_utils import verify_versions_compatibility
from .image_operators import __file__ as _
from .inference import __file__ as _
from .instructions import __file__ as _
from .llm_as_judge import __file__ as _
from .loaders import __file__ as _
from .logging_utils import __file__ as _
from .logging_utils import get_logger
from .metric import __file__ as _
from .metric_utils import __file__ as _
from .metrics import __file__ as _
from .normalizers import __file__ as _
from .operator import __file__ as _
from .operators import __file__ as _
from .parsing_utils import __file__ as _
from .processors import __file__ as _
from .random_utils import __file__ as _
from .recipe import __file__ as _
from .register import __file__ as _
from .schema import __file__ as _
from .serializers import __file__ as _
from .settings_utils import __file__ as _
from .settings_utils import get_constants
from .span_lableing_operators import __file__ as _
from .split_utils import __file__ as _
from .splitters import __file__ as _
from .standard import __file__ as _
from .stream import __file__ as _
from .stream_operators import __file__ as _
from .string_operators import __file__ as _
from .struct_data_operators import __file__ as _
from .system_prompts import __file__ as _
from .task import __file__ as _
from .templates import __file__ as _
from .text_utils import __file__ as _
from .type_utils import __file__ as _
from .types import __file__ as _
from .utils import __file__ as _
from .utils import is_package_installed
from .validate import __file__ as _
from .version import __file__ as _
from .version import version
logger = get_logger()
constants = get_constants()
class Dataset(datasets.GeneratorBasedBuilder):
"""TODO: Short description of my dataset."""
VERSION = constants.version
@property
def generators(self):
if not hasattr(self, "_generators") or self._generators is None:
if is_package_installed("unitxt"):
verify_versions_compatibility("dataset", self.VERSION)
from unitxt.dataset_utils import \
get_dataset_artifact as get_dataset_artifact_installed
logger.info("Loading with installed unitxt library...")
dataset = get_dataset_artifact_installed(self.config.name)
else:
logger.info("Loading with huggingface unitxt copy...")
dataset = get_dataset_artifact(self.config.name)
self._generators = dataset()
return self._generators
def _info(self):
return datasets.DatasetInfo()
def _split_generators(self, _):
return [
datasets.SplitGenerator(name=name, gen_kwargs={"split_name": name})
for name in self.generators.keys()
]
def _generate_examples(self, split_name):
generator = self.generators[split_name]
yield from enumerate(generator)
def _download_and_prepare(
self, dl_manager, verification_mode, **prepare_splits_kwargs
):
return super()._download_and_prepare(
dl_manager, "no_checks", **prepare_splits_kwargs
)
|