Skip to content

guidellm.data.deserializers.huggingface

HuggingFaceDataArgs

Bases: DataArgs

Model for Hugging Face dataset deserializer arguments.

Source code in src/guidellm/data/deserializers/huggingface.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
@DataArgs.register(["huggingface", "hf"])
class HuggingFaceDataArgs(DataArgs):
    """Model for Hugging Face dataset deserializer arguments."""

    kind: Literal["huggingface", "hf"] = Field(
        default="huggingface",
        description="Type identifier for the Hugging Face dataset deserializer.",
    )
    source: str | Dataset | IterableDataset | DatasetDict | IterableDatasetDict = Field(
        validation_alias=AliasChoices("source", "src", "from", "path", "name"),
        description=(
            "Data input for the Hugging Face dataset deserializer. This can be a "
            "Dataset, IterableDataset, DatasetDict, IterableDatasetDict, a string or "
            "Path to a local dataset directory or a local .py dataset script, or a "
            "dataset identifier from the Hugging Face Hub."
        ),
        examples=["wikimedia/structured-wikipedia", "./dataset.json"],
    )

HuggingFaceDatasetDeserializer

Bases: DatasetDeserializer

Deserializer for Hugging Face datasets.

Source code in src/guidellm/data/deserializers/huggingface.py 
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
@DatasetDeserializerFactory.register(["huggingface", "hf"])
class HuggingFaceDatasetDeserializer(DatasetDeserializer):
    """Deserializer for Hugging Face datasets."""

    def __call__(
        self,
        config: HuggingFaceDataArgs,
        processor_factory: Callable[[], PreTrainedTokenizerBase],
        random_seed: int,
    ) -> Dataset | IterableDataset | DatasetDict | IterableDatasetDict:
        _ = (processor_factory, random_seed)
        data = config.source

        if isinstance(
            data, Dataset | IterableDataset | DatasetDict | IterableDatasetDict
        ):
            return data

        load_error = None

        if (
            isinstance(data, str | Path)
            and (path := Path(data)).exists()
            and ((path.is_file() and path.suffix == ".py") or path.is_dir())
        ):
            # Handle python script or nested python script in a directory
            try:
                return load_dataset(str(data), **config.load_kwargs)
            except (
                FileNotFoundDatasetsError,
                DatasetNotFoundError,
                DataFilesNotFoundError,
            ) as err:
                load_error = err
            except Exception:  # noqa: BLE001
                # Try loading as a local dataset directory next
                try:
                    return load_from_disk(str(data), **config.load_kwargs)
                except (
                    FileNotFoundDatasetsError,
                    DatasetNotFoundError,
                    DataFilesNotFoundError,
                ) as err2:
                    load_error = err2

        try:
            # Handle dataset identifier from the Hugging Face Hub
            return load_dataset(str(data), **config.load_kwargs)
        except (
            FileNotFoundDatasetsError,
            DatasetNotFoundError,
            DataFilesNotFoundError,
        ) as err:
            load_error = err

        not_supported = DataNotSupportedError(
            "Unsupported data for HuggingFaceDatasetDeserializer, "
            "expected Dataset, IterableDataset, DatasetDict, IterableDatasetDict, "
            "str or Path to a local dataset directory or a local .py dataset script, "
            f"got {data} and HF load error: {load_error}"
        )

        if load_error is not None:
            raise not_supported from load_error
        else:
            raise not_supported