guidellm.data.tokenizers.tokenizer

class TokenizerRegistry(RegistryMixin[type[DataTokenizer]]):
    @classmethod
    def create(cls, config: DataTokenizerArgs) -> DataTokenizer:
        """
        Factory method to create a DatasetTokenizer instance based on configuration.

        :param config: A DataTokenizerArgs object containing the configuration.
        """
        kind = config.kind
        tokenizer_cls = cls.get_registered_object(kind)

        if tokenizer_cls is None:
            raise ValueError(
                f"DataTokenizer type '{kind}' is not registered."
                f"Available types: {list(cls.registry.keys()) if cls.registry else []}"
            )

        return tokenizer_cls(config)