Skip to content

backends

Module to define the backends in charge of creating logits processors.

get_cfg_logits_processor(backend_name, model, grammar)

Create a logits processor from a context-free grammar.

Parameters:

Name Type Description Default
backend_name str | None

The name of the backend to use.

required
model SteerableModel

The Outlines model of the user.

required
grammar str

The context-free grammar to create a logits processor from.

required

Returns:

Type Description
LogitsProcessorType

The logits processor.

Source code in outlines/backends/__init__.py
def get_cfg_logits_processor(
    backend_name: str | None,
    model: SteerableModel,
    grammar: str,
) -> LogitsProcessorType:
    """Create a logits processor from a context-free grammar.

    Parameters
    ----------
    backend_name: str | None
        The name of the backend to use.
    model: Model
        The Outlines model of the user.
    grammar: str
        The context-free grammar to create a logits processor from.

    Returns
    -------
    LogitsProcessorType
        The logits processor.

    """
    backend = _get_backend(
        backend_name or CFG_DEFAULT_BACKEND,
        model,
    )
    return backend.get_cfg_logits_processor(grammar)

get_json_schema_logits_processor(backend_name, model, json_schema)

Create a logits processor from a JSON schema.

Parameters:

Name Type Description Default
backend_name str | None

The name of the backend to use.

required
model SteerableModel

The Outlines model of the user.

required
json_schema str

The JSON schema to create a logits processor from.

required

Returns:

Type Description
LogitsProcessorType

The logits processor.

Source code in outlines/backends/__init__.py
def get_json_schema_logits_processor(
    backend_name: str | None,
    model: SteerableModel,
    json_schema: str,
) -> LogitsProcessorType:
    """Create a logits processor from a JSON schema.

    Parameters
    ----------
    backend_name: str | None
        The name of the backend to use.
    model: Model
        The Outlines model of the user.
    json_schema: str
        The JSON schema to create a logits processor from.

    Returns
    -------
    LogitsProcessorType
        The logits processor.

    """
    backend = _get_backend(
        backend_name or JSON_SCHEMA_DEFAULT_BACKEND,
        model,
    )
    return backend.get_json_schema_logits_processor(json_schema)

get_regex_logits_processor(backend_name, model, regex)

Create a logits processor from a regex.

Parameters:

Name Type Description Default
backend_name str | None

The name of the backend to use.

required
model SteerableModel

The Outlines model of the user.

required
regex str

The regex to create a logits processor from.

required

Returns:

Type Description
LogitsProcessorType

The logits processor.

Source code in outlines/backends/__init__.py
def get_regex_logits_processor(
    backend_name: str | None,
    model: SteerableModel,
    regex: str,
) -> LogitsProcessorType:
    """Create a logits processor from a regex.

    Parameters
    ----------
    backend_name: str | None
        The name of the backend to use.
    model: Model
        The Outlines model of the user.
    regex: str
        The regex to create a logits processor from.

    Returns
    -------
    LogitsProcessorType
        The logits processor.

    """
    backend = _get_backend(
        backend_name or REGEX_DEFAULT_BACKEND,
        model,
    )
    return backend.get_regex_logits_processor(regex)

base

Base class for all backends.

BaseBackend

Bases: ABC

Base class for all backends.

The subclasses must implement methods that create a logits processor from a JSON schema, regex or CFG.

Source code in outlines/backends/base.py
class BaseBackend(ABC):
    """Base class for all backends.

    The subclasses must implement methods that create a logits processor
    from a JSON schema, regex or CFG.

    """

    @abstractmethod
    def get_json_schema_logits_processor(
        self, json_schema: str
    ) -> LogitsProcessorType:
        """Create a logits processor from a JSON schema.

        Parameters
        ----------
        json_schema: str
            The JSON schema to create a logits processor from.

        Returns
        -------
        LogitsProcessorType
            The logits processor.

        """
        ...

    @abstractmethod
    def get_regex_logits_processor(self, regex: str) -> LogitsProcessorType:
        """Create a logits processor from a regex.

        Parameters
        ----------
        regex: str
            The regex to create a logits processor from.

        Returns
        -------
        LogitsProcessorType
            The logits processor.

        """
        ...

    @abstractmethod
    def get_cfg_logits_processor(self, grammar: str) -> LogitsProcessorType:
        """Create a logits processor from a context-free grammar.

        Parameters
        ----------
        grammar: str
            The context-free grammar to create a logits processor from.

        Returns
        -------
        LogitsProcessorType
            The logits processor.

        """
        ...

get_cfg_logits_processor(grammar) abstractmethod

Create a logits processor from a context-free grammar.

Parameters:

Name Type Description Default
grammar str

The context-free grammar to create a logits processor from.

required

Returns:

Type Description
LogitsProcessorType

The logits processor.

Source code in outlines/backends/base.py
@abstractmethod
def get_cfg_logits_processor(self, grammar: str) -> LogitsProcessorType:
    """Create a logits processor from a context-free grammar.

    Parameters
    ----------
    grammar: str
        The context-free grammar to create a logits processor from.

    Returns
    -------
    LogitsProcessorType
        The logits processor.

    """
    ...

get_json_schema_logits_processor(json_schema) abstractmethod

Create a logits processor from a JSON schema.

Parameters:

Name Type Description Default
json_schema str

The JSON schema to create a logits processor from.

required

Returns:

Type Description
LogitsProcessorType

The logits processor.

Source code in outlines/backends/base.py
@abstractmethod
def get_json_schema_logits_processor(
    self, json_schema: str
) -> LogitsProcessorType:
    """Create a logits processor from a JSON schema.

    Parameters
    ----------
    json_schema: str
        The JSON schema to create a logits processor from.

    Returns
    -------
    LogitsProcessorType
        The logits processor.

    """
    ...

get_regex_logits_processor(regex) abstractmethod

Create a logits processor from a regex.

Parameters:

Name Type Description Default
regex str

The regex to create a logits processor from.

required

Returns:

Type Description
LogitsProcessorType

The logits processor.

Source code in outlines/backends/base.py
@abstractmethod
def get_regex_logits_processor(self, regex: str) -> LogitsProcessorType:
    """Create a logits processor from a regex.

    Parameters
    ----------
    regex: str
        The regex to create a logits processor from.

    Returns
    -------
    LogitsProcessorType
        The logits processor.

    """
    ...

llguidance

Backend class for LLGuidance.

LLGuidanceBackend

Bases: BaseBackend

Backend for LLGuidance.

Source code in outlines/backends/llguidance.py
class LLGuidanceBackend(BaseBackend):
    """Backend for LLGuidance."""

    def __init__(self, model: SteerableModel):
        """
        Parameters
        ----------
        model
            The Outlines model of the user.

        """
        import llguidance as llg

        self.llg = llg
        self.tensor_library_name = model.tensor_library_name
        self.llg_tokenizer = self._create_llg_tokenizer(model)

    def _create_llg_tokenizer(self, model: SteerableModel) -> "LLGTokenizer":
        """Create an llg tokenizer from the Outlines model's tokenizer.

        Parameters
        ----------
        model: Model
            The Outlines model.

        Returns
        -------
        LLGTokenizer
            The llg tokenizer.

        """
        if isinstance(model, Transformers):
            import llguidance.hf

            return llguidance.hf.from_tokenizer(model.hf_tokenizer)

        elif isinstance(model, LlamaCpp):
            import llama_cpp
            import llguidance.llamacpp

            vocab = llama_cpp.llama_model_get_vocab(model.model.model)
            return llguidance.llamacpp.lltokenizer_from_vocab(vocab)

        elif isinstance(model, MLXLM): # pragma: no cover
            import llguidance.hf

            return llguidance.hf.from_tokenizer(
                model.mlx_tokenizer._tokenizer
            )

        else: # pragma: no cover
            raise ValueError(
                f"Unsupported model type: {type(model)}. "
                "Llguidance only supports LlamaCpp, MLXLM "
                "and Transformers models."
            )

    def get_json_schema_logits_processor(
        self, json_schema: str
    ) -> LLGuidanceLogitsProcessor:
        """Create a logits processor from a JSON schema.

        Parameters
        ----------
        json_schema: str
            The JSON schema to create a logits processor from.

        Returns
        -------
        LogitsProcessor
            The logits processor to use to constrain the generation.

        """
        grammar_spec = self.llg.grammar_from("json_schema", json_schema)
        return LLGuidanceLogitsProcessor(
            grammar_spec, self.llg_tokenizer, self.tensor_library_name
        )

    def get_regex_logits_processor(
        self, regex: str
    ) -> LLGuidanceLogitsProcessor:
        """Create a logits processor from a regex.

        Parameters
        ----------
        regex: str
            The regex to create a logits processor from.

        Returns
        -------
        LogitsProcessor
            The logits processor to use to constrain the generation.

        """
        grammar_spec = self.llg.grammar_from("regex", regex)
        return LLGuidanceLogitsProcessor(
            grammar_spec, self.llg_tokenizer, self.tensor_library_name
        )

    def get_cfg_logits_processor(
        self, grammar: str
    ) -> LLGuidanceLogitsProcessor:
        """Create a logits processor from a context-free grammar.

        Parameters
        ----------
        grammar: str
            The context-free grammar to create a logits processor from.

        Returns
        -------
        LogitsProcessor
            The logits processor to use to constrain the generation.

        """
        # We try both lark and ebnf
        try:
            grammar_spec = self.llg.grammar_from("grammar", grammar)
        except ValueError:
            grammar_spec = self.llg.grammar_from("lark", grammar)
        return LLGuidanceLogitsProcessor(
            grammar_spec, self.llg_tokenizer, self.tensor_library_name
        )

__init__(model)

Parameters:

Name Type Description Default
model SteerableModel

The Outlines model of the user.

required
Source code in outlines/backends/llguidance.py
def __init__(self, model: SteerableModel):
    """
    Parameters
    ----------
    model
        The Outlines model of the user.

    """
    import llguidance as llg

    self.llg = llg
    self.tensor_library_name = model.tensor_library_name
    self.llg_tokenizer = self._create_llg_tokenizer(model)

get_cfg_logits_processor(grammar)

Create a logits processor from a context-free grammar.

Parameters:

Name Type Description Default
grammar str

The context-free grammar to create a logits processor from.

required

Returns:

Type Description
LogitsProcessor

The logits processor to use to constrain the generation.

Source code in outlines/backends/llguidance.py
def get_cfg_logits_processor(
    self, grammar: str
) -> LLGuidanceLogitsProcessor:
    """Create a logits processor from a context-free grammar.

    Parameters
    ----------
    grammar: str
        The context-free grammar to create a logits processor from.

    Returns
    -------
    LogitsProcessor
        The logits processor to use to constrain the generation.

    """
    # We try both lark and ebnf
    try:
        grammar_spec = self.llg.grammar_from("grammar", grammar)
    except ValueError:
        grammar_spec = self.llg.grammar_from("lark", grammar)
    return LLGuidanceLogitsProcessor(
        grammar_spec, self.llg_tokenizer, self.tensor_library_name
    )

get_json_schema_logits_processor(json_schema)

Create a logits processor from a JSON schema.

Parameters:

Name Type Description Default
json_schema str

The JSON schema to create a logits processor from.

required

Returns:

Type Description
LogitsProcessor

The logits processor to use to constrain the generation.

Source code in outlines/backends/llguidance.py
def get_json_schema_logits_processor(
    self, json_schema: str
) -> LLGuidanceLogitsProcessor:
    """Create a logits processor from a JSON schema.

    Parameters
    ----------
    json_schema: str
        The JSON schema to create a logits processor from.

    Returns
    -------
    LogitsProcessor
        The logits processor to use to constrain the generation.

    """
    grammar_spec = self.llg.grammar_from("json_schema", json_schema)
    return LLGuidanceLogitsProcessor(
        grammar_spec, self.llg_tokenizer, self.tensor_library_name
    )

get_regex_logits_processor(regex)

Create a logits processor from a regex.

Parameters:

Name Type Description Default
regex str

The regex to create a logits processor from.

required

Returns:

Type Description
LogitsProcessor

The logits processor to use to constrain the generation.

Source code in outlines/backends/llguidance.py
def get_regex_logits_processor(
    self, regex: str
) -> LLGuidanceLogitsProcessor:
    """Create a logits processor from a regex.

    Parameters
    ----------
    regex: str
        The regex to create a logits processor from.

    Returns
    -------
    LogitsProcessor
        The logits processor to use to constrain the generation.

    """
    grammar_spec = self.llg.grammar_from("regex", regex)
    return LLGuidanceLogitsProcessor(
        grammar_spec, self.llg_tokenizer, self.tensor_library_name
    )

LLGuidanceLogitsProcessor

Bases: OutlinesLogitsProcessor

Logits Processor for the LLGuidance backend.

Source code in outlines/backends/llguidance.py
class LLGuidanceLogitsProcessor(OutlinesLogitsProcessor):
    """Logits Processor for the LLGuidance backend."""

    def __init__(
        self,
        grammar: str,
        llg_tokenizer,
        tensor_library_name: str,
    ) -> None:
        """
        Parameters
        ----------
        grammar: str
            The grammar spec to use to create the LLMatcher
        llg_tokenizer: LLTokenizer
            The LLGuidance tokenizer
        tensor_library_name: str
            The name of the tensor library used by the model

        """
        if tensor_library_name not in SUPPORTED_TENSOR_LIBRARIES:
            raise TypeError(f"Unsupported tensor library: {tensor_library_name}")

        self.is_first_token = True
        self.grammar = grammar
        self.llg_tokenizer = llg_tokenizer
        self.tensor_library_name = tensor_library_name
        super().__init__(tensor_library_name)

    def reset(self):
        """Ensure self._setup is called again for the next generation."""
        self.is_first_token = True

    def _setup(self, batch_size: int) -> None:
        """Setup the LLMatchers, the bitmask and some functions used in the
        `process_logits` method.

        This method is called when the first token is generated instead of
        at initialization because we need to know the batch size.

        Parameters
        ----------
        batch_size: int
            The batch size of the input

        """
        from llguidance import LLMatcher

        self.ll_matchers = [
            LLMatcher(self.llg_tokenizer, self.grammar)
            for _ in range(batch_size)
        ]

        # we must adapt the bitmask creation and the bias function to the
        # tensor library used by the model
        if self.tensor_library_name == "torch":
            import llguidance.torch

            self.bitmask = llguidance.torch.allocate_token_bitmask(batch_size, self.llg_tokenizer.vocab_size)
            self._bias_logits = self._bias_logits_torch
        elif self.tensor_library_name == "numpy":
            import llguidance.numpy

            self.bitmask = llguidance.numpy.allocate_token_bitmask(batch_size, self.llg_tokenizer.vocab_size)
            self._bias_logits = self._bias_logits_numpy
        elif self.tensor_library_name == "mlx": # pragma: no cover
            import llguidance.numpy

            self.bitmask = llguidance.numpy.allocate_token_bitmask(batch_size, self.llg_tokenizer.vocab_size)
            self._bias_logits = self._bias_logits_mlx
        else: # pragma: no cover
            raise ValueError(f"Unsupported tensor library: {self.tensor_library_name}")

    def _bias_logits_mlx( # pragma: no cover
        self, input_ids: TensorType, logits: TensorType
    ) -> TensorType:
        """Bias the logits for the MLX backend."""
        import llguidance.mlx
        import llguidance.numpy

        biased_logits_array = []
        for i in range(self.tensor_adapter.shape(input_ids)[0]):
            llguidance.numpy.fill_next_token_bitmask(self.ll_matchers[i], self.bitmask, i)
            biased_logits = llguidance.mlx.apply_token_bitmask(
                logits[i], self.bitmask[i] # type: ignore
            )
            biased_logits_array.append(biased_logits)

        return self.tensor_adapter.concatenate(biased_logits_array)

    def _bias_logits_torch(
        self, input_ids: TensorType, logits: TensorType
    ) -> TensorType:
        """Bias the logits for the Torch backend."""
        import llguidance.torch

        for i in range(self.tensor_adapter.shape(input_ids)[0]):
            llguidance.torch.fill_next_token_bitmask(self.ll_matchers[i], self.bitmask, i)
            llguidance.torch.apply_token_bitmask_inplace(
                logits[i], self.bitmask[i] # type: ignore
            )

        return logits

    def _bias_logits_numpy(
        self, input_ids: TensorType, logits: TensorType
    ) -> TensorType:
        """Bias the logits for the Numpy backend."""
        import llguidance.numpy

        for i in range(self.tensor_adapter.shape(input_ids)[0]):
            llguidance.numpy.fill_next_token_bitmask(self.ll_matchers[i], self.bitmask, i)
            llguidance.numpy.apply_token_bitmask_inplace(
                logits[i], self.bitmask[i] # type: ignore
            )

        return logits

    def process_logits(
        self, input_ids: TensorType, logits: TensorType
    ) -> TensorType:
        """Use the instances of LLMatcher to bias the logits.

        Parameters
        ----------
        input_ids
            The ids of the tokens of the existing sequences.
        logits
            The logits for the current generation step.

        Returns
        -------
        TensorType
            The biased logits.

        """
        if self.is_first_token:
            self._setup(self.tensor_adapter.shape(input_ids)[0])
            self.is_first_token = False

        # we do not make the matchers consume the last token during the first
        # generation step because no tokens have been generated yet
        else:
            for i in range(self.tensor_adapter.shape(input_ids)[0]):
                sequence = input_ids[i] # type: ignore
                last_token = sequence[-1].item()
                self.ll_matchers[i].consume_token(last_token)
                error = self.ll_matchers[i].get_error()
                if error:
                    warnings.warn(f"Error in LLMatcher: {error}")

        return self._bias_logits(input_ids, logits)

__init__(grammar, llg_tokenizer, tensor_library_name)

Parameters:

Name Type Description Default
grammar str

The grammar spec to use to create the LLMatcher

required
llg_tokenizer

The LLGuidance tokenizer

required
tensor_library_name str

The name of the tensor library used by the model

required
Source code in outlines/backends/llguidance.py
def __init__(
    self,
    grammar: str,
    llg_tokenizer,
    tensor_library_name: str,
) -> None:
    """
    Parameters
    ----------
    grammar: str
        The grammar spec to use to create the LLMatcher
    llg_tokenizer: LLTokenizer
        The LLGuidance tokenizer
    tensor_library_name: str
        The name of the tensor library used by the model

    """
    if tensor_library_name not in SUPPORTED_TENSOR_LIBRARIES:
        raise TypeError(f"Unsupported tensor library: {tensor_library_name}")

    self.is_first_token = True
    self.grammar = grammar
    self.llg_tokenizer = llg_tokenizer
    self.tensor_library_name = tensor_library_name
    super().__init__(tensor_library_name)

process_logits(input_ids, logits)

Use the instances of LLMatcher to bias the logits.

Parameters:

Name Type Description Default
input_ids TensorType

The ids of the tokens of the existing sequences.

required
logits TensorType

The logits for the current generation step.

required

Returns:

Type Description
TensorType

The biased logits.

Source code in outlines/backends/llguidance.py
def process_logits(
    self, input_ids: TensorType, logits: TensorType
) -> TensorType:
    """Use the instances of LLMatcher to bias the logits.

    Parameters
    ----------
    input_ids
        The ids of the tokens of the existing sequences.
    logits
        The logits for the current generation step.

    Returns
    -------
    TensorType
        The biased logits.

    """
    if self.is_first_token:
        self._setup(self.tensor_adapter.shape(input_ids)[0])
        self.is_first_token = False

    # we do not make the matchers consume the last token during the first
    # generation step because no tokens have been generated yet
    else:
        for i in range(self.tensor_adapter.shape(input_ids)[0]):
            sequence = input_ids[i] # type: ignore
            last_token = sequence[-1].item()
            self.ll_matchers[i].consume_token(last_token)
            error = self.ll_matchers[i].get_error()
            if error:
                warnings.warn(f"Error in LLMatcher: {error}")

    return self._bias_logits(input_ids, logits)

reset()

Ensure self._setup is called again for the next generation.

Source code in outlines/backends/llguidance.py
def reset(self):
    """Ensure self._setup is called again for the next generation."""
    self.is_first_token = True

outlines_core

Backend class for Outlines Core.

OutlinesCoreBackend

Bases: BaseBackend

Backend for Outlines Core.

Source code in outlines/backends/outlines_core.py
class OutlinesCoreBackend(BaseBackend):
    """Backend for Outlines Core."""

    def __init__(self, model: SteerableModel):
        """
        Parameters
        ----------
        model
            The Outlines model of the user.

        """
        if isinstance(model, Transformers):
            vocabulary = model.hf_tokenizer.get_vocab()
            eos_token_id = model.hf_tokenizer.eos_token_id
            eos_token = model.hf_tokenizer.eos_token
        elif isinstance(model, LlamaCpp):
            vocabulary = model.tokenizer.vocabulary
            eos_token_id = model.tokenizer.eos_token_id
            eos_token = model.tokenizer.eos_token
        elif isinstance(model, MLXLM):
            vocabulary = model.mlx_tokenizer._tokenizer.get_vocab()
            eos_token_id = model.mlx_tokenizer._tokenizer.eos_token_id
            eos_token = model.mlx_tokenizer._tokenizer.eos_token
        else:
            raise ValueError(f"Unsupported model type: {type(model)}")

        self.eos_token_id = eos_token_id
        self.vocabulary = self.create_outlines_core_vocabulary(
            vocabulary, eos_token_id, eos_token
        )
        self.tensor_library_name = model.tensor_library_name

    def get_json_schema_logits_processor(
        self, json_schema: str
    ):
        """Create a logits processor from a JSON schema.

        Parameters
        ----------
        json_schema: str
            The JSON schema to create a logits processor from.

        Returns
        -------
        LogitsProcessor
            The logits processor to use to constrain the generation.

        """
        regex = outlines_core.json_schema.build_regex_from_schema(json_schema)
        return self.get_regex_logits_processor(regex)

    def get_regex_logits_processor(self, regex: str):
        """Create a logits processor from a regex.

        Parameters
        ----------
        regex: str
            The regex to create a logits processor from.

        Returns
        -------
        LogitsProcessor
            The logits processor to use to constrain the generation.

        """
        index = Index(regex, self.vocabulary)
        return OutlinesCoreLogitsProcessor(index, self.tensor_library_name)

    def get_cfg_logits_processor(self, grammar):
        raise NotImplementedError(
            "Outlines Core does not support context-free grammar."
        )

    @staticmethod
    def create_outlines_core_vocabulary(
        vocab: Dict[str, int], eos_token_id: int, eos_token: str
    ) -> Vocabulary:
        """Create an Outlines Core Vocabulary instance.

        Parameters
        ----------
        vocab: Dict[str, int]
            The vocabulary to create an Outlines Core vocabulary from.
        eos_token_id: int
            The EOS token ID.
        eos_token: str
            The EOS token.

        Returns
        -------
        Vocabulary
            The Outlines Core Vocabulary instance.

        """
        formatted_vocab = {}
        for token, token_id in vocab.items():
            formatted_vocab[token] = [token_id]
        formatted_vocab.pop(eos_token)
        return Vocabulary(eos_token_id, formatted_vocab)

__init__(model)

Parameters:

Name Type Description Default
model SteerableModel

The Outlines model of the user.

required
Source code in outlines/backends/outlines_core.py
def __init__(self, model: SteerableModel):
    """
    Parameters
    ----------
    model
        The Outlines model of the user.

    """
    if isinstance(model, Transformers):
        vocabulary = model.hf_tokenizer.get_vocab()
        eos_token_id = model.hf_tokenizer.eos_token_id
        eos_token = model.hf_tokenizer.eos_token
    elif isinstance(model, LlamaCpp):
        vocabulary = model.tokenizer.vocabulary
        eos_token_id = model.tokenizer.eos_token_id
        eos_token = model.tokenizer.eos_token
    elif isinstance(model, MLXLM):
        vocabulary = model.mlx_tokenizer._tokenizer.get_vocab()
        eos_token_id = model.mlx_tokenizer._tokenizer.eos_token_id
        eos_token = model.mlx_tokenizer._tokenizer.eos_token
    else:
        raise ValueError(f"Unsupported model type: {type(model)}")

    self.eos_token_id = eos_token_id
    self.vocabulary = self.create_outlines_core_vocabulary(
        vocabulary, eos_token_id, eos_token
    )
    self.tensor_library_name = model.tensor_library_name

create_outlines_core_vocabulary(vocab, eos_token_id, eos_token) staticmethod

Create an Outlines Core Vocabulary instance.

Parameters:

Name Type Description Default
vocab Dict[str, int]

The vocabulary to create an Outlines Core vocabulary from.

required
eos_token_id int

The EOS token ID.

required
eos_token str

The EOS token.

required

Returns:

Type Description
Vocabulary

The Outlines Core Vocabulary instance.

Source code in outlines/backends/outlines_core.py
@staticmethod
def create_outlines_core_vocabulary(
    vocab: Dict[str, int], eos_token_id: int, eos_token: str
) -> Vocabulary:
    """Create an Outlines Core Vocabulary instance.

    Parameters
    ----------
    vocab: Dict[str, int]
        The vocabulary to create an Outlines Core vocabulary from.
    eos_token_id: int
        The EOS token ID.
    eos_token: str
        The EOS token.

    Returns
    -------
    Vocabulary
        The Outlines Core Vocabulary instance.

    """
    formatted_vocab = {}
    for token, token_id in vocab.items():
        formatted_vocab[token] = [token_id]
    formatted_vocab.pop(eos_token)
    return Vocabulary(eos_token_id, formatted_vocab)

get_json_schema_logits_processor(json_schema)

Create a logits processor from a JSON schema.

Parameters:

Name Type Description Default
json_schema str

The JSON schema to create a logits processor from.

required

Returns:

Type Description
LogitsProcessor

The logits processor to use to constrain the generation.

Source code in outlines/backends/outlines_core.py
def get_json_schema_logits_processor(
    self, json_schema: str
):
    """Create a logits processor from a JSON schema.

    Parameters
    ----------
    json_schema: str
        The JSON schema to create a logits processor from.

    Returns
    -------
    LogitsProcessor
        The logits processor to use to constrain the generation.

    """
    regex = outlines_core.json_schema.build_regex_from_schema(json_schema)
    return self.get_regex_logits_processor(regex)

get_regex_logits_processor(regex)

Create a logits processor from a regex.

Parameters:

Name Type Description Default
regex str

The regex to create a logits processor from.

required

Returns:

Type Description
LogitsProcessor

The logits processor to use to constrain the generation.

Source code in outlines/backends/outlines_core.py
def get_regex_logits_processor(self, regex: str):
    """Create a logits processor from a regex.

    Parameters
    ----------
    regex: str
        The regex to create a logits processor from.

    Returns
    -------
    LogitsProcessor
        The logits processor to use to constrain the generation.

    """
    index = Index(regex, self.vocabulary)
    return OutlinesCoreLogitsProcessor(index, self.tensor_library_name)

OutlinesCoreLogitsProcessor

Bases: OutlinesLogitsProcessor

Logits processor for Outlines Core.

Source code in outlines/backends/outlines_core.py
class OutlinesCoreLogitsProcessor(OutlinesLogitsProcessor):
    """Logits processor for Outlines Core."""

    def __init__(
        self, index: Index, tensor_library_name: str
    ):
        """
        Parameters
        ----------
        index: Index
            The Outlines Core `Index` instance to use to create the Outlines
            Core `Guide` instances that will be used to bias the logits
        tensor_library_name: str
            The tensor library name to use for the logits processor.

        """
        self.index = index
        self.tensor_library_name = tensor_library_name
        self.is_first_token = True
        super().__init__(tensor_library_name)

    def reset(self) -> None:
        """Reset the logits processor."""
        self.is_first_token = True

    def _setup(self, batch_size: int, vocab_size: int) -> None:
        """Set the guides, bitmasks and some functions used in the
        `process_logits` method.

        This method is called when the first token is generated instead of
        at initialization because we need to know the batch size.

        """
        if self.tensor_library_name == "torch":
            from outlines_core.kernels.torch import allocate_token_bitmask

            self.allocate_token_bitmask = allocate_token_bitmask
            self.bias_logits = self._bias_logits_torch

        elif self.tensor_library_name == "numpy":
            from outlines_core.kernels.numpy import allocate_token_bitmask

            self.allocate_token_bitmask = allocate_token_bitmask
            self.bias_logits = self._bias_logits_numpy

        elif self.tensor_library_name == "mlx":
            from outlines_core.kernels.mlx import (
                allocate_token_bitmask
            )

            self.allocate_token_bitmask = allocate_token_bitmask
            self.bias_logits = self._bias_logits_mlx

        else:
            raise ValueError(
                f"Unsupported tensor library: {self.tensor_library_name}"
            )

        self._guides = [Guide(self.index) for _ in range(batch_size)]
        self._bitmasks = [
            self.allocate_token_bitmask(vocab_size)
            for _ in range(batch_size)
        ]

    def _bias_logits_mlx( # pragma: no cover
        self, batch_size: int, logits: TensorType
    ) -> TensorType:
        """Bias the logits for MLX tensors."""
        from outlines_core.kernels.mlx import (
            apply_token_bitmask,
            fill_next_token_bitmask
        )

        biased_logits_array = []
        for i in range(batch_size):
            fill_next_token_bitmask(self._guides[i], self._bitmasks[i])
            biased_logits = apply_token_bitmask(
                self.tensor_adapter.unsqueeze(logits[i]), self._bitmasks[i] # type: ignore
            )
            biased_logits_array.append(biased_logits)

        return self.tensor_adapter.concatenate(biased_logits_array)

    def _bias_logits_torch(
        self, batch_size: int, logits: TensorType
    ) -> TensorType:
        """Bias the logits for Torch tensors."""
        from outlines_core.kernels.torch import (
            apply_token_bitmask_inplace,
            fill_next_token_bitmask
        )

        for i in range(batch_size):
            fill_next_token_bitmask(self._guides[i], self._bitmasks[i])
            apply_token_bitmask_inplace(
                self.tensor_adapter.unsqueeze(logits[i]), # type: ignore
                self._bitmasks[i]
            )

        return logits

    def _bias_logits_numpy(
        self, batch_size: int, logits: TensorType
    ) -> TensorType:
        """Bias the logits for Numpy tensors."""
        from outlines_core.kernels.numpy import (
            apply_token_bitmask_inplace,
            fill_next_token_bitmask
        )

        for i in range(batch_size):
            fill_next_token_bitmask(self._guides[i], self._bitmasks[i])
            apply_token_bitmask_inplace(
                self.tensor_adapter.unsqueeze(logits[i]), # type: ignore
                self._bitmasks[i]
            )

        return logits

    def process_logits(
        self, input_ids: TensorType, logits: TensorType
    ) -> TensorType:
        """Use the guides to bias the logits.

        Parameters
        ----------
        input_ids
            The ids of the tokens of the existing sequences.
        logits
            The logits for the current generation step.

        Returns
        -------
        TensorType
            The biased logits.

        """
        batch_size = self.tensor_adapter.shape(input_ids)[0]
        vocab_size = self.tensor_adapter.shape(logits)[1]

        if self.is_first_token:
            self._setup(batch_size, vocab_size)
            self.is_first_token = False
        else:
            for i in range(batch_size):
                last_token_id = self.tensor_adapter.to_scalar(input_ids[i][-1]) # type: ignore
                if not self._guides[i].is_finished():
                    self._guides[i].advance(
                        token_id=last_token_id,
                        return_tokens=False
                    )

        return self.bias_logits(batch_size, logits)

__init__(index, tensor_library_name)

Parameters:

Name Type Description Default
index Index

The Outlines Core Index instance to use to create the Outlines Core Guide instances that will be used to bias the logits

required
tensor_library_name str

The tensor library name to use for the logits processor.

required
Source code in outlines/backends/outlines_core.py
def __init__(
    self, index: Index, tensor_library_name: str
):
    """
    Parameters
    ----------
    index: Index
        The Outlines Core `Index` instance to use to create the Outlines
        Core `Guide` instances that will be used to bias the logits
    tensor_library_name: str
        The tensor library name to use for the logits processor.

    """
    self.index = index
    self.tensor_library_name = tensor_library_name
    self.is_first_token = True
    super().__init__(tensor_library_name)

process_logits(input_ids, logits)

Use the guides to bias the logits.

Parameters:

Name Type Description Default
input_ids TensorType

The ids of the tokens of the existing sequences.

required
logits TensorType

The logits for the current generation step.

required

Returns:

Type Description
TensorType

The biased logits.

Source code in outlines/backends/outlines_core.py
def process_logits(
    self, input_ids: TensorType, logits: TensorType
) -> TensorType:
    """Use the guides to bias the logits.

    Parameters
    ----------
    input_ids
        The ids of the tokens of the existing sequences.
    logits
        The logits for the current generation step.

    Returns
    -------
    TensorType
        The biased logits.

    """
    batch_size = self.tensor_adapter.shape(input_ids)[0]
    vocab_size = self.tensor_adapter.shape(logits)[1]

    if self.is_first_token:
        self._setup(batch_size, vocab_size)
        self.is_first_token = False
    else:
        for i in range(batch_size):
            last_token_id = self.tensor_adapter.to_scalar(input_ids[i][-1]) # type: ignore
            if not self._guides[i].is_finished():
                self._guides[i].advance(
                    token_id=last_token_id,
                    return_tokens=False
                )

    return self.bias_logits(batch_size, logits)

reset()

Reset the logits processor.

Source code in outlines/backends/outlines_core.py
def reset(self) -> None:
    """Reset the logits processor."""
    self.is_first_token = True

xgrammar

Backend class for XGrammar.

XGrammarBackend

Bases: BaseBackend

Backend for XGRammar.

Source code in outlines/backends/xgrammar.py
class XGrammarBackend(BaseBackend):
    """Backend for XGRammar."""

    def __init__(self, model: SteerableModel):
        """
        Parameters
        ----------
        model
            The Outlines model of the user.

        """
        import xgrammar as xgr
        from transformers import AutoConfig

        if not isinstance(model, Transformers):
            raise ValueError(
                "The xgrammar backend only supports Transformers models"
            )

        vocab_size = AutoConfig.from_pretrained(
            model.model.config._name_or_path
        ).vocab_size
        tokenizer_info = xgr.TokenizerInfo.from_huggingface(
            model.hf_tokenizer,
            vocab_size=vocab_size
        )
        self.grammar_compiler = xgr.GrammarCompiler(tokenizer_info)

    def get_json_schema_logits_processor(
        self, json_schema: str
    ) -> "LogitsProcessor":
        """Create a logits processor from a JSON schema.

        Parameters
        ----------
        json_schema: str
            The JSON schema to create a logits processor from.

        Returns
        -------
        LogitsProcessor
            The logits processor to use to constrain the generation.

        """
        compiled_grammar = self.grammar_compiler.compile_json_schema(
            json_schema
        )
        return XGrammarLogitsProcessor(compiled_grammar)

    def get_regex_logits_processor(self, regex: str) -> "LogitsProcessor":
        """Create a logits processor from a regex.

        Parameters
        ----------
        regex: str
            The regex to create a logits processor from.

        Returns
        -------
        LogitsProcessor
            The logits processor to use to constrain the generation.

        """
        compiled_grammar = self.grammar_compiler.compile_regex(regex)
        return XGrammarLogitsProcessor(compiled_grammar)

    def get_cfg_logits_processor(self, grammar: str) -> "LogitsProcessor":
        """Create a logits processor from a context-free grammar.

        Parameters
        ----------
        grammar: str
            The context-free grammar to create a logits processor from.

        Returns
        -------
        LogitsProcessor
            The logits processor to use to constrain the generation.

        """
        compiled_grammar = self.grammar_compiler.compile_grammar(grammar)
        return XGrammarLogitsProcessor(compiled_grammar)

__init__(model)

Parameters:

Name Type Description Default
model SteerableModel

The Outlines model of the user.

required
Source code in outlines/backends/xgrammar.py
def __init__(self, model: SteerableModel):
    """
    Parameters
    ----------
    model
        The Outlines model of the user.

    """
    import xgrammar as xgr
    from transformers import AutoConfig

    if not isinstance(model, Transformers):
        raise ValueError(
            "The xgrammar backend only supports Transformers models"
        )

    vocab_size = AutoConfig.from_pretrained(
        model.model.config._name_or_path
    ).vocab_size
    tokenizer_info = xgr.TokenizerInfo.from_huggingface(
        model.hf_tokenizer,
        vocab_size=vocab_size
    )
    self.grammar_compiler = xgr.GrammarCompiler(tokenizer_info)

get_cfg_logits_processor(grammar)

Create a logits processor from a context-free grammar.

Parameters:

Name Type Description Default
grammar str

The context-free grammar to create a logits processor from.

required

Returns:

Type Description
LogitsProcessor

The logits processor to use to constrain the generation.

Source code in outlines/backends/xgrammar.py
def get_cfg_logits_processor(self, grammar: str) -> "LogitsProcessor":
    """Create a logits processor from a context-free grammar.

    Parameters
    ----------
    grammar: str
        The context-free grammar to create a logits processor from.

    Returns
    -------
    LogitsProcessor
        The logits processor to use to constrain the generation.

    """
    compiled_grammar = self.grammar_compiler.compile_grammar(grammar)
    return XGrammarLogitsProcessor(compiled_grammar)

get_json_schema_logits_processor(json_schema)

Create a logits processor from a JSON schema.

Parameters:

Name Type Description Default
json_schema str

The JSON schema to create a logits processor from.

required

Returns:

Type Description
LogitsProcessor

The logits processor to use to constrain the generation.

Source code in outlines/backends/xgrammar.py
def get_json_schema_logits_processor(
    self, json_schema: str
) -> "LogitsProcessor":
    """Create a logits processor from a JSON schema.

    Parameters
    ----------
    json_schema: str
        The JSON schema to create a logits processor from.

    Returns
    -------
    LogitsProcessor
        The logits processor to use to constrain the generation.

    """
    compiled_grammar = self.grammar_compiler.compile_json_schema(
        json_schema
    )
    return XGrammarLogitsProcessor(compiled_grammar)

get_regex_logits_processor(regex)

Create a logits processor from a regex.

Parameters:

Name Type Description Default
regex str

The regex to create a logits processor from.

required

Returns:

Type Description
LogitsProcessor

The logits processor to use to constrain the generation.

Source code in outlines/backends/xgrammar.py
def get_regex_logits_processor(self, regex: str) -> "LogitsProcessor":
    """Create a logits processor from a regex.

    Parameters
    ----------
    regex: str
        The regex to create a logits processor from.

    Returns
    -------
    LogitsProcessor
        The logits processor to use to constrain the generation.

    """
    compiled_grammar = self.grammar_compiler.compile_regex(regex)
    return XGrammarLogitsProcessor(compiled_grammar)

XGrammarLogitsProcessor

Bases: OutlinesLogitsProcessor

Logits processor for XGrammar.

This class wraps the xgr.contrib.hf.LogitsProcessor class and adds a reset method to reset the logits processor for a new generation.

Source code in outlines/backends/xgrammar.py
class XGrammarLogitsProcessor(OutlinesLogitsProcessor):
    """Logits processor for XGrammar.

    This class wraps the `xgr.contrib.hf.LogitsProcessor` class and adds
    a `reset` method to reset the logits processor for a new generation.

    """

    def __init__(self, compiled_grammar: str):
        """
        Parameters
        ----------
        compiled_grammar: str
            The compiled grammar to use to create the logits processor.

        """
        import xgrammar as xgr

        self.xgr = xgr
        self.compiled_grammar = compiled_grammar
        self.xgrammar_logits_processor = None
        super().__init__("torch")

    def reset(self):
        """Reset the logits processor for a new generation."""
        self.xgrammar_logits_processor = None

    def process_logits(self, input_ids: TensorType, logits: TensorType) -> TensorType:
        """Bias the logits."""
        if self.xgrammar_logits_processor is None:
            self.xgrammar_logits_processor = self.xgr.contrib.hf.LogitsProcessor(
                self.compiled_grammar
            )
        return self.xgrammar_logits_processor(input_ids, logits) # type: ignore

__init__(compiled_grammar)

Parameters:

Name Type Description Default
compiled_grammar str

The compiled grammar to use to create the logits processor.

required
Source code in outlines/backends/xgrammar.py
def __init__(self, compiled_grammar: str):
    """
    Parameters
    ----------
    compiled_grammar: str
        The compiled grammar to use to create the logits processor.

    """
    import xgrammar as xgr

    self.xgr = xgr
    self.compiled_grammar = compiled_grammar
    self.xgrammar_logits_processor = None
    super().__init__("torch")

process_logits(input_ids, logits)

Bias the logits.

Source code in outlines/backends/xgrammar.py
def process_logits(self, input_ids: TensorType, logits: TensorType) -> TensorType:
    """Bias the logits."""
    if self.xgrammar_logits_processor is None:
        self.xgrammar_logits_processor = self.xgr.contrib.hf.LogitsProcessor(
            self.compiled_grammar
        )
    return self.xgrammar_logits_processor(input_ids, logits) # type: ignore

reset()

Reset the logits processor for a new generation.

Source code in outlines/backends/xgrammar.py
def reset(self):
    """Reset the logits processor for a new generation."""
    self.xgrammar_logits_processor = None