Skip to content

outlines

Outlines is a Generative Model Programming Framework.

Application

Application is a class that encapsulates a prompt template and an output type. It can be called to generate a response by providing a model, the values to be substituted in the template in a dictionary and optional inference parameters.

Parameters:

Name Type Description Default
template Union[Template, Callable]

A callable that takes arguments and returns a prompt string.

 required
output_type Any

The expected output type of the generated response.

 None

Examples:

from pydantic import BaseModel
from transformers import AutoModelForCausalLM, AutoTokenizer
from outlines import models, Application
from outlines.types import JsonType
from outlines.templates import Template

class OutputModel(BaseModel):
    result: int

model = models.from_transformers(
    AutoModelForCausalLM.from_pretrained("microsoft/Phi-3-mini-4k-instruct"),
    AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
)

template_string = "What is 2 times {{ num }}?"
template = Template.from_string(template_string)

application = Application(template, JsonType(OutputModel))

result = application(model, {"num": 3}, max_new_tokens=20)
print(result)  # Expected output: { "result" : 6 }
Source code in outlines/applications.py 
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
class Application:
    """
    Application is a class that encapsulates a prompt template and an
    output type. It can be called to generate a response by providing a
    model, the values to be substituted in the template in a dictionary
    and optional inference parameters.

    Parameters
    ----------
    template : Union[Template, Callable]
        A callable that takes arguments and returns a prompt string.
    output_type : Any
        The expected output type of the generated response.

    Examples
    --------
    ```python
    from pydantic import BaseModel
    from transformers import AutoModelForCausalLM, AutoTokenizer
    from outlines import models, Application
    from outlines.types import JsonType
    from outlines.templates import Template

    class OutputModel(BaseModel):
        result: int

    model = models.from_transformers(
        AutoModelForCausalLM.from_pretrained("microsoft/Phi-3-mini-4k-instruct"),
        AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
    )

    template_string = "What is 2 times {{ num }}?"
    template = Template.from_string(template_string)

    application = Application(template, JsonType(OutputModel))

    result = application(model, {"num": 3}, max_new_tokens=20)
    print(result)  # Expected output: { "result" : 6 }
    ```

    """
    def __init__(
        self,
        template: Union[Template, Callable],
        output_type: Optional[Any] = None,
    ):
        """
        Parameters
        ----------
        template
            The template to use to build the prompt.
        output_type
            The output type provided to the generator.

        """
        self.template = template
        self.output_type = output_type
        self.generator: Optional[Union[
            BlackBoxGenerator, SteerableGenerator
        ]] = None
        self.model: Optional[Model] = None

    def __call__(
        self,
        model: Model,
        template_vars: Dict[str, Any],
        **inference_kwargs
    ) -> Any:
        """
        Parameters
        ----------
        model
            The model to use to generate the response.
        template_vars
            The variables to be substituted in the template.
        **inference_kwargs
            Additional keyword arguments to pass to the model.
        Returns
        -------
        Any
            The generated response.
        """
        if model is None:
            raise ValueError("you must provide a model")
        # We save the generator to avoid creating a new one for each call.
        # If the model has changed since the last call, we create a new
        # generator.
        if model != self.model:
            self.model = model
            self.generator = Generator(model, self.output_type)  # type: ignore

        prompt = self.template(**template_vars)
        assert self.generator is not None
        return self.generator(prompt, **inference_kwargs)

__call__(model, template_vars, **inference_kwargs)

Parameters:

Name Type Description Default
model Model

The model to use to generate the response.

 required
template_vars Dict[str, Any]

The variables to be substituted in the template.

 required
**inference_kwargs

Additional keyword arguments to pass to the model.

 {}

Returns:

Type Description
Any

The generated response.
Source code in outlines/applications.py 
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
def __call__(
    self,
    model: Model,
    template_vars: Dict[str, Any],
    **inference_kwargs
) -> Any:
    """
    Parameters
    ----------
    model
        The model to use to generate the response.
    template_vars
        The variables to be substituted in the template.
    **inference_kwargs
        Additional keyword arguments to pass to the model.
    Returns
    -------
    Any
        The generated response.
    """
    if model is None:
        raise ValueError("you must provide a model")
    # We save the generator to avoid creating a new one for each call.
    # If the model has changed since the last call, we create a new
    # generator.
    if model != self.model:
        self.model = model
        self.generator = Generator(model, self.output_type)  # type: ignore

    prompt = self.template(**template_vars)
    assert self.generator is not None
    return self.generator(prompt, **inference_kwargs)

__init__(template, output_type=None)

Parameters:

Name Type Description Default
template Union[Template, Callable]

The template to use to build the prompt.

 required
output_type Optional[Any]

The output type provided to the generator.

 None
Source code in outlines/applications.py 
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
def __init__(
    self,
    template: Union[Template, Callable],
    output_type: Optional[Any] = None,
):
    """
    Parameters
    ----------
    template
        The template to use to build the prompt.
    output_type
        The output type provided to the generator.

    """
    self.template = template
    self.output_type = output_type
    self.generator: Optional[Union[
        BlackBoxGenerator, SteerableGenerator
    ]] = None
    self.model: Optional[Model] = None

Template dataclass

Represents a prompt template.

We return a Template class instead of a simple function so the template can be accessed by callers.

Source code in outlines/templates.py 
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
@dataclass
class Template:
    """Represents a prompt template.

    We return a `Template` class instead of a simple function so the
    template can be accessed by callers.

    """
    template: jinja2.Template
    signature: Optional[inspect.Signature]

    def __call__(self, *args, **kwargs) -> str:
        """Render and return the template.

        Returns
        -------
        str
            The rendered template as a Python string.

        """
        if self.signature is not None:
            bound_arguments = self.signature.bind(*args, **kwargs)
            bound_arguments.apply_defaults()
            return self.template.render(**bound_arguments.arguments)
        else:
            return self.template.render(**kwargs)

    @classmethod
    def from_string(cls, content: str, filters: Dict[str, Callable] = {}):
        """Create a `Template` instance from a string containing a Jinja
        template.

        Parameters
        ----------
        content : str
            The string content to be converted into a template.

        Returns
        -------
        Template
            An instance of the class with the provided content as a template.

        """
        return cls(build_template_from_string(content, filters), None)

    @classmethod
    def from_file(cls, path: Path, filters: Dict[str, Callable] = {}):
        """Create a `Template` instance from a file containing a Jinja
        template.

        Note: This method does not allow to include and inheritance to
        reference files that are outside the folder or subfolders of the file
        given to `from_file`.

        Parameters
        ----------
        path : Path
            The path to the file containing the Jinja template.

        Returns
        -------
        Template
            An instance of the Template class with the template loaded from the
            file.

        """
        # We don't use a `Signature` here because it seems not feasible to
        # infer one from a Jinja2 environment that is
        # split across multiple files (since e.g. we support features like
        # Jinja2 includes and template inheritance)
        return cls(build_template_from_file(path, filters), None)

__call__(*args, **kwargs)

Render and return the template.

Returns:

Type Description
str

The rendered template as a Python string.
Source code in outlines/templates.py 
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
def __call__(self, *args, **kwargs) -> str:
    """Render and return the template.

    Returns
    -------
    str
        The rendered template as a Python string.

    """
    if self.signature is not None:
        bound_arguments = self.signature.bind(*args, **kwargs)
        bound_arguments.apply_defaults()
        return self.template.render(**bound_arguments.arguments)
    else:
        return self.template.render(**kwargs)

from_file(path, filters={}) classmethod

Create a Template instance from a file containing a Jinja template.

Note: This method does not allow to include and inheritance to reference files that are outside the folder or subfolders of the file given to from_file.

Parameters:

Name Type Description Default
path Path

The path to the file containing the Jinja template.

 required

Returns:

Type Description
Template

An instance of the Template class with the template loaded from the file.
Source code in outlines/templates.py 
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
@classmethod
def from_file(cls, path: Path, filters: Dict[str, Callable] = {}):
    """Create a `Template` instance from a file containing a Jinja
    template.

    Note: This method does not allow to include and inheritance to
    reference files that are outside the folder or subfolders of the file
    given to `from_file`.

    Parameters
    ----------
    path : Path
        The path to the file containing the Jinja template.

    Returns
    -------
    Template
        An instance of the Template class with the template loaded from the
        file.

    """
    # We don't use a `Signature` here because it seems not feasible to
    # infer one from a Jinja2 environment that is
    # split across multiple files (since e.g. we support features like
    # Jinja2 includes and template inheritance)
    return cls(build_template_from_file(path, filters), None)

from_string(content, filters={}) classmethod

Create a Template instance from a string containing a Jinja template.

Parameters:

Name Type Description Default
content str

The string content to be converted into a template.

 required

Returns:

Type Description
Template

An instance of the class with the provided content as a template.
Source code in outlines/templates.py 
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
@classmethod
def from_string(cls, content: str, filters: Dict[str, Callable] = {}):
    """Create a `Template` instance from a string containing a Jinja
    template.

    Parameters
    ----------
    content : str
        The string content to be converted into a template.

    Returns
    -------
    Template
        An instance of the class with the provided content as a template.

    """
    return cls(build_template_from_string(content, filters), None)

Generator(model, output_type=None, *, processor=None)

Create a generator for the given model and output parameters.

The 2 parameters output_type and processor are mutually exclusive. The parameters processor is only supported for SteerableModel instances (typically local models) and is intended to be only used by advanced users.

Parameters:

Name Type Description Default
model Union[Model, AsyncModel]

An instance of an Outlines model.

 required
output_type Optional[Any]

The output type expressed as a Python type or a type defined in the outlines.types.dsl module.

 None
processor Optional[OutlinesLogitsProcessor]

An instance of an OutlinesLogitsProcessor.

 None

Returns:

Type Description
Union[SteerableGenerator, BlackBoxGenerator, AsyncBlackBoxGenerator]

A generator instance.
Source code in outlines/generator.py 
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
def Generator(
    model: Union[Model, AsyncModel],
    output_type: Optional[Any] = None,
    *,
    processor: Optional[OutlinesLogitsProcessor] = None,
) -> Union[SteerableGenerator, BlackBoxGenerator, AsyncBlackBoxGenerator]:
    """Create a generator for the given model and output parameters.

    The 2 parameters output_type and processor are mutually exclusive. The
    parameters processor is only supported for SteerableModel instances
    (typically local models) and is intended to be only used by advanced users.

    Parameters
    ----------
    model
        An instance of an Outlines model.
    output_type
        The output type expressed as a Python type or a type defined in the
        outlines.types.dsl module.
    processor
        An instance of an OutlinesLogitsProcessor.

    Returns
    -------
    Union[SteerableGenerator, BlackBoxGenerator, AsyncBlackBoxGenerator]
        A generator instance.

    """
    provided_output_params = sum(
        param is not None
        for param in [output_type, processor]
    )
    if provided_output_params > 1:
        raise ValueError(
            "At most one of output_type or processor can be provided"
        )

    if isinstance(model, SteerableModel): # type: ignore
        if processor is not None:
            return SteerableGenerator.from_processor(model, processor) # type: ignore
        else:
            return SteerableGenerator(model, output_type) # type: ignore
    else:
        if processor is not None:
            raise NotImplementedError(
                "This model does not support logits processors"
            )
        if isinstance(model, AsyncBlackBoxModel): # type: ignore
            return AsyncBlackBoxGenerator(model, output_type) # type: ignore
        elif isinstance(model, BlackBoxModel): # type: ignore
            return BlackBoxGenerator(model, output_type) # type: ignore
        else:
            raise ValueError(
                "The model argument must be an instance of "
                "SteerableModel, BlackBoxModel or AsyncBlackBoxModel"
            )

clear_cache()

Erase the cache completely.

Source code in outlines/caching.py 
180
181
182
183
def clear_cache():
    """Erase the cache completely."""
    memory = get_cache()
    memory.clear()

disable_cache()

Disable the cache for this session.

Generative models output different results each time they are called when sampling. This can be a desirable property for some workflows, in which case one can call outlines.call.disable to disable the cache for the session.

This function does not delete the cache, call outlines.cache.clear instead. It also does not overwrite the cache with the values returned during the session.

Example

outlines.cache.disable should be called right after importing outlines:

import outlines.caching as cache cache.disable_cache()

Source code in outlines/caching.py 
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
def disable_cache():
    """Disable the cache for this session.

    Generative models output different results each time they are called when
    sampling. This can be a desirable property for some workflows, in which case
    one can call `outlines.call.disable` to disable the cache for the session.

    This function does not delete the cache, call `outlines.cache.clear`
    instead. It also does not overwrite the cache with the values returned
    during the session.

    Example
    -------

    `outlines.cache.disable` should be called right after importing outlines:

    >>> import outlines.caching as cache
    >>> cache.disable_cache()

    """
    global _caching_enabled
    _caching_enabled = False

get_cache() cached

Get the context object that contains previously-computed return values.

The cache is used to avoid unnecessary computations and API calls, which can be long and expensive for large models.

The cache directory defaults to HOMEDIR/.cache/outlines, but this choice can be overridden by the user by setting the value of the OUTLINES_CACHE_DIR environment variable.

Source code in outlines/caching.py 
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
@functools.lru_cache(1)
def get_cache():
    """Get the context object that contains previously-computed return values.

    The cache is used to avoid unnecessary computations and API calls, which can
    be long and expensive for large models.

    The cache directory defaults to `HOMEDIR/.cache/outlines`, but this choice
    can be overridden by the user by setting the value of the `OUTLINES_CACHE_DIR`
    environment variable.

    """
    from outlines._version import __version__ as outlines_version  # type: ignore

    outlines_cache_dir = os.environ.get("OUTLINES_CACHE_DIR")
    xdg_cache_home = os.environ.get("XDG_CACHE_HOME")
    home_dir = os.path.normpath(os.path.expanduser("~"))
    if outlines_cache_dir:
        # OUTLINES_CACHE_DIR takes precedence
        cache_dir = outlines_cache_dir
    elif xdg_cache_home:  # pragma: no cover
        cache_dir = os.path.join(xdg_cache_home, ".cache", "outlines")
    elif home_dir != "/":
        cache_dir = os.path.join(home_dir, ".cache", "outlines")
    else:  # pragma: no cover
        # home_dir may be / inside a docker container without existing user
        tempdir = tempfile.gettempdir()
        cache_dir = os.path.join(tempdir, ".cache", "outlines")

    memory = Cache(
        cache_dir,
        eviction_policy="none",
        cull_limit=0,
        disk=CloudpickleDisk,
    )

    # ensure if version upgrade occurs, old cache is pruned
    if outlines_version != memory.get("__version__"):
        memory.clear()
    memory["__version__"] = outlines_version

    return memory

prompt(fn=None, filters={})

Decorate a function that contains a prompt template.

This allows to define prompts in the docstring of a function and simplify their manipulation by providing some degree of encapsulation. It uses the render function internally to render templates.

>>> import outlines
>>>
>>> @outlines.prompt
>>> def build_prompt(question):
...    "I have a ${question}"
...
>>> prompt = build_prompt("How are you?")

This API can also be helpful in an "agent" context where parts of the prompt are set when the agent is initialized and never modified later. In this situation we can partially apply the prompt function at initialization.

>>> import outlines
>>> import functools as ft
...
>>> @outlines.prompt
... def solve_task(name: str, objective: str, task: str):
...     """Your name is {{name}}.
...     Your overall objective is to {{objective}}.
...     Please solve the following task: {{task}}
...     """
...
>>> hal = ft.partial(solve_task, "HAL", "Travel to Jupiter")

Additional Jinja2 filters can be provided as keyword arguments to the decorator.

>>> def reverse(s: str) -> str:
...     return s[::-1]
...
>>> @outlines.prompt(filters={ 'reverse': reverse })
... def reverse_prompt(text):
...     """{{ text | reverse }}"""
...
>>> prompt = reverse_prompt("Hello")
>>> print(prompt)
... "olleH"

Returns:

Type Description
A `Template` callable class which will render the template when called.
Source code in outlines/templates.py 
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
def prompt(
    fn: Optional[Callable] = None,
    filters: Dict[str, Callable] = {},
) -> Callable:
    """Decorate a function that contains a prompt template.

    This allows to define prompts in the docstring of a function and simplify their
    manipulation by providing some degree of encapsulation. It uses the `render`
    function internally to render templates.

    ```pycon
    >>> import outlines
    >>>
    >>> @outlines.prompt
    >>> def build_prompt(question):
    ...    "I have a ${question}"
    ...
    >>> prompt = build_prompt("How are you?")
    ```

    This API can also be helpful in an "agent" context where parts of the prompt
    are set when the agent is initialized and never modified later. In this situation
    we can partially apply the prompt function at initialization.

    ```pycon
    >>> import outlines
    >>> import functools as ft
    ...
    >>> @outlines.prompt
    ... def solve_task(name: str, objective: str, task: str):
    ...     \"""Your name is {{name}}.
    ...     Your overall objective is to {{objective}}.
    ...     Please solve the following task: {{task}}
    ...     \"""
    ...
    >>> hal = ft.partial(solve_task, "HAL", "Travel to Jupiter")
    ```

    Additional Jinja2 filters can be provided as keyword arguments to the decorator.

    ```pycon
    >>> def reverse(s: str) -> str:
    ...     return s[::-1]
    ...
    >>> @outlines.prompt(filters={ 'reverse': reverse })
    ... def reverse_prompt(text):
    ...     \"""{{ text | reverse }}\"""
    ...
    >>> prompt = reverse_prompt("Hello")
    >>> print(prompt)
    ... "olleH"
    ```

    Returns
    -------
    A `Template` callable class which will render the template when called.

    """
    warnings.warn(
        "The @prompt decorator is deprecated and will be removed in outlines 1.1.0. "
        "Instead of using docstring templates, please use Template.from_file() to "
        "load your prompts from separate template files, or a simple Python function "
        "that returns text. This helps keep prompt content separate from code and is "
        "more maintainable.",
        DeprecationWarning,
        stacklevel=2,
    )

    if fn is None:
        return lambda fn: prompt(fn, cast(Dict[str, Callable], filters))

    signature = inspect.signature(fn)

    # The docstring contains the template that will be rendered to be used
    # as a prompt to the language model.
    docstring = fn.__doc__
    if docstring is None:
        raise TypeError("Could not find a template in the function's docstring.")

    template = build_template_from_string(cast(str, docstring), filters)

    return Template(template, signature)

grammars

A few common Lark grammars.

read_grammar(grammar_file_name, base_grammar_path=GRAMMAR_PATH)

Read grammar file from default grammar path.

Parameters:

Name Type Description Default
grammar_file_name str

The name of the grammar file to read.

 required
base_grammar_path Path

The path to the directory containing the grammar file.

 GRAMMAR_PATH

Returns:

Type Description
str

The contents of the grammar file.
Source code in outlines/grammars.py 
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
def read_grammar(
    grammar_file_name: str,
    base_grammar_path: Path = GRAMMAR_PATH,
) -> str:
    """Read grammar file from default grammar path.

    Parameters
    ----------
    grammar_file_name
        The name of the grammar file to read.
    base_grammar_path
        The path to the directory containing the grammar file.

    Returns
    -------
    str
        The contents of the grammar file.

    """
    full_path = base_grammar_path / grammar_file_name
    with open(full_path) as file:
        return file.read()