Skip to content

ollama

Integration with the ollama library.

Ollama

Bases: Model

Thin wrapper around the ollama.Client client.

This wrapper is used to convert the input and output types specified by the users at a higher level to arguments to the ollama.Client client.

Source code in outlines/models/ollama.py 
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
class Ollama(Model):
    """Thin wrapper around the `ollama.Client` client.

    This wrapper is used to convert the input and output types specified by the
    users at a higher level to arguments to the `ollama.Client` client.

    """

    def __init__(
        self,client: "OllamaClient", model_name: Optional[str] = None,
    ):
        """
        Parameters
        ----------
        client
            The `ollama.Client` client.
        model_name
            The name of the model to use.

        """
        self.client = client
        self.model_name = model_name
        self.type_adapter = OllamaTypeAdapter()

    def generate(self,
        model_input: str,
        output_type: Optional[Any] = None,
        **kwargs: Any,
    ) -> str:
        """Generate text using Ollama.

        Parameters
        ----------
        model_input
            The prompt based on which the model will generate a response.
        output_type
            The desired format of the response generated by the model. The
            output type must be of a type that can be converted to a JSON
            schema.
        **kwargs
            Additional keyword arguments to pass to the client.

        Returns
        -------
        str
            The text generated by the model.

        """
        if "model" not in kwargs and self.model_name is not None:
            kwargs["model"] = self.model_name

        response = self.client.generate(
            prompt=self.type_adapter.format_input(model_input),
            format=self.type_adapter.format_output_type(output_type),
            **kwargs,
        )
        return response.response

    def generate_stream(
        self,
        model_input: str,
        output_type: Optional[Any] = None,
        **kwargs: Any,
    ) -> Iterator[str]:
        """Stream text using Ollama.

        Parameters
        ----------
        model_input
            The prompt based on which the model will generate a response.
        output_type
            The desired format of the response generated by the model. The
            output type must be of a type that can be converted to a JSON
            schema.
        **kwargs
            Additional keyword arguments to pass to the client.

        Returns
        -------
        Iterator[str]
            An iterator that yields the text generated by the model.

        """
        if "model" not in kwargs and self.model_name is not None:
            kwargs["model"] = self.model_name

        response = self.client.generate(
            prompt=self.type_adapter.format_input(model_input),
            format=self.type_adapter.format_output_type(output_type),
            stream=True,
            **kwargs,
        )
        for chunk in response:
            yield chunk.response

__init__(client, model_name=None)

Parameters:

Name Type Description Default
client Client

The ollama.Client client.

 required
model_name Optional[str]

The name of the model to use.

 None
Source code in outlines/models/ollama.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
def __init__(
    self,client: "OllamaClient", model_name: Optional[str] = None,
):
    """
    Parameters
    ----------
    client
        The `ollama.Client` client.
    model_name
        The name of the model to use.

    """
    self.client = client
    self.model_name = model_name
    self.type_adapter = OllamaTypeAdapter()

generate(model_input, output_type=None, **kwargs)

Generate text using Ollama.

Parameters:

Name Type Description Default
model_input str

The prompt based on which the model will generate a response.

 required
output_type Optional[Any]

The desired format of the response generated by the model. The output type must be of a type that can be converted to a JSON schema.

 None
**kwargs Any

Additional keyword arguments to pass to the client.

 {}

Returns:

Type Description
str

The text generated by the model.
Source code in outlines/models/ollama.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
def generate(self,
    model_input: str,
    output_type: Optional[Any] = None,
    **kwargs: Any,
) -> str:
    """Generate text using Ollama.

    Parameters
    ----------
    model_input
        The prompt based on which the model will generate a response.
    output_type
        The desired format of the response generated by the model. The
        output type must be of a type that can be converted to a JSON
        schema.
    **kwargs
        Additional keyword arguments to pass to the client.

    Returns
    -------
    str
        The text generated by the model.

    """
    if "model" not in kwargs and self.model_name is not None:
        kwargs["model"] = self.model_name

    response = self.client.generate(
        prompt=self.type_adapter.format_input(model_input),
        format=self.type_adapter.format_output_type(output_type),
        **kwargs,
    )
    return response.response

generate_stream(model_input, output_type=None, **kwargs)

Stream text using Ollama.

Parameters:

Name Type Description Default
model_input str

The prompt based on which the model will generate a response.

 required
output_type Optional[Any]

The desired format of the response generated by the model. The output type must be of a type that can be converted to a JSON schema.

 None
**kwargs Any

Additional keyword arguments to pass to the client.

 {}

Returns:

Type Description
Iterator[str]

An iterator that yields the text generated by the model.
Source code in outlines/models/ollama.py 
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
def generate_stream(
    self,
    model_input: str,
    output_type: Optional[Any] = None,
    **kwargs: Any,
) -> Iterator[str]:
    """Stream text using Ollama.

    Parameters
    ----------
    model_input
        The prompt based on which the model will generate a response.
    output_type
        The desired format of the response generated by the model. The
        output type must be of a type that can be converted to a JSON
        schema.
    **kwargs
        Additional keyword arguments to pass to the client.

    Returns
    -------
    Iterator[str]
        An iterator that yields the text generated by the model.

    """
    if "model" not in kwargs and self.model_name is not None:
        kwargs["model"] = self.model_name

    response = self.client.generate(
        prompt=self.type_adapter.format_input(model_input),
        format=self.type_adapter.format_output_type(output_type),
        stream=True,
        **kwargs,
    )
    for chunk in response:
        yield chunk.response

OllamaTypeAdapter

Bases: ModelTypeAdapter

Type adapter for the Ollama model.

Source code in outlines/models/ollama.py 
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
class OllamaTypeAdapter(ModelTypeAdapter):
    """Type adapter for the `Ollama` model."""

    def format_input(self, model_input: str) -> str:
        """Generate the prompt argument to pass to the model.

        Parameters
        ----------
        model_input
            The input provided by the user.

        Returns
        -------
        str
            The formatted input to be passed to the model.

        """
        if isinstance(model_input, str):
            return model_input
        raise TypeError(
            f"The input type {model_input} is not available. "
            "Ollama does not support batch inference."
        )

    def format_output_type(self, output_type: Optional[Any] = None) -> Optional[str]:
        """Format the output type to pass to the client.

        TODO: `int`, `float` and other Python types could be supported via
        JSON Schema.

        Parameters
        ----------
        output_type
            The output type provided by the user.

        Returns
        -------
        Optional[str]
            The formatted output type to be passed to the model.

        """
        if isinstance(output_type, Regex):
            raise TypeError(
                "Regex-based structured outputs are not supported by Ollama. "
                "Use an open source model in the meantime."
            )
        elif isinstance(output_type, CFG):
            raise TypeError(
                "CFG-based structured outputs are not supported by Ollama. "
                "Use an open source model in the meantime."
            )

        if output_type is None:
            return None
        elif isinstance(output_type, JsonSchema):
            return json.loads(output_type.schema)
        elif is_dataclass(output_type):
            schema = TypeAdapter(output_type).json_schema()
            return schema
        elif is_typed_dict(output_type):
            schema = TypeAdapter(output_type).json_schema()
            return schema
        elif is_pydantic_model(output_type):
            schema = output_type.model_json_schema()
            return schema
        elif is_genson_schema_builder(output_type):
            return output_type.to_json()
        else:
            type_name = getattr(output_type, "__name__", output_type)
            raise TypeError(
                f"The type `{type_name}` is not supported by Ollama. "
                "Consider using a local model instead."
            )

format_input(model_input)

Generate the prompt argument to pass to the model.

Parameters:

Name Type Description Default
model_input str

The input provided by the user.

 required

Returns:

Type Description
str

The formatted input to be passed to the model.
Source code in outlines/models/ollama.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
def format_input(self, model_input: str) -> str:
    """Generate the prompt argument to pass to the model.

    Parameters
    ----------
    model_input
        The input provided by the user.

    Returns
    -------
    str
        The formatted input to be passed to the model.

    """
    if isinstance(model_input, str):
        return model_input
    raise TypeError(
        f"The input type {model_input} is not available. "
        "Ollama does not support batch inference."
    )

format_output_type(output_type=None)

Format the output type to pass to the client.

TODO: int, float and other Python types could be supported via JSON Schema.

Parameters:

Name Type Description Default
output_type Optional[Any]

The output type provided by the user.

 None

Returns:

Type Description
Optional[str]

The formatted output type to be passed to the model.
Source code in outlines/models/ollama.py 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
def format_output_type(self, output_type: Optional[Any] = None) -> Optional[str]:
    """Format the output type to pass to the client.

    TODO: `int`, `float` and other Python types could be supported via
    JSON Schema.

    Parameters
    ----------
    output_type
        The output type provided by the user.

    Returns
    -------
    Optional[str]
        The formatted output type to be passed to the model.

    """
    if isinstance(output_type, Regex):
        raise TypeError(
            "Regex-based structured outputs are not supported by Ollama. "
            "Use an open source model in the meantime."
        )
    elif isinstance(output_type, CFG):
        raise TypeError(
            "CFG-based structured outputs are not supported by Ollama. "
            "Use an open source model in the meantime."
        )

    if output_type is None:
        return None
    elif isinstance(output_type, JsonSchema):
        return json.loads(output_type.schema)
    elif is_dataclass(output_type):
        schema = TypeAdapter(output_type).json_schema()
        return schema
    elif is_typed_dict(output_type):
        schema = TypeAdapter(output_type).json_schema()
        return schema
    elif is_pydantic_model(output_type):
        schema = output_type.model_json_schema()
        return schema
    elif is_genson_schema_builder(output_type):
        return output_type.to_json()
    else:
        type_name = getattr(output_type, "__name__", output_type)
        raise TypeError(
            f"The type `{type_name}` is not supported by Ollama. "
            "Consider using a local model instead."
        )

from_ollama(client, model_name=None)

Create an Outlines Ollama model instance from an ollama.Client client.

Parameters:

Name Type Description Default
client Client

A ollama.Client client instance.

 required
model_name Optional[str]

The name of the model to use.

 None

Returns:

Type Description
Ollama

An Outlines Ollama model instance.
Source code in outlines/models/ollama.py 
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
def from_ollama(
    client: "OllamaClient", model_name: Optional[str] = None
) -> Ollama:
    """Create an Outlines `Ollama` model instance from an `ollama.Client`
    client.

    Parameters
    ----------
    client
        A `ollama.Client` client instance.
    model_name
        The name of the model to use.

    Returns
    -------
    Ollama
        An Outlines `Ollama` model instance.

    """
    return Ollama(client, model_name)