diff --git a/docs/12 - OpenAI API.md b/docs/12 - OpenAI API.md index 81b3a8f9..d792c2cf 100644 --- a/docs/12 - OpenAI API.md +++ b/docs/12 - OpenAI API.md @@ -23,20 +23,6 @@ For the documentation with all the endpoints, parameters and their types, consul The official examples in the [OpenAI documentation](https://platform.openai.com/docs/api-reference) should also work, and the same parameters apply (although the API here has more optional parameters). -#### Completions - -```shell -curl http://127.0.0.1:5000/v1/completions \ - -H "Content-Type: application/json" \ - -d '{ - "prompt": "This is a cake recipe:\n\n1.", - "max_tokens": 512, - "temperature": 0.6, - "top_p": 0.95, - "top_k": 20 - }' -``` - #### Chat completions Works best with instruction-following models. If the "instruction_template" variable is not provided, it will be detected automatically from the model metadata. @@ -57,7 +43,21 @@ curl http://127.0.0.1:5000/v1/chat/completions \ }' ``` -#### Chat completions with characters +#### Completions + +```shell +curl http://127.0.0.1:5000/v1/completions \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "This is a cake recipe:\n\n1.", + "max_tokens": 512, + "temperature": 0.6, + "top_p": 0.95, + "top_k": 20 + }' +``` + +#### SSE streaming ```shell curl http://127.0.0.1:5000/v1/chat/completions \ @@ -66,17 +66,105 @@ curl http://127.0.0.1:5000/v1/chat/completions \ "messages": [ { "role": "user", - "content": "Hello! Who are you?" + "content": "Hello!" } ], - "mode": "chat-instruct", - "character": "Example", "temperature": 0.6, "top_p": 0.95, - "top_k": 20 + "top_k": 20, + "stream": true }' ``` +#### Tool/Function calling + +Use a model with tool calling support (Qwen, Mistral, GPT-OSS, etc). Tools are passed via the `tools` parameter and the prompt is automatically formatted using the model's Jinja2 template. + +When the model decides to call a tool, the response will have `finish_reason: "tool_calls"` and a `tool_calls` array with structured function names and arguments. You then execute the tool, send the result back as a `role: "tool"` message, and continue until the model responds with `finish_reason: "stop"`. + +Some models call multiple tools in parallel (Qwen, Mistral), while others call one at a time (GPT-OSS). The loop below handles both styles. + +```python +import json +import requests + +url = "http://127.0.0.1:5000/v1/chat/completions" + +# Define your tools +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a given location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City name"}, + }, + "required": ["location"] + } + } + }, + { + "type": "function", + "function": { + "name": "get_time", + "description": "Get the current time in a given timezone", + "parameters": { + "type": "object", + "properties": { + "timezone": {"type": "string", "description": "IANA timezone string"}, + }, + "required": ["timezone"] + } + } + }, +] + + +def execute_tool(name, arguments): + """Replace this with your actual tool implementations.""" + if name == "get_weather": + return {"temperature": 22, "condition": "sunny", "humidity": 45} + elif name == "get_time": + return {"time": "2:30 PM", "timezone": "JST"} + return {"error": f"Unknown tool: {name}"} + + +messages = [{"role": "user", "content": "What time is it in Tokyo and what's the weather like there?"}] + +# Tool-calling loop: keep going until the model gives a final answer +for _ in range(10): + response = requests.post(url, json={"messages": messages, "tools": tools}).json() + choice = response["choices"][0] + + if choice["finish_reason"] == "tool_calls": + # Add the assistant's response (with tool_calls) to history + messages.append({ + "role": "assistant", + "content": choice["message"]["content"], + "tool_calls": choice["message"]["tool_calls"], + }) + + # Execute each tool and add results to history + for tool_call in choice["message"]["tool_calls"]: + name = tool_call["function"]["name"] + arguments = json.loads(tool_call["function"]["arguments"]) + result = execute_tool(name, arguments) + + print(f"Tool call: {name}({arguments}) => {result}") + messages.append({ + "role": "tool", + "tool_call_id": tool_call["id"], + "content": json.dumps(result), + }) + else: + # Final answer + print(f"\nAssistant: {choice['message']['content']}") + break +``` + #### Multimodal/vision (llama.cpp and ExLlamaV3) ##### With /v1/chat/completions (recommended!) @@ -139,77 +227,6 @@ curl http://127.0.0.1:5000/v1/completions \ For base64-encoded images, just replace the inner "url" values with this format: `data:image/FORMAT;base64,BASE64_STRING` where FORMAT is the file type (png, jpeg, gif, etc.) and BASE64_STRING is your base64-encoded image data. -#### Image generation - -```shell -curl http://127.0.0.1:5000/v1/images/generations \ - -H "Content-Type: application/json" \ - -d '{ - "prompt": "an orange tree", - "steps": 9, - "cfg_scale": 0, - "batch_size": 1, - "batch_count": 1 - }' -``` - -You need to load an image model first. You can do this via the UI, or by adding `--image-model your_model_name` when launching the server. - -The output is a JSON object containing a `data` array. Each element has a `b64_json` field with the base64-encoded PNG image: - -```json -{ - "created": 1764791227, - "data": [ - { - "b64_json": "iVBORw0KGgo..." - } - ] -} -``` - -#### SSE streaming - -```shell -curl http://127.0.0.1:5000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ], - "temperature": 0.6, - "top_p": 0.95, - "top_k": 20, - "stream": true - }' -``` - -#### Logits - -```shell -curl -k http://127.0.0.1:5000/v1/internal/logits \ - -H "Content-Type: application/json" \ - -d '{ - "prompt": "Who is best, Asuka or Rei? Answer:", - "use_samplers": false - }' -``` - -#### Logits after sampling parameters - -```shell -curl -k http://127.0.0.1:5000/v1/internal/logits \ - -H "Content-Type: application/json" \ - -d '{ - "prompt": "Who is best, Asuka or Rei? Answer:", - "use_samplers": true, - "top_k": 3 - }' -``` - #### List models ```shell @@ -242,6 +259,78 @@ curl -k http://127.0.0.1:5000/v1/internal/model/load \ }' ``` +#### Chat completions with characters + +```shell +curl http://127.0.0.1:5000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [ + { + "role": "user", + "content": "Hello! Who are you?" + } + ], + "mode": "chat-instruct", + "character": "Example", + "temperature": 0.6, + "top_p": 0.95, + "top_k": 20 + }' +``` + +#### Image generation + +```shell +curl http://127.0.0.1:5000/v1/images/generations \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "an orange tree", + "steps": 9, + "cfg_scale": 0, + "batch_size": 1, + "batch_count": 1 + }' +``` + +You need to load an image model first. You can do this via the UI, or by adding `--image-model your_model_name` when launching the server. + +The output is a JSON object containing a `data` array. Each element has a `b64_json` field with the base64-encoded PNG image: + +```json +{ + "created": 1764791227, + "data": [ + { + "b64_json": "iVBORw0KGgo..." + } + ] +} +``` + +#### Logits + +```shell +curl -k http://127.0.0.1:5000/v1/internal/logits \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "Who is best, Asuka or Rei? Answer:", + "use_samplers": false + }' +``` + +#### Logits after sampling parameters + +```shell +curl -k http://127.0.0.1:5000/v1/internal/logits \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "Who is best, Asuka or Rei? Answer:", + "use_samplers": true, + "top_k": 3 + }' +``` + #### Python chat example ```python @@ -348,6 +437,27 @@ for event in client.events(): print() ``` +#### Python example with API key + +Replace + +```python +headers = { + "Content-Type": "application/json" +} +``` + +with + +```python +headers = { + "Content-Type": "application/json", + "Authorization": "Bearer yourPassword123" +} +``` + +in any of the examples above. + #### Python parallel requests example The API supports handling multiple requests in parallel. For ExLlamaV3, this works out of the box. For llama.cpp, you need to pass `--parallel N` to set the number of concurrent slots. @@ -377,116 +487,6 @@ for prompt, result in zip(prompts, results): print(f"Q: {prompt}\nA: {result}\n") ``` -#### Python example with API key - -Replace - -```python -headers = { - "Content-Type": "application/json" -} -``` - -with - -```python -headers = { - "Content-Type": "application/json", - "Authorization": "Bearer yourPassword123" -} -``` - -in any of the examples above. - -#### Tool/Function calling - -Use a model with tool calling support (Qwen, Mistral, GPT-OSS, etc). Tools are passed via the `tools` parameter and the prompt is automatically formatted using the model's Jinja2 template. - -When the model decides to call a tool, the response will have `finish_reason: "tool_calls"` and a `tool_calls` array with structured function names and arguments. You then execute the tool, send the result back as a `role: "tool"` message, and continue until the model responds with `finish_reason: "stop"`. - -Some models call multiple tools in parallel (Qwen, Mistral), while others call one at a time (GPT-OSS). The loop below handles both styles. - -```python -import json -import requests - -url = "http://127.0.0.1:5000/v1/chat/completions" - -# Define your tools -tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a given location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "City name"}, - }, - "required": ["location"] - } - } - }, - { - "type": "function", - "function": { - "name": "get_time", - "description": "Get the current time in a given timezone", - "parameters": { - "type": "object", - "properties": { - "timezone": {"type": "string", "description": "IANA timezone string"}, - }, - "required": ["timezone"] - } - } - }, -] - - -def execute_tool(name, arguments): - """Replace this with your actual tool implementations.""" - if name == "get_weather": - return {"temperature": 22, "condition": "sunny", "humidity": 45} - elif name == "get_time": - return {"time": "2:30 PM", "timezone": "JST"} - return {"error": f"Unknown tool: {name}"} - - -messages = [{"role": "user", "content": "What time is it in Tokyo and what's the weather like there?"}] - -# Tool-calling loop: keep going until the model gives a final answer -for _ in range(10): - response = requests.post(url, json={"messages": messages, "tools": tools}).json() - choice = response["choices"][0] - - if choice["finish_reason"] == "tool_calls": - # Add the assistant's response (with tool_calls) to history - messages.append({ - "role": "assistant", - "content": choice["message"]["content"], - "tool_calls": choice["message"]["tool_calls"], - }) - - # Execute each tool and add results to history - for tool_call in choice["message"]["tool_calls"]: - name = tool_call["function"]["name"] - arguments = json.loads(tool_call["function"]["arguments"]) - result = execute_tool(name, arguments) - - print(f"Tool call: {name}({arguments}) => {result}") - messages.append({ - "role": "tool", - "tool_call_id": tool_call["id"], - "content": json.dumps(result), - }) - else: - # Final answer - print(f"\nAssistant: {choice['message']['content']}") - break -``` - ### Environment variables The following environment variables can be used (they take precedence over everything else):