upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-23 11:10:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

no more json in header

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-07-07 22:10:31 +02:00
parent a87b2d77cf
commit 50ed8076fb
3 changed files with 63 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
tools/cli/cli-client.cpp
View File

@@ -25,7 +25,7 @@ static std::string join_path(const common_http_url & parts, const std::string &
return prefix + path;
}
json cli_client::get(const std::string & path) {
std::string cli_client::get(const std::string & path) {
auto [cli, parts] = common_http_client(server_base);
cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);
auto path_with_model = path + (model.empty() ? "" : ("?model=" + model));
@@ -36,38 +36,26 @@ json cli_client::get(const std::string & path) {
if (res->status < 200 || res->status >= 300) {
throw std::runtime_error("GET " + path + " failed with status " + std::to_string(res->status) + ": " + res->body);
}
json result = json::parse(res->body, nullptr, false);
if (result.is_discarded()) {
throw std::runtime_error("GET " + path + " returned invalid JSON");
}
return result;
return res->body;
}
json cli_client::post(const std::string & path, const json & body) {
std::string cli_client::post(const std::string & path, const std::string & body) {
auto [cli, parts] = common_http_client(server_base);
cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);
auto body_with_model = body;
if (!model.empty()) {
body_with_model["model"] = model;
}
auto res = cli.Post(join_path(parts, path), body_with_model.dump(), "application/json");
auto res = cli.Post(join_path(parts, path), body, "application/json");
if (!res) {
throw std::runtime_error("failed to connect to " + server_base + ": " + httplib::to_string(res.error()));
}
if (res->status < 200 || res->status >= 300) {
throw std::runtime_error("POST " + path + " failed with status " + std::to_string(res->status) + ": " + res->body);
}
json result = json::parse(res->body, nullptr, false);
if (result.is_discarded()) {
throw std::runtime_error("POST " + path + " returned invalid JSON");
}
return result;
return res->body;
}
json cli_client::post_sse(const std::string & path,
const json & body,
const std::function<bool()> & should_stop,
const std::function<void(const json &)> & on_data) {
std::string cli_client::post_sse(const std::string & path,
const std::string & body,
const std::function<bool()> & should_stop,
const std::function<void(const std::string &)> & on_data) {
auto [cli, parts] = common_http_client(server_base);
cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);
@@ -96,35 +84,27 @@ json cli_client::post_sse(const std::string & path,
if (payload == "[DONE]") {
continue;
}
json event = json::parse(payload, nullptr, false);
if (!event.is_discarded()) {
on_data(event);
}
on_data(payload);
}
return true;
};
httplib::Headers headers = {{"Accept", "text/event-stream"}};
auto body_with_model = body;
if (!model.empty()) {
body_with_model["model"] = model;
}
auto res = cli.Post(join_path(parts, path), headers, body_with_model.dump(), "application/json", receiver);
auto res = cli.Post(join_path(parts, path), headers, body, "application/json", receiver);
if (!res) {
if (res.error() == httplib::Error::Canceled && should_stop()) {
return json(); // cancelled by the user
return ""; // cancelled by the user
}
return json {{"error", {{"message", "failed to connect to " + server_base + ": " + httplib::to_string(res.error())}}}};
return "failed to connect to " + server_base + ": " + httplib::to_string(res.error());
}
if (res->status < 200 || res->status >= 300) {
json error_body = json::parse(raw_body, nullptr, false);
if (!error_body.is_discarded() && error_body.contains("error")) {
return error_body;
if (!raw_body.empty()) {
return raw_body;
}
return json {{"error", {{"message", "request failed with status " + std::to_string(res->status)}}}};
return "request failed with status " + std::to_string(res->status);
}
return json();
return "";
}
bool cli_client::wait_health(const std::function<bool()> & is_aborted) {
@@ -148,17 +128,3 @@ bool cli_client::wait_health(const std::function<bool()> & is_aborted) {
last_error = "aborted while waiting for the server to become ready";
return false;
}
std::vector<std::string> cli_client::list_models() {
json resp = get("/v1/models");
if (!resp.contains("data") || !resp.at("data").is_array()) {
throw std::runtime_error("invalid response from /v1/models");
}
std::vector<std::string> models;
for (const auto & m : resp.at("data")) {
if (m.contains("id") && m.at("id").is_string()) {
models.push_back(m.at("id").get<std::string>());
}
}
return models;
}

47
tools/cli/cli-client.h
View File

@@ -1,15 +1,8 @@
#pragma once
#include "ggml.h"
#define JSON_ASSERT GGML_ASSERT
#include <nlohmann/json.hpp>
#include <functional>
#include <string>
using json = nlohmann::ordered_json;
// openai-like client for CLI
struct cli_client {
std::string server_base; // base url, for example "http://127.0.0.1:8080"
@@ -17,40 +10,24 @@ struct cli_client {
std::string model; // optional, set when the server has multiple models (router mode)
// simple GET request, returns the response json
// simple GET request, returns the raw response body
// throws std::runtime_error on transport error or non-2xx status
json get(const std::string & path);
std::string get(const std::string & path);
// simple POST request, returns the response json
// simple POST request, returns the raw response body
// throws std::runtime_error on transport error or non-2xx status
json post(const std::string & path, const json & body);
std::string post(const std::string & path, const std::string & body);
// POST request with an SSE streaming response; on_data is invoked once
// per "data:" event; the function returns after the stream is finished:
// a null json on graceful exit (incl. cancellation via should_stop),
// the error response json otherwise
json post_sse(const std::string & path,
const json & body,
const std::function<bool()> & should_stop,
const std::function<void(const json &)> & on_data);
// POST request with an SSE streaming response
// on_data is invoked per "data:" event with the raw event payload
// returns after the stream is finished (empty string on graceful exit)
// otherwise, the raw error response body
std::string post_sse(const std::string & path,
const std::string & body,
const std::function<bool()> & should_stop,
const std::function<void(const std::string &)> & on_data);
// poll /health until the server is ready to accept requests
// returns false if is_aborted returned true or the server is unreachable
bool wait_health(const std::function<bool()> & is_aborted);
//
// higher-level wrappers
//
json create_chat_completion(const json & request,
const std::function<bool()> & should_stop,
const std::function<void(const json &)> & on_data) {
return post_sse("/v1/chat/completions", request, should_stop, on_data);
}
json get_props() {
return get("/props");
}
std::vector<std::string> list_models();
};

38
tools/cli/cli-context.cpp
View File

@@ -6,12 +6,17 @@
#include "log.h"
#include "console.h"
#define JSON_ASSERT GGML_ASSERT
#include <nlohmann/json.hpp>
#include <algorithm>
#include <filesystem>
#include <fstream>
#include <map>
#include <set>
using json = nlohmann::ordered_json;
struct cli_context_impl {
json messages = json::array();
json pending_media = json::array(); // staged multimodal content parts
@@ -65,6 +70,15 @@ static std::string format_error_message(const json & err) {
return err.dump();
}
// err is the raw response body of a failed request; it may or may not be JSON
static std::string format_error_message(const std::string & err) {
json parsed = json::parse(err, nullptr, false);
if (!parsed.is_discarded()) {
return format_error_message(parsed);
}
return err;
}
static std::string media_type_from_ext(const std::string & fname) {
std::string ext = std::filesystem::path(fname).extension().string();
std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
@@ -152,7 +166,7 @@ bool cli_context::init() {
void cli_context::fetch_server_props() {
try {
json props = client.get_props();
json props = json::parse(client.get("/props"));
model_name = props.value("model_alias", "");
if (model_name.empty()) {
const std::string path = props.value("model_path", "");
@@ -175,7 +189,16 @@ void cli_context::fetch_server_props() {
}
bool cli_context::list_and_ask_models() {
auto models = client.list_models();
json resp = json::parse(client.get("/v1/models"));
if (!resp.contains("data") || !resp.at("data").is_array()) {
throw std::runtime_error("invalid response from /v1/models");
}
std::vector<std::string> models;
for (const auto & m : resp.at("data")) {
if (m.contains("id") && m.at("id").is_string()) {
models.push_back(m.at("id").get<std::string>());
}
}
// only one model: use it without asking
if (models.size() == 1) {
@@ -292,12 +315,19 @@ bool cli_context::generate_completion(std::string & assistant_content, cli_timin
// in order to get timings even when we cancel mid-way
{"timings_per_token", true},
};
if (!client.model.empty()) {
body["model"] = client.model;
}
bool stream_error = false;
ui::assistant_turn a;
json err = client.create_chat_completion(body, should_stop, [&](const json & chunk) {
std::string err = client.post_sse("/v1/chat/completions", body.dump(), should_stop, [&](const std::string & payload) {
json chunk = json::parse(payload, nullptr, false);
if (chunk.is_discarded()) {
return;
}
if (chunk.contains("error")) {
stream_error = true;
ui::show_error(format_error_message(chunk));
@@ -333,7 +363,7 @@ bool cli_context::generate_completion(std::string & assistant_content, cli_timin
cli_context::interrupted().store(false);
if (!err.is_null()) {
if (!err.empty()) {
ui::show_error(format_error_message(err));
return false;
}