upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-23 11:10:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-06-23 13:28:14 +02:00
parent 59797670dc
commit f7421eabe8
6 changed files with 20 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tools/cli/CMakeLists.txt
View File

@@ -8,7 +8,7 @@ add_library(${TARGET} cli.cpp
set_target_properties(${TARGET} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ../server)
target_link_libraries(${TARGET} PUBLIC server-context llama-common ${CMAKE_THREAD_LIBS_INIT})
target_link_libraries(${TARGET} PUBLIC llama-server-impl llama-common ${CMAKE_THREAD_LIBS_INIT})
if(LLAMA_TOOLS_INSTALL)
install(TARGETS ${TARGET} LIBRARY)

54
tools/cli/cli-context.cpp
View File

@@ -41,56 +41,6 @@ static int arg_num_values(const common_arg & opt) {
return 0;
}
// keep only the args that llama-server understands, so that the remainder
// of the command line can be forwarded to the spawned server child
static std::vector<std::string> filter_server_args(int argc, char ** argv) {
std::map<std::string, int> cli_n_values; // arg -> number of values
std::set<std::string> server_args;
common_params dummy_cli;
auto ctx_cli = common_params_parser_init(dummy_cli, LLAMA_EXAMPLE_CLI);
for (const auto & opt : ctx_cli.options) {
for (const char * a : opt.args) {
cli_n_values[a] = arg_num_values(opt);
}
for (const char * a : opt.args_neg) {
cli_n_values[a] = 0;
}
}
common_params dummy_server;
auto ctx_server = common_params_parser_init(dummy_server, LLAMA_EXAMPLE_SERVER);
for (const auto & opt : ctx_server.options) {
for (const char * a : opt.args) {
server_args.insert(a);
}
for (const char * a : opt.args_neg) {
server_args.insert(a);
}
}
std::vector<std::string> result;
for (int i = 1; i < argc; i++) {
const std::string arg = argv[i];
auto it = cli_n_values.find(arg);
if (it == cli_n_values.end()) {
// not a known arg (should not happen when parsing succeeded)
continue;
}
const bool forward = server_args.count(arg) > 0;
if (forward) {
result.push_back(arg);
}
for (int j = 0; j < it->second && i + 1 < argc; j++) {
i++;
if (forward) {
result.push_back(argv[i]);
}
}
}
return result;
}
static std::string format_error_message(const json & err) {
if (err.contains("error") && err.at("error").is_object()) {
const auto & e = err.at("error");
@@ -113,7 +63,7 @@ static std::string media_type_from_ext(const std::string & fname) {
return "image";
}
bool cli_context::init(int argc, char ** argv) {
bool cli_context::init() {
std::optional<view::spinner> spinner;
if (!params.server_base.empty()) {
@@ -138,7 +88,7 @@ bool cli_context::init(int argc, char ** argv) {
spinner.emplace("Loading model...");
server.emplace();
if (!server->start(filter_server_args(argc, argv))) {
if (!server->start(params)) {
view::show_error("server start failed");
return false;
}

2
tools/cli/cli-context.h
View File

@@ -62,7 +62,7 @@ struct cli_context {
// connect to --server-base or spawn a local llama-server child;
// argc/argv are needed to forward the server-relevant args to the child
bool init(int argc, char ** argv);
bool init();
// run the interactive chat loop, returns the process exit code
int run();

20
tools/cli/cli-server.h
View File

@@ -8,7 +8,7 @@
// note: in the future, we may have a server running as daemon and the CLI can connect to it automatically
// llama_server will be available as a dynamic library symbol
int llama_server(int argc, char ** argv);
int llama_server(common_params & params, int argc, char ** argv);
struct cli_server {
std::thread th;
@@ -24,26 +24,16 @@ struct cli_server {
}
}
bool start(std::vector<std::string> args) {
bool start(common_params & params) {
port = common_http_get_free_port();
if (port <= 0) {
fprintf(stderr, "failed to get a free port\n");
exit(1);
}
th = std::thread([&, args_ = args]() {
auto args = args_; // copy to modify
args.push_back("--port");
args.push_back(std::to_string(port));
// convert to char* array
std::vector<char *> argv;
for (auto & arg : args) {
argv.push_back(arg.data());
}
argv.push_back(nullptr);
int res = llama_server(static_cast<int>(args.size()), argv.data());
th = std::thread([&]() {
// argc / argv are only used in router mode, we can skip them for now
int res = llama_server(params, 0, nullptr);
if (res != 0) {
fprintf(stderr, "llama_server exited with code %d\n", res);
}

2
tools/cli/cli.cpp
View File

@@ -182,7 +182,7 @@ int llama_cli(int argc, char ** argv) {
cli_context ctx_cli(params);
if (!ctx_cli.init(argc, argv)) {
if (!ctx_cli.init()) {
ctx_cli.shutdown();
return 1;
}

13
tools/server/server.cpp
View File

@@ -21,6 +21,12 @@
#include <windows.h>
#endif
// satisfies -Wmissing-declarations (used by llama command)
int llama_server(int argc, char ** argv);
// to be used via CLI (argc / argv are used by router mode only)
int llama_server(common_params & params, int argc, char ** argv);
static std::function<void(int)> shutdown_handler;
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
@@ -71,9 +77,6 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t
};
}
// satisfies -Wmissing-declarations
int llama_server(int argc, char ** argv);
int llama_server(int argc, char ** argv) {
std::setlocale(LC_NUMERIC, "C");
@@ -89,6 +92,10 @@ int llama_server(int argc, char ** argv) {
llama_backend_init();
llama_numa_init(params.numa);
return llama_server(params, argc, argv);
}
int llama_server(common_params & params, int argc, char ** argv) {
// router server never loads a model and must not touch the GPU
const bool is_router_server = params.model.path.empty()
&& params.model.hf_repo.empty();