upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-29 01:17:36 -05:00
Code Issues Packages Projects Releases Wiki Activity

common : fix HF cache paths on Windows (#29475)

Browse Source 
Supersedes #29158

Signed-off-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
Adrien Gallouët
2026-09-28 16:25:24 +02:00
committed by GitHub
parent f00a64c147
commit 6c7a87f7e5
4 changed files with 43 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/common.cpp
+11
View File
@@ -969,6 +969,17 @@ void common_set_env(const std::string & name, const std::string & value) {
#endif
}
std::filesystem::path common_get_path_from_env(const std::string & name) {
#if defined(_WIN32)
const std::wstring wname = utf8_to_wstring(name);
const wchar_t * wvalue = _wgetenv(wname.c_str());
return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path();
#else
const char * value = std::getenv(name.c_str());
return value ? std::filesystem::path(value) : std::filesystem::path();
#endif
}
std::string fs_get_cache_directory() {
std::string cache_directory = "";
auto ensure_trailing_slash = [](std::string p) {
common/common.h
+3
View File
@@ -904,6 +904,9 @@ std::string fs_path_to_utf8(const std::filesystem::path & path);
std::string common_get_env(const std::string & name);
void common_set_env(const std::string & name, const std::string & value);
// reads a path from the environment, an unset variable gives an empty path
std::filesystem::path common_get_path_from_env(const std::string & name);
//
// Filesystem utils
//
common/download.cpp
+2 -2
View File
@@ -286,7 +286,7 @@ static int common_download_file_single_online(const std::string & url,
static const int max_attempts = 3;
static const int retry_delay_seconds = 2;
const bool file_exists = std::filesystem::exists(path);
const bool file_exists = std::filesystem::exists(std::filesystem::u8path(path));
if (file_exists && skip_etag) {
LOG_DBG("%s: using cached file: %s\n", __func__, path.c_str());
@@ -477,7 +477,7 @@ int common_download_file_single(const std::string & url,
return common_download_file_single_online(url, path, online_opts, skip_etag);
}
if (!std::filesystem::exists(path)) {
if (!std::filesystem::exists(std::filesystem::u8path(path))) {
LOG_ERR("%s: required file is not available in cache (offline mode): %s\n", __func__, path.c_str());
return -1;
}
common/hf-cache.cpp
+27 -19
View File
@@ -44,8 +44,7 @@ static fs::path get_cache_directory() {
{HOME_DIR, fs::path(".cache") / "huggingface" / "hub"}
};
for (const auto & entry : entries) {
if (auto * p = std::getenv(entry.var); p && *p) {
fs::path base(p);
if (fs::path base = common_get_path_from_env(entry.var); !base.empty()) {
return entry.path.empty() ? base : base / entry.path;
}
}
@@ -174,7 +173,8 @@ static bool is_valid_subpath(const fs::path & path, const fs::path & subpath) {
}
static void safe_write_file(const fs::path & path, const std::string & data) {
fs::path path_tmp = path.string() + ".tmp";
fs::path path_tmp = path;
path_tmp += ".tmp";
if (path.has_parent_path()) {
fs::create_directories(path.parent_path());
@@ -191,7 +191,7 @@ static void safe_write_file(const fs::path & path, const std::string & data) {
}
if (file.fail() || ec) {
fs::remove(path_tmp, ec);
throw std::runtime_error("failed to write file: " + path.string());
throw std::runtime_error("failed to write file: " + fs_path_to_utf8(path));
}
}
@@ -241,6 +241,7 @@ static std::string get_repo_commit(const std::string & repo_id,
fs::path refs_path = get_repo_path(repo_id) / "refs";
std::string name;
std::string commit;
fs::path name_path;
for (const auto & branch : json["branches"]) {
if (!branch.is_object() ||
@@ -251,24 +252,28 @@ static std::string get_repo_commit(const std::string & repo_id,
std::string _name = branch["name"].get<std::string>();
std::string _commit = branch["targetCommit"].get<std::string>();
if (!is_valid_subpath(refs_path, _name)) {
LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str());
continue;
}
if (!is_valid_commit(_commit)) {
LOG_WRN("%s: skip invalid commit: %s\n", __func__, _commit.c_str());
continue;
}
const fs::path candidate = fs::u8path(_name);
if (!is_valid_subpath(refs_path, candidate)) {
LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str());
continue;
}
if (_name == "main") {
name = _name;
commit = _commit;
name_path = candidate;
break;
}
if (name.empty() || commit.empty()) {
name = _name;
commit = _commit;
name_path = candidate;
}
}
@@ -277,7 +282,7 @@ static std::string get_repo_commit(const std::string & repo_id,
return {};
}
safe_write_file(refs_path / name, commit);
safe_write_file(refs_path / name_path, commit);
return commit;
} catch (const common_json_error & e) {
@@ -326,7 +331,9 @@ hf_files get_repo_files(const std::string & repo_id,
file.repo_id = repo_id;
file.path = item["path"].get<std::string>();
if (!is_valid_subpath(commit_path, file.path)) {
const fs::path subpath = fs::u8path(file.path);
if (!is_valid_subpath(commit_path, subpath)) {
LOG_WRN("%s: skip invalid path: %s\n", __func__, file.path.c_str());
continue;
}
@@ -346,12 +353,12 @@ hf_files get_repo_files(const std::string & repo_id,
file.url = endpoint + repo_id + "/resolve/" + commit + "/" + file.path;
fs::path final_path = commit_path / file.path;
file.final_path = final_path.string();
fs::path final_path = commit_path / subpath;
file.final_path = fs_path_to_utf8(final_path);
if (!file.oid.empty() && !fs::exists(final_path)) {
fs::path local_path = blobs_path / file.oid;
file.local_path = local_path.string();
file.local_path = fs_path_to_utf8(local_path);
} else {
file.local_path = file.final_path;
}
@@ -418,7 +425,7 @@ hf_files get_cached_files(const std::string & repo_id) {
if (!fs::exists(snapshots_path)) {
continue;
}
std::string _repo_id = folder_name_to_repo(repo.path().filename().string());
std::string _repo_id = folder_name_to_repo(fs_path_to_utf8(repo.path().filename()));
if (!is_valid_repo_id(_repo_id)) {
continue;
@@ -441,8 +448,9 @@ hf_files get_cached_files(const std::string & repo_id) {
if (!path.empty()) {
hf_file file;
file.repo_id = _repo_id;
file.path = path.generic_string();
file.local_path = entry.path().string();
const auto generic_path = path.generic_u8string();
file.path = std::string(generic_path.begin(), generic_path.end());
file.local_path = fs_path_to_utf8(entry.path());
file.final_path = file.local_path;
files.push_back(std::move(file));
}
@@ -456,8 +464,8 @@ std::string finalize_file(const hf_file & file) {
static std::atomic<bool> symlinks_disabled{false};
std::error_code ec;
fs::path local_path(file.local_path);
fs::path final_path(file.final_path);
fs::path local_path = fs::u8path(file.local_path);
fs::path final_path = fs::u8path(file.final_path);
if (local_path == final_path || fs::exists(final_path, ec)) {
return file.final_path;
@@ -504,7 +512,7 @@ bool remove_cached_repo(const std::string & repo_id) {
std::error_code ec;
auto removed = fs::remove_all(repo_path, ec);
if (ec) {
LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str());
LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, fs_path_to_utf8(repo_path).c_str(), ec.message().c_str());
return false;
}
return removed > 0;