diff --git a/common/common.cpp b/common/common.cpp index ae44f054ab..6099f2ecc4 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -969,6 +969,17 @@ void common_set_env(const std::string & name, const std::string & value) { #endif } +std::filesystem::path common_get_path_from_env(const std::string & name) { +#if defined(_WIN32) + const std::wstring wname = utf8_to_wstring(name); + const wchar_t * wvalue = _wgetenv(wname.c_str()); + return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path(); +#else + const char * value = std::getenv(name.c_str()); + return value ? std::filesystem::path(value) : std::filesystem::path(); +#endif +} + std::string fs_get_cache_directory() { std::string cache_directory = ""; auto ensure_trailing_slash = [](std::string p) { diff --git a/common/common.h b/common/common.h index 527f4be7d8..3294694109 100644 --- a/common/common.h +++ b/common/common.h @@ -904,6 +904,9 @@ std::string fs_path_to_utf8(const std::filesystem::path & path); std::string common_get_env(const std::string & name); void common_set_env(const std::string & name, const std::string & value); +// reads a path from the environment, an unset variable gives an empty path +std::filesystem::path common_get_path_from_env(const std::string & name); + // // Filesystem utils // diff --git a/common/download.cpp b/common/download.cpp index 4b28a708c8..2edd2a11ca 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -286,7 +286,7 @@ static int common_download_file_single_online(const std::string & url, static const int max_attempts = 3; static const int retry_delay_seconds = 2; - const bool file_exists = std::filesystem::exists(path); + const bool file_exists = std::filesystem::exists(std::filesystem::u8path(path)); if (file_exists && skip_etag) { LOG_DBG("%s: using cached file: %s\n", __func__, path.c_str()); @@ -477,7 +477,7 @@ int common_download_file_single(const std::string & url, return common_download_file_single_online(url, path, online_opts, skip_etag); } - if (!std::filesystem::exists(path)) { + if (!std::filesystem::exists(std::filesystem::u8path(path))) { LOG_ERR("%s: required file is not available in cache (offline mode): %s\n", __func__, path.c_str()); return -1; } diff --git a/common/hf-cache.cpp b/common/hf-cache.cpp index 65c9722f33..5f37084d70 100644 --- a/common/hf-cache.cpp +++ b/common/hf-cache.cpp @@ -44,8 +44,7 @@ static fs::path get_cache_directory() { {HOME_DIR, fs::path(".cache") / "huggingface" / "hub"} }; for (const auto & entry : entries) { - if (auto * p = std::getenv(entry.var); p && *p) { - fs::path base(p); + if (fs::path base = common_get_path_from_env(entry.var); !base.empty()) { return entry.path.empty() ? base : base / entry.path; } } @@ -174,7 +173,8 @@ static bool is_valid_subpath(const fs::path & path, const fs::path & subpath) { } static void safe_write_file(const fs::path & path, const std::string & data) { - fs::path path_tmp = path.string() + ".tmp"; + fs::path path_tmp = path; + path_tmp += ".tmp"; if (path.has_parent_path()) { fs::create_directories(path.parent_path()); @@ -191,7 +191,7 @@ static void safe_write_file(const fs::path & path, const std::string & data) { } if (file.fail() || ec) { fs::remove(path_tmp, ec); - throw std::runtime_error("failed to write file: " + path.string()); + throw std::runtime_error("failed to write file: " + fs_path_to_utf8(path)); } } @@ -241,6 +241,7 @@ static std::string get_repo_commit(const std::string & repo_id, fs::path refs_path = get_repo_path(repo_id) / "refs"; std::string name; std::string commit; + fs::path name_path; for (const auto & branch : json["branches"]) { if (!branch.is_object() || @@ -251,24 +252,28 @@ static std::string get_repo_commit(const std::string & repo_id, std::string _name = branch["name"].get(); std::string _commit = branch["targetCommit"].get(); - if (!is_valid_subpath(refs_path, _name)) { - LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str()); - continue; - } if (!is_valid_commit(_commit)) { LOG_WRN("%s: skip invalid commit: %s\n", __func__, _commit.c_str()); continue; } + const fs::path candidate = fs::u8path(_name); + + if (!is_valid_subpath(refs_path, candidate)) { + LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str()); + continue; + } if (_name == "main") { name = _name; commit = _commit; + name_path = candidate; break; } if (name.empty() || commit.empty()) { name = _name; commit = _commit; + name_path = candidate; } } @@ -277,7 +282,7 @@ static std::string get_repo_commit(const std::string & repo_id, return {}; } - safe_write_file(refs_path / name, commit); + safe_write_file(refs_path / name_path, commit); return commit; } catch (const common_json_error & e) { @@ -326,7 +331,9 @@ hf_files get_repo_files(const std::string & repo_id, file.repo_id = repo_id; file.path = item["path"].get(); - if (!is_valid_subpath(commit_path, file.path)) { + const fs::path subpath = fs::u8path(file.path); + + if (!is_valid_subpath(commit_path, subpath)) { LOG_WRN("%s: skip invalid path: %s\n", __func__, file.path.c_str()); continue; } @@ -346,12 +353,12 @@ hf_files get_repo_files(const std::string & repo_id, file.url = endpoint + repo_id + "/resolve/" + commit + "/" + file.path; - fs::path final_path = commit_path / file.path; - file.final_path = final_path.string(); + fs::path final_path = commit_path / subpath; + file.final_path = fs_path_to_utf8(final_path); if (!file.oid.empty() && !fs::exists(final_path)) { fs::path local_path = blobs_path / file.oid; - file.local_path = local_path.string(); + file.local_path = fs_path_to_utf8(local_path); } else { file.local_path = file.final_path; } @@ -418,7 +425,7 @@ hf_files get_cached_files(const std::string & repo_id) { if (!fs::exists(snapshots_path)) { continue; } - std::string _repo_id = folder_name_to_repo(repo.path().filename().string()); + std::string _repo_id = folder_name_to_repo(fs_path_to_utf8(repo.path().filename())); if (!is_valid_repo_id(_repo_id)) { continue; @@ -441,8 +448,9 @@ hf_files get_cached_files(const std::string & repo_id) { if (!path.empty()) { hf_file file; file.repo_id = _repo_id; - file.path = path.generic_string(); - file.local_path = entry.path().string(); + const auto generic_path = path.generic_u8string(); + file.path = std::string(generic_path.begin(), generic_path.end()); + file.local_path = fs_path_to_utf8(entry.path()); file.final_path = file.local_path; files.push_back(std::move(file)); } @@ -456,8 +464,8 @@ std::string finalize_file(const hf_file & file) { static std::atomic symlinks_disabled{false}; std::error_code ec; - fs::path local_path(file.local_path); - fs::path final_path(file.final_path); + fs::path local_path = fs::u8path(file.local_path); + fs::path final_path = fs::u8path(file.final_path); if (local_path == final_path || fs::exists(final_path, ec)) { return file.final_path; @@ -504,7 +512,7 @@ bool remove_cached_repo(const std::string & repo_id) { std::error_code ec; auto removed = fs::remove_all(repo_path, ec); if (ec) { - LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str()); + LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, fs_path_to_utf8(repo_path).c_str(), ec.message().c_str()); return false; } return removed > 0;