upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-29 17:37:39 -05:00
Code Issues Packages Projects Releases Wiki Activity
Files
b8bdf73bb9baf044caadd33be2a51be70156ec57
llama.cpp/src/llama-mmap.cpp
T
Daniel Han b8bdf73bb9 llama: fold the random-access prefetch into its own feature flag 
LLAMA_MMAP_RANDOM_PREFETCH existed to measure the two halves of the feature
apart, and the measurement is done: on a cold cache over the same wikitext
run, MADV_RANDOM without the batched readahead takes 94.4 s against 36.7 s
for an untouched mapping, while the pair together take 34.1 s. Suppressing
the kernel's readahead only pays if we replace it, so the split let a user
select a 2.6x regression through a documented switch.

Keep the accessor, since the call site reads better than a mode comparison,
but derive it from the mode alone.
2026-08-27 10:01:47 +00:00

1074 lines
36 KiB
C++
Raw Blame History

#include "llama-mmap.h"
#include "llama-impl.h"
#include "ggml.h"
#include <cstring>
#include <cstdlib>
#include <climits>
#include <vector>
#include <stdexcept>
#include <cerrno>
#include <algorithm>
#ifdef __has_include
#if __has_include(<unistd.h>)
#include <unistd.h>
#include <fcntl.h>
#include <sys/stat.h>
#if defined(_POSIX_MAPPED_FILES)
#include <sys/mman.h>
#endif
#if defined(_POSIX_MEMLOCK_RANGE)
#include <sys/resource.h>
#endif
#endif
#endif
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#ifndef PATH_MAX
#define PATH_MAX MAX_PATH
#endif
#include <io.h>
#endif
#if defined(__APPLE__)
#include <TargetConditionals.h>
#endif
#ifdef _WIN32
# define llama_mmap_ftell _ftelli64
# define llama_mmap_fseek _fseeki64
#else
# define llama_mmap_ftell ftello
# define llama_mmap_fseek fseeko
#endif
// TODO: consider moving to llama-impl.h if needed in more places
#if defined(_WIN32)
static std::string llama_format_win_err(DWORD err) {
LPSTR buf;
size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);
if (!size) {
return "FormatMessageA failed";
}
std::string ret(buf, size);
LocalFree(buf);
return ret;
}
#endif
// llama_file
struct llama_file::impl {
#if defined(_WIN32)
HANDLE fp_win32;
std::string GetErrorMessageWin32(DWORD error_code) const {
std::string ret;
LPSTR lpMsgBuf = NULL;
DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL);
if (!bufLen) {
ret = format("Win32 error code: %lx", error_code);
} else {
ret = lpMsgBuf;
LocalFree(lpMsgBuf);
}
return ret;
}
impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) {
fp = ggml_fopen(fname, mode);
if (fp == NULL) {
throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
}
fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));
seek(0, SEEK_END);
size = tell();
seek(0, SEEK_SET);
}
impl(FILE * file) : owns_fp(false) {
fp = file;
fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));
seek(0, SEEK_END);
size = tell();
seek(0, SEEK_SET);
}
size_t tell() const {
LARGE_INTEGER li;
li.QuadPart = 0;
BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT);
if (!ret) {
throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
}
return li.QuadPart;
}
void seek(size_t offset, int whence) const {
static_assert(SEEK_SET == FILE_BEGIN, "SEEK_SET != FILE_BEGIN");
static_assert(SEEK_CUR == FILE_CURRENT, "SEEK_CUR != FILE_CURRENT");
static_assert(SEEK_END == FILE_END, "SEEK_END != FILE_END");
LARGE_INTEGER li;
li.QuadPart = offset;
BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence);
if (!ret) {
throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
}
}
void read_raw(void * ptr, size_t len) {
size_t bytes_read = 0;
while (bytes_read < len) {
size_t chunk_size = std::min<size_t>(len - bytes_read, 64*1024*1024);
DWORD chunk_read = 0;
BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);
if (!result) {
throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
}
if (chunk_read < chunk_size || chunk_read == 0) {
throw std::runtime_error("unexpectedly reached end of file");
}
bytes_read += chunk_read;
}
}
uint32_t read_u32() {
uint32_t val;
read_raw(&val, sizeof(val));
return val;
}
void write_raw(const void * ptr, size_t len) const {
size_t bytes_written = 0;
while (bytes_written < len) {
size_t chunk_size = std::min<size_t>(len - bytes_written, 64*1024*1024);
DWORD chunk_written = 0;
BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL);
if (!result) {
throw std::runtime_error(format("write error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
}
if (chunk_written < chunk_size || chunk_written == 0) {
throw std::runtime_error("unexpectedly failed to write bytes");
}
bytes_written += chunk_written;
}
}
void write_u32(uint32_t val) const {
write_raw(&val, sizeof(val));
}
bool has_direct_io() const {
return true;
}
~impl() {
if (fp && owns_fp) {
std::fclose(fp);
}
}
#else
impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {
#ifdef __linux__
// Try unbuffered I/O for read only
if (use_direct_io && std::strcmp(mode, "rb") == 0) {
if (init_fd()) {
return;
}
LLAMA_LOG_WARN("Failed to open file '%s' with error: %s. Falling back to buffered I/O",
fname, strerror(errno));
}
#endif
init_fp(mode);
}
#ifdef __linux__
bool init_fd() {
fd = open(fname.c_str(), O_RDONLY | O_DIRECT);
if (fd != -1) {
struct stat file_stats{};
fstat(fd, &file_stats);
size = file_stats.st_size;
alignment = file_stats.st_blksize;
off_t ret = lseek(fd, 0, SEEK_SET);
if (ret == -1) {
throw std::runtime_error(format("seek error: %s", strerror(errno)));
}
return true;
}
return false;
}
#endif
void init_fp(const char * mode) {
fp = ggml_fopen(fname.c_str(), mode);
if (fp == NULL) {
throw std::runtime_error(format("failed to open %s: %s", fname.c_str(), strerror(errno)));
}
seek(0, SEEK_END);
size = tell();
seek(0, SEEK_SET);
}
impl(FILE * file) : fname("(file*)"), owns_fp(false) {
fp = file;
seek(0, SEEK_END);
size = tell();
seek(0, SEEK_SET);
}
size_t tell() const {
if (fd == -1) {
off_t ret = llama_mmap_ftell(fp);
if (ret == -1) {
throw std::runtime_error(format("ftell error: %s", strerror(errno)));
}
return (size_t) ret;
}
off_t pos = lseek(fd, 0, SEEK_CUR);
if (pos == -1) {
throw std::runtime_error(format("lseek error: %s", strerror(errno)));
}
return (size_t) pos;
}
void seek(size_t offset, int whence) const {
off_t ret = 0;
if (fd == -1) {
ret = llama_mmap_fseek(fp, offset, whence);
} else {
ret = lseek(fd, offset, whence);
}
if (ret == -1) {
throw std::runtime_error(format("seek error: %s", strerror(errno)));
}
}
void read_raw_unsafe(void * ptr, size_t len) {
if (len == 0) {
return;
}
errno = 0;
if (fd == -1) {
const size_t curr_off = tell();
const size_t to_read = std::min(len, size - curr_off);
std::size_t ret = std::fread(ptr, to_read, 1, fp);
if (ferror(fp)) {
throw std::runtime_error(format("read error: %s", strerror(errno)));
}
if (to_read > 0 && ret != 1) {
throw std::runtime_error("unexpectedly reached end of file");
}
} else {
size_t bytes_read = 0;
while (bytes_read < len) {
const size_t to_read = len - bytes_read;
ssize_t ret = ::read(fd, reinterpret_cast<char *>(ptr) + bytes_read, to_read);
if (ret == -1) {
if (errno == EINTR) {
continue; // Interrupted by signal, retry
}
// Fallback to std::fread in case the DMA controller cannot access the buffer
if (errno == EFAULT || errno == EINVAL) {
LLAMA_LOG_WARN("%s: Falling back to buffered IO due to %s\n", __func__, strerror(errno));
auto curr_off = tell();
close(fd);
fd = -1;
alignment = 1;
init_fp("rb");
seek(curr_off, SEEK_SET);
read_raw_unsafe(ptr, len);
return;
}
throw std::runtime_error(format("read error: %s", strerror(errno)));
}
if (ret == 0) {
// EOF: allow if this read was only pulling alignment padding past file end
off_t pos = lseek(fd, 0, SEEK_CUR);
if (pos != -1 && (size_t) pos == size) {
std::memset(reinterpret_cast<char *>(ptr) + bytes_read, 0, len - bytes_read);
return;
}
throw std::runtime_error("unexpectedly reached end of file");
}
bytes_read += (size_t) ret;
}
}
}
void read_aligned_chunk(void * dest, size_t size) {
size_t offset = tell();
off_t aligned_offset = offset & ~(alignment - 1);
off_t offset_from_alignment = offset - aligned_offset;
size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1);
void * raw_buffer = nullptr;
int ret = posix_memalign(&raw_buffer, alignment, bytes_to_read);
if (ret != 0) {
throw std::runtime_error(format("posix_memalign failed with error %d", ret));
}
struct aligned_buffer_deleter {
void operator()(void * p) const { free(p); }
};
std::unique_ptr<void, aligned_buffer_deleter> buffer(raw_buffer);
seek(aligned_offset, SEEK_SET);
read_raw_unsafe(buffer.get(), bytes_to_read);
uintptr_t actual_data = reinterpret_cast<uintptr_t>(buffer.get()) + offset_from_alignment;
memcpy(dest, reinterpret_cast<void *>(actual_data), size);
}
void read_raw(void * ptr, size_t len) {
if (has_direct_io()) {
read_aligned_chunk(ptr, len);
} else {
read_raw_unsafe(ptr, len);
}
}
uint32_t read_u32() {
uint32_t ret;
read_raw(&ret, sizeof(ret));
return ret;
}
void write_raw(const void * ptr, size_t len) const {
if (len == 0) {
return;
}
errno = 0;
size_t ret = std::fwrite(ptr, len, 1, fp);
if (ret != 1) {
throw std::runtime_error(format("write error: %s", strerror(errno)));
}
}
void write_u32(uint32_t val) const {
write_raw(&val, sizeof(val));
}
bool has_direct_io() const {
return fd != -1 && alignment > 1;
}
~impl() {
if (fd != -1) {
close(fd);
} else if (owns_fp) {
std::fclose(fp);
}
}
int fd = -1;
std::string fname;
#endif
size_t read_alignment() const {
return alignment;
}
size_t alignment = 1;
FILE * fp{};
size_t size{};
bool owns_fp = true;
};
llama_file::llama_file(const char * fname, const char * mode, const bool use_direct_io) :
pimpl(std::make_unique<impl>(fname, mode, use_direct_io)) {}
llama_file::llama_file(FILE * file) : pimpl(std::make_unique<impl>(file)) {}
llama_file::~llama_file() = default;
size_t llama_file::tell() const { return pimpl->tell(); }
size_t llama_file::size() const { return pimpl->size; }
size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }
int llama_file::file_id() const {
#ifdef _WIN32
return _fileno(pimpl->fp);
#else
if (pimpl->fd != -1) {
return pimpl->fd;
}
#if defined(fileno)
return fileno(pimpl->fp);
#else
return ::fileno(pimpl->fp);
#endif
#endif
}
void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); }
void llama_file::read_raw(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }
#ifdef _WIN32
void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }
#else
void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw_unsafe(ptr, len); }
#endif
uint32_t llama_file::read_u32() { return pimpl->read_u32(); }
void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }
void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }
// llama_mmap
llama_mmap_random_mode llama_mmap_random_mode_get() {
// read once: this is consulted per mapping and per gather
static const llama_mmap_random_mode mode = []() {
const char * env = getenv("LLAMA_MMAP_RANDOM");
if (env == nullptr || strcmp(env, "0") == 0 || env[0] == '\0') {
return LLAMA_MMAP_RANDOM_OFF;
}
if (strcmp(env, "drop") == 0) {
return LLAMA_MMAP_RANDOM_DROP;
}
return LLAMA_MMAP_RANDOM_ON;
}();
return mode;
}
bool llama_mmap_random_prefetch_enabled() {
return llama_mmap_random_mode_get() != LLAMA_MMAP_RANDOM_OFF;
}
static size_t llama_mmap_page_size() {
#if defined(_WIN32)
SYSTEM_INFO si;
GetSystemInfo(&si);
return (size_t) si.dwPageSize;
#elif defined(_SC_PAGESIZE)
return (size_t) sysconf(_SC_PAGESIZE);
#else
return 4096;
#endif
}
// the distinct pages the given rows fall on, as offsets into the mapping, merged into runs.
// a row is much smaller than a page and rows repeat within a batch, so this is what turns a
// hint per row into a hint per page. platform independent: the callers differ only in which
// syscall they hand the result to.
static std::vector<std::pair<size_t, size_t>> llama_mmap_row_pages(
size_t base_off, size_t stride, size_t row_size, size_t map_size,
const int32_t * rows, size_t n_rows, size_t page_size) {
std::vector<size_t> pages;
pages.reserve(n_rows);
for (size_t i = 0; i < n_rows; ++i) {
if (rows[i] < 0) {
continue;
}
const size_t first = base_off + (size_t) rows[i] * stride;
const size_t last = first + row_size;
// a corrupt or unexpected index must not turn into a hint outside the mapping
if (row_size == 0 || last > map_size || first < base_off) {
continue;
}
for (size_t p = first / page_size; p <= (last - 1) / page_size; ++p) {
pages.push_back(p);
}
}
std::sort(pages.begin(), pages.end());
pages.erase(std::unique(pages.begin(), pages.end()), pages.end());
std::vector<std::pair<size_t, size_t>> ranges;
for (size_t i = 0; i < pages.size(); ) {
size_t j = i + 1;
while (j < pages.size() && pages[j] == pages[j - 1] + 1) {
++j;
}
const size_t off = pages[i] * page_size;
ranges.emplace_back(off, std::min((pages[j - 1] - pages[i] + 1) * page_size, map_size - off));
i = j;
}
return ranges;
}
struct llama_mmap::impl {
#ifdef _POSIX_MAPPED_FILES
std::vector<std::pair<size_t, size_t>> mapped_fragments;
impl(struct llama_file * file, size_t prefetch, bool numa) {
size = file->size();
int fd = file->file_id();
fd_advise = fd;
int flags = MAP_SHARED;
if (numa) { prefetch = 0; }
#ifdef __linux__
if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {
LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",
strerror(errno));
}
if (prefetch) { flags |= MAP_POPULATE; }
#endif
addr = mmap(NULL, file->size(), PROT_READ, flags, fd, 0);
if (addr == MAP_FAILED) {
throw std::runtime_error(format("mmap failed: %s", strerror(errno)));
}
if (prefetch > 0) {
if (posix_madvise(addr, std::min(file->size(), prefetch), POSIX_MADV_WILLNEED)) {
LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",
strerror(errno));
}
}
if (numa) {
if (posix_madvise(addr, file->size(), POSIX_MADV_RANDOM)) {
LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",
strerror(errno));
}
}
mapped_fragments.emplace_back(0, file->size());
}
// the load path asks for POSIX_FADV_SEQUENTIAL, which is right while the file is being
// streamed once into buffers and wrong for whatever stays host-resident afterwards: those
// tensors are read by sparse gathers, where readahead buys nothing and costs page cache.
// flipping the advice only after load, and only over the tensor, keeps everything else on
// the loader's behaviour.
void advise_random_range(size_t offset, size_t len, bool drop) {
if (offset >= size || len == 0) {
return;
}
len = std::min(len, size - offset);
// madvise rejects an unaligned start and rounds the length up, so round both out. that
// can take in the tail of the tensor before and the head of the one after, one page each
const size_t page = llama_mmap_page_size();
const size_t first = offset & ~(page - 1);
const size_t last = std::min(size, (offset + len + page - 1) & ~(page - 1));
#if defined(__linux__)
if (drop) {
// on a shared file map this only tears down our page tables
if (madvise((char *) addr + first, last - first, MADV_DONTNEED)) {
LLAMA_LOG_WARN("warning: madvise(.., MADV_DONTNEED) failed: %s\n", strerror(errno));
}
// and this frees the page cache. it takes the range and spares partial pages, so a
// tensor sharing the first or last page keeps its cache
if (fd_advise >= 0 && posix_fadvise(fd_advise, (off_t) offset, (off_t) len, POSIX_FADV_DONTNEED)) {
LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_DONTNEED) failed: %s\n", strerror(errno));
}
}
#else
GGML_UNUSED(drop);
#endif
// no POSIX_FADV_RANDOM to go with this: it ignores the range and marks the whole open
// file, and the FMODE_RANDOM it sets is only read by the read() path, never by a fault
if (posix_madvise((char *) addr + first, last - first, POSIX_MADV_RANDOM)) {
LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n", strerror(errno));
}
}
void prefetch_except(const std::vector<std::pair<size_t, size_t>> & skip) {
const size_t page = llama_mmap_page_size();
size_t pos = 0;
for (const auto & [off, len] : skip) {
const size_t first = off & ~(page - 1);
if (first > pos) {
prefetch_range(pos, first - pos);
}
pos = std::max(pos, std::min(size, (off + len + page - 1) & ~(page - 1)));
}
if (pos < size) {
prefetch_range(pos, size - pos);
}
}
void prefetch_range(size_t offset, size_t len) const {
if (posix_madvise((char *) addr + offset, len, POSIX_MADV_WILLNEED)) {
LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n", strerror(errno));
}
}
void prefetch_rows(const void * base, size_t stride, size_t row_size,
const int32_t * rows, size_t n_rows) const {
#if defined(_POSIX_MAPPED_FILES)
const size_t base_off = (const char *) base - (const char *) addr;
for (const auto & [off, len] : llama_mmap_row_pages(
base_off, stride, row_size, size, rows, n_rows, llama_mmap_page_size())) {
// deliberately unchecked: this is a hint issued thousands of times per batch, and a
// failed hint only costs the fault it would have avoided
posix_madvise((char *) addr + off, len, POSIX_MADV_WILLNEED);
}
#else
GGML_UNUSED(base);
GGML_UNUSED(stride);
GGML_UNUSED(row_size);
GGML_UNUSED(rows);
GGML_UNUSED(n_rows);
#endif
}
static void align_range(size_t * first, size_t * last, size_t page_size) {
size_t offset_in_page = *first & (page_size - 1);
size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;
*first += offset_to_page;
*last = *last & ~(page_size - 1);
if (*last <= *first) {
*last = *first;
}
}
void unmap_fragment(size_t first, size_t last) {
int page_size = sysconf(_SC_PAGESIZE);
align_range(&first, &last, page_size);
size_t len = last - first;
if (len == 0) {
return;
}
GGML_ASSERT(first % page_size == 0);
GGML_ASSERT(last % page_size == 0);
GGML_ASSERT(last > first);
void * next_page_start = (uint8_t *) addr + first;
if (munmap(next_page_start, len)) {
LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));
}
std::vector<std::pair<size_t, size_t>> new_mapped_fragments;
for (const auto & frag : mapped_fragments) {
if (frag.first < first && frag.second > last) {
new_mapped_fragments.emplace_back(frag.first, first);
new_mapped_fragments.emplace_back(last, frag.second);
} else if (frag.first < first && frag.second > first) {
new_mapped_fragments.emplace_back(frag.first, first);
} else if (frag.first < last && frag.second > last) {
new_mapped_fragments.emplace_back(last, frag.second);
} else if (frag.first >= first && frag.second <= last) {
} else {
new_mapped_fragments.push_back(frag);
}
}
mapped_fragments = std::move(new_mapped_fragments);
}
~impl() {
for (const auto & frag : mapped_fragments) {
if (munmap((char *) addr + frag.first, frag.second - frag.first)) {
LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));
}
}
}
#elif defined(_WIN32)
HANDLE hMapping = nullptr;
impl(struct llama_file * file, size_t prefetch, bool numa) {
GGML_UNUSED(numa);
size = file->size();
HANDLE hFile = (HANDLE) _get_osfhandle(file->file_id());
hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
if (hMapping == NULL) {
DWORD error = GetLastError();
throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));
}
addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
DWORD error = GetLastError();
if (addr == NULL) {
CloseHandle(hMapping);
throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));
}
if (prefetch > 0) {
#if _WIN32_WINNT >= 0x602
BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");
pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");
if (pPrefetchVirtualMemory) {
WIN32_MEMORY_RANGE_ENTRY range;
range.VirtualAddress = addr;
range.NumberOfBytes = (SIZE_T) std::min(size, prefetch);
if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {
LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",
llama_format_win_err(GetLastError()).c_str());
}
}
#else
LLAMA_LOG_DEBUG("skipping PrefetchVirtualMemory because _WIN32_WINNT < 0x602\n");
#endif
}
}
void unmap_fragment(size_t first, size_t last) {
GGML_UNUSED(first);
GGML_UNUSED(last);
}
// Windows has no "read this range randomly" hint. not pulling the range in is what keeps the
// pages out; there is nothing further to say here, and nothing to drop back.
void advise_random_range(size_t offset, size_t len, bool drop) {
GGML_UNUSED(offset);
GGML_UNUSED(len);
GGML_UNUSED(drop);
}
void prefetch_except(const std::vector<std::pair<size_t, size_t>> & skip) {
#if _WIN32_WINNT >= 0x602
BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");
pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");
if (!pPrefetchVirtualMemory) {
return;
}
const size_t page = llama_mmap_page_size();
std::vector<WIN32_MEMORY_RANGE_ENTRY> entries;
size_t pos = 0;
for (const auto & [off, len] : skip) {
const size_t first = off & ~(page - 1);
if (first > pos) {
WIN32_MEMORY_RANGE_ENTRY e;
e.VirtualAddress = (char *) addr + pos;
e.NumberOfBytes = (SIZE_T) (first - pos);
entries.push_back(e);
}
pos = std::max(pos, std::min(size, (off + len + page - 1) & ~(page - 1)));
}
if (pos < size) {
WIN32_MEMORY_RANGE_ENTRY e;
e.VirtualAddress = (char *) addr + pos;
e.NumberOfBytes = (SIZE_T) (size - pos);
entries.push_back(e);
}
if (!entries.empty() && !pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0)) {
LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",
llama_format_win_err(GetLastError()).c_str());
}
#else
GGML_UNUSED(skip);
LLAMA_LOG_DEBUG("skipping PrefetchVirtualMemory because _WIN32_WINNT < 0x602\n");
#endif
}
// PrefetchVirtualMemory takes the whole set of ranges in one call, which is exactly the
// batching this wants: the reads are issued together instead of one fault at a time.
void prefetch_rows(const void * base, size_t stride, size_t row_size,
const int32_t * rows, size_t n_rows) const {
#if _WIN32_WINNT >= 0x602
BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");
pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");
if (!pPrefetchVirtualMemory) {
return;
}
const size_t base_off = (const char *) base - (const char *) addr;
std::vector<WIN32_MEMORY_RANGE_ENTRY> entries;
for (const auto & [off, len] : llama_mmap_row_pages(
base_off, stride, row_size, size, rows, n_rows, llama_mmap_page_size())) {
WIN32_MEMORY_RANGE_ENTRY e;
e.VirtualAddress = (char *) addr + off;
e.NumberOfBytes = (SIZE_T) len;
entries.push_back(e);
}
if (!entries.empty()) {
// unchecked for the same reason as the POSIX branch: it is only a hint
pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0);
}
#else
GGML_UNUSED(base);
GGML_UNUSED(stride);
GGML_UNUSED(row_size);
GGML_UNUSED(rows);
GGML_UNUSED(n_rows);
#endif
}
~impl() {
if (hMapping) {
if (addr) {
if (!UnmapViewOfFile(addr)) {
LLAMA_LOG_WARN("warning: UnmapViewOfFile failed: %s\n",
llama_format_win_err(GetLastError()).c_str());
}
}
if (!CloseHandle(hMapping)) {
LLAMA_LOG_WARN("warning: CloseHandle failed: %s\n",
llama_format_win_err(GetLastError()).c_str());
}
}
}
#else
impl(struct llama_file * file, size_t prefetch, bool numa) {
GGML_UNUSED(file);
GGML_UNUSED(prefetch);
GGML_UNUSED(numa);
throw std::runtime_error("mmap not supported");
}
void unmap_fragment(size_t first, size_t last) {
GGML_UNUSED(first);
GGML_UNUSED(last);
throw std::runtime_error("mmap not supported");
}
void advise_random_range(size_t offset, size_t len, bool drop) {
GGML_UNUSED(offset);
GGML_UNUSED(len);
GGML_UNUSED(drop);
throw std::runtime_error("mmap not supported");
}
void prefetch_except(const std::vector<std::pair<size_t, size_t>> & skip) {
GGML_UNUSED(skip);
throw std::runtime_error("mmap not supported");
}
void prefetch_rows(const void * base, size_t stride, size_t row_size,
const int32_t * rows, size_t n_rows) const {
GGML_UNUSED(base);
GGML_UNUSED(stride);
GGML_UNUSED(row_size);
GGML_UNUSED(rows);
GGML_UNUSED(n_rows);
throw std::runtime_error("mmap not supported");
}
#endif
bool contains(const void * ptr, size_t len) const {
const char * p = (const char *) ptr;
const char * b = (const char *) addr;
return p >= b && len <= size && (size_t) (p - b) <= size - len;
}
void * addr;
size_t size;
// the fd is kept only to re-advise the file; the mapping owns no reference to it
int fd_advise = -1;
};
llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique<impl>(file, prefetch, numa)) {}
llama_mmap::~llama_mmap() = default;
size_t llama_mmap::size() const { return pimpl->size; }
void * llama_mmap::addr() const { return pimpl->addr; }
void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); }
void llama_mmap::advise_random_range(size_t offset, size_t len, bool drop) {
pimpl->advise_random_range(offset, len, drop);
}
void llama_mmap::prefetch_except(const std::vector<std::pair<size_t, size_t>> & skip) {
pimpl->prefetch_except(skip);
}
bool llama_mmap::contains(const void * ptr, size_t len) const { return pimpl->contains(ptr, len); }
void llama_mmap::prefetch_rows(const void * base, size_t stride, size_t row_size,
const int32_t * rows, size_t n_rows) const {
pimpl->prefetch_rows(base, stride, row_size, rows, n_rows);
}
#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)
const bool llama_mmap::SUPPORTED = true;
#else
const bool llama_mmap::SUPPORTED = false;
#endif
// llama_mlock
struct llama_mlock::impl {
#ifdef _POSIX_MEMLOCK_RANGE
static size_t lock_granularity() {
return (size_t) sysconf(_SC_PAGESIZE);
}
bool raw_lock(const void * addr, size_t size) const {
if (!mlock(addr, size)) {
return true;
}
#ifdef __APPLE__
#define MLOCK_SUGGESTION \
"Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \
"decreasing 'vm.global_no_user_wire_amount'. Also try increasing RLIMIT_MEMLOCK (ulimit -l).\n"
#else
#define MLOCK_SUGGESTION \
"Try increasing RLIMIT_MEMLOCK ('ulimit -l' as root).\n"
#endif
char* errmsg = std::strerror(errno);
bool suggest = (errno == ENOMEM);
#if defined(TARGET_OS_VISION) || defined(TARGET_OS_TV) || defined(_AIX) || defined(__HAIKU__)
// visionOS/tvOS/Haiku don't support RLIMIT_MEMLOCK
// Skip resource limit checks on these platforms
suggest = false;
#else
struct rlimit lock_limit;
if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) {
suggest = false;
}
if (suggest && ((uint64_t)lock_limit.rlim_max > (uint64_t)lock_limit.rlim_cur + size)) {
suggest = false;
}
#endif
LLAMA_LOG_WARN("warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",
size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");
return false;
}
static void raw_unlock(void * addr, size_t size) {
if (munlock(addr, size)) {
LLAMA_LOG_WARN("warning: failed to munlock buffer: %s\n", std::strerror(errno));
}
}
#elif defined(_WIN32)
static size_t lock_granularity() {
SYSTEM_INFO si;
GetSystemInfo(&si);
return (size_t) si.dwPageSize;
}
bool raw_lock(void * ptr, size_t len) const {
for (int tries = 1; ; tries++) {
if (VirtualLock(ptr, len)) {
return true;
}
if (tries == 2) {
LLAMA_LOG_WARN("warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",
len, size, llama_format_win_err(GetLastError()).c_str());
return false;
}
SIZE_T min_ws_size, max_ws_size;
if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {
LLAMA_LOG_WARN("warning: GetProcessWorkingSetSize failed: %s\n",
llama_format_win_err(GetLastError()).c_str());
return false;
}
size_t increment = len + 1048576;
min_ws_size += increment;
max_ws_size += increment;
if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {
LLAMA_LOG_WARN("warning: SetProcessWorkingSetSize failed: %s\n",
llama_format_win_err(GetLastError()).c_str());
return false;
}
}
}
static void raw_unlock(void * ptr, size_t len) {
if (!VirtualUnlock(ptr, len)) {
LLAMA_LOG_WARN("warning: failed to VirtualUnlock buffer: %s\n",
llama_format_win_err(GetLastError()).c_str());
}
}
#else
static size_t lock_granularity() {
return (size_t) 65536;
}
bool raw_lock(const void * addr, size_t len) const {
LLAMA_LOG_WARN("warning: mlock not supported on this system\n");
return false;
}
static void raw_unlock(const void * addr, size_t len) {}
#endif
impl() : addr(NULL), size(0), failed_already(false) {}
void init(void * ptr) {
GGML_ASSERT(addr == NULL && size == 0);
addr = ptr;
}
void grow_to(size_t target_size) {
GGML_ASSERT(addr);
if (failed_already) {
return;
}
size_t granularity = lock_granularity();
target_size = (target_size + granularity - 1) & ~(granularity - 1);
if (target_size > size) {
if (raw_lock((uint8_t *) addr + size, target_size - size)) {
size = target_size;
} else {
failed_already = true;
}
}
}
void * addr;
size_t size;
bool failed_already;
};
llama_mlock::llama_mlock() : pimpl(std::make_unique<impl>()) {}
llama_mlock::~llama_mlock() = default;
void llama_mlock::init(void * ptr) { pimpl->init(ptr); }
void llama_mlock::grow_to(size_t target_size) { pimpl->grow_to(target_size); }
#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)
const bool llama_mlock::SUPPORTED = true;
#else
const bool llama_mlock::SUPPORTED = false;
#endif
size_t llama_path_max() {
return PATH_MAX;
}
Reference in New Issue View Git Blame Copy Permalink