Added support for _POSIX_MAPPED_FILES if defined in source (#564)

Getting rid of them fixed GA Ubuntu, but broke GA MacOS. Let's try a
different strategy.

.github/workflows/build.yml

@@ -33,6 +33,20 @@ jobs:
         run: |
           make
 
+  windows-latest:
+    runs-on: windows-latest
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v1
+
+      - name: Build
+        run: |
+          mkdir build
+          cd build
+          cmake ..
+          cmake --build . --config Release
+
 #  ubuntu-latest-gcc:
 #    runs-on: ubuntu-latest
 #

.gitignore

@@ -18,6 +18,9 @@ models/*
 
 /main
 /quantize
+/magic.dat
 
 arm_neon.h
 compile_commands.json
+CMakeFiles/
+CMakeCache.txt

CMakeLists.txt

@@ -0,0 +1,130 @@
+cmake_minimum_required(VERSION 3.8)
+project("llama.cpp")
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED true)
+set(CMAKE_C_STANDARD 11)
+
+if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
+    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
+endif()
+
+option(LLAMA_ALL_WARNINGS            "llama: enable all compiler warnings"                   ON)
+option(LLAMA_ALL_WARNINGS_3RD_PARTY  "llama: enable all compiler warnings in 3rd party libs" OFF)
+
+option(LLAMA_SANITIZE_THREAD         "llama: enable thread sanitizer"    OFF)
+option(LLAMA_SANITIZE_ADDRESS        "llama: enable address sanitizer"   OFF)
+option(LLAMA_SANITIZE_UNDEFINED      "llama: enable undefined sanitizer" OFF)
+
+if (APPLE)
+    option(LLAMA_NO_ACCELERATE       "llama: disable Accelerate framework" OFF)
+    option(LLAMA_NO_AVX              "llama: disable AVX" OFF)
+    option(LLAMA_NO_AVX2             "llama: disable AVX2" OFF)
+    option(LLAMA_NO_FMA              "llama: disable FMA" OFF)
+endif()
+
+if (NOT MSVC)
+    if (LLAMA_SANITIZE_THREAD)
+        set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS}   -fsanitize=thread")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread")
+    endif()
+
+    if (LLAMA_SANITIZE_ADDRESS)
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -fsanitize=address -fno-omit-frame-pointer")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer")
+    endif()
+
+    if (LLAMA_SANITIZE_UNDEFINED)
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -fsanitize=undefined")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined")
+    endif()
+endif()
+
+if (APPLE AND NOT LLAMA_NO_ACCELERATE)
+    find_library(ACCELERATE_FRAMEWORK Accelerate)
+    if (ACCELERATE_FRAMEWORK)
+        message(STATUS "Accelerate framework found")
+
+        set(LLAMA_EXTRA_LIBS  ${LLAMA_EXTRA_LIBS}  ${ACCELERATE_FRAMEWORK})
+        set(LLAMA_EXTRA_FLAGS ${LLAMA_EXTRA_FLAGS} -DGGML_USE_ACCELERATE)
+    else()
+        message(WARNING "Accelerate framework not found")
+    endif()
+endif()
+
+if (LLAMA_ALL_WARNINGS)
+    if (NOT MSVC)
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} \
+            -Wall                           \
+            -Wextra                         \
+            -Wpedantic                      \
+            -Wshadow                        \
+            -Wcast-qual                     \
+            -Wstrict-prototypes             \
+            -Wpointer-arith                 \
+            -Wno-unused-function            \
+        ")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
+            -Wall                           \
+            -Wextra                         \
+            -Wpedantic                      \
+            -Wcast-qual                     \
+        ")
+    else()
+        # todo : msvc
+    endif()
+endif()
+
+message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
+
+if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
+    message(STATUS "ARM detected")
+else()
+    message(STATUS "x86 detected")
+    if (MSVC)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
+        set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /arch:AVX2")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /arch:AVX2")
+    else()
+        if(NOT LLAMA_NO_AVX)
+            set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
+        endif()
+        if(NOT LLAMA_NO_AVX2)
+            set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
+        endif()
+        if(NOT LLAMA_NO_FMA)
+            set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma")
+        endif()
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mf16c")
+    endif()
+endif()
+
+# if (LLAMA_PERF)
+#     set(LLAMA_EXTRA_FLAGS ${LLAMA_EXTRA_FLAGS} -DGGML_PERF)
+# endif()
+
+add_executable(llama
+    main.cpp
+    utils.cpp
+    utils.h
+    mmap.c
+    mmap.h)
+
+add_executable(quantize
+    quantize.cpp
+    utils.cpp
+    utils.h)
+
+add_library(ggml
+    ggml.c
+    ggml.h)
+
+target_compile_definitions(ggml PUBLIC ${LLAMA_EXTRA_FLAGS})
+target_compile_definitions(llama PUBLIC ${LLAMA_EXTRA_FLAGS})
+target_compile_definitions(quantize PUBLIC ${LLAMA_EXTRA_FLAGS})
+
+target_link_libraries(ggml PRIVATE ${LLAMA_EXTRA_LIBS})
+target_include_directories(ggml PUBLIC .)
+target_link_libraries(quantize PRIVATE ggml)
+target_link_libraries(llama PRIVATE ggml)

Makefile

@@ -48,6 +48,10 @@ ifeq ($(UNAME_S),FreeBSD)
 	CFLAGS   += -pthread
 	CXXFLAGS += -pthread
 endif
+ifeq ($(UNAME_S),NetBSD)
+	CFLAGS   += -pthread
+	CXXFLAGS += -pthread
+endif
 ifeq ($(UNAME_S),Haiku)
 	CFLAGS   += -pthread
 	CXXFLAGS += -pthread
@@ -181,14 +185,17 @@ default: main quantize
 ggml.o: ggml.c ggml.h
 	$(CC)  $(CFLAGS)   -c ggml.c -o ggml.o
 
+mmap.o: mmap.c mmap.h
+	$(CC) $(CFLAGS) -c mmap.c -o mmap.o
+
 utils.o: utils.cpp utils.h
 	$(CXX) $(CXXFLAGS) -c utils.cpp -o utils.o
 
 clean:
 	rm -f *.o main quantize
 
-main: main.cpp ggml.o utils.o
-	$(CXX) $(CXXFLAGS) main.cpp ggml.o utils.o -o main $(LDFLAGS)
+main: main.cpp ggml.o utils.o mmap.o
+	$(CXX) $(CXXFLAGS) main.cpp ggml.o utils.o mmap.o -o main $(LDFLAGS)
 	./main -h
 
 quantize: quantize.cpp ggml.o utils.o

README.md

@@ -5,17 +5,17 @@
 
 Inference of [Facebook's LLaMA](https://github.com/facebookresearch/llama) model in pure C/C++
 
-**Hot topics**
+**Hot topics:**
 
-- Running on Windows: https://github.com/ggerganov/llama.cpp/issues/22
-- Fix Tokenizer / Unicode support: https://github.com/ggerganov/llama.cpp/issues/11
+- Cache input prompts for faster initialization: https://github.com/ggerganov/llama.cpp/issues/64
+- Create a `llama.cpp` logo: https://github.com/ggerganov/llama.cpp/issues/105
 
 ## Description
 
 The main goal is to run the model using 4-bit quantization on a MacBook
 
 - Plain C/C++ implementation without dependencies
-- Apple silicon first-class citizen - optimized via Arm Neon and Accelerate framework
+- Apple silicon first-class citizen - optimized via ARM NEON
 - AVX2 support for x86 architectures
 - Mixed F16 / F32 precision
 - 4-bit quantization support
@@ -23,14 +23,14 @@ The main goal is to run the model using 4-bit quantization on a MacBook
 
 This was [hacked in an evening](https://github.com/ggerganov/llama.cpp/issues/33#issuecomment-1465108022) - I have no idea if it works correctly.
 Please do not make conclusions about the models based on the results from this implementation.
-For all I know, it can be completely wrong. This project is for educational purposes and is not going to be maintained properly.
-New features will probably be added mostly through community contributions, if any.
+For all I know, it can be completely wrong. This project is for educational purposes.
+New features will probably be added mostly through community contributions.
 
 Supported platforms:
 
 - [X] Mac OS
 - [X] Linux
-- [ ] Windows (soon)
+- [X] Windows (via CMake)
 
 ---
 
@@ -145,44 +145,16 @@ python3 -m pip install torch numpy sentencepiece
 python3 convert-pth-to-ggml.py models/7B/ 1
 
 # quantize the model to 4-bits
-./quantize ./models/7B/ggml-model-f16.bin ./models/7B/ggml-model-q4_0.bin 2
+./quantize.sh 7B
 
 # run the inference
 ./main -m ./models/7B/ggml-model-q4_0.bin -t 8 -n 128
 ```
 
-For the bigger models, there are a few extra quantization steps. For example, for LLaMA-13B, converting to FP16 format
-will create 2 ggml files, instead of one:
-
-```bash
-ggml-model-f16.bin
-ggml-model-f16.bin.1
-```
-
-You need to quantize each of them separately like this:
-
-```bash
-./quantize ./models/13B/ggml-model-f16.bin   ./models/13B/ggml-model-q4_0.bin 2
-./quantize ./models/13B/ggml-model-f16.bin.1 ./models/13B/ggml-model-q4_0.bin.1 2
-```
-
-Everything else is the same. Simply run:
-
-```bash
-./main -m ./models/13B/ggml-model-q4_0.bin -t 8 -n 128
-```
-
-The number of files generated for each model is as follows:
-
-```
-7B  -> 1 file
-13B -> 2 files
-30B -> 4 files
-65B -> 8 files
-```
-
 When running the larger models, make sure you have enough disk space to store all the intermediate files.
 
+TODO: add model disk/mem requirements
+
 ### Interactive mode
 
 If you want a more ChatGPT-like experience, you can run in interactive mode by passing `-i` as a parameter.
@@ -205,16 +177,47 @@ Note the use of `--color` to distinguish between user input and generated text.
 
 ![image](https://user-images.githubusercontent.com/1991296/224575029-2af3c7dc-5a65-4f64-a6bb-517a532aea38.png)
 
+### Android
+
+You can easily run `llama.cpp` on Android device with [termux](https://play.google.com/store/apps/details?id=com.termux).
+First, obtain the [Android NDK](https://developer.android.com/ndk) and then build with CMake:
+```
+$ mkdir build-android
+$ cd build-android
+$ export NDK=<your_ndk_directory>
+$ cmake -DCMAKE_TOOLCHAIN_FILE=$NDK/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-23 -DCMAKE_C_FLAGS=-march=armv8.4a+dotprod ..
+$ make
+```
+Install [termux](https://play.google.com/store/apps/details?id=com.termux) on your device and run `termux-setup-storage` to get access to your SD card.
+Finally, copy the `llama` binary and the model files to your device storage. Here is a demo of an interactive session running on Pixel 5 phone:
+
+https://user-images.githubusercontent.com/271616/225014776-1d567049-ad71-4ef2-b050-55b0b3b9274c.mp4
+
+
 ## Limitations
 
-- Not sure if my tokenizer is correct. There are a few places where we might have a mistake:
-  - https://github.com/ggerganov/llama.cpp/blob/26c084662903ddaca19bef982831bfb0856e8257/convert-pth-to-ggml.py#L79-L87
-  - https://github.com/ggerganov/llama.cpp/blob/26c084662903ddaca19bef982831bfb0856e8257/utils.h#L65-L69
-  In general, it seems to work, but I think it fails for unicode character support. Hopefully, someone can help with that
-- I don't know yet how much the quantization affects the quality of the generated text
+- We don't know yet how much the quantization affects the quality of the generated text
 - Probably the token sampling can be improved
 - The Accelerate framework is actually currently unused since I found that for tensor shapes typical for the Decoder,
   there is no benefit compared to the ARM_NEON intrinsics implementation. Of course, it's possible that I simlpy don't
   know how to utilize it properly. But in any case, you can even disable it with `LLAMA_NO_ACCELERATE=1 make` and the
   performance will be the same, since no BLAS calls are invoked by the current implementation
 
+### Contributing
+
+- Contributors can open PRs
+- Collaborators can push to branches in the `llama.cpp` repo
+- Collaborators will be invited based on contributions
+
+### Coding guidelines
+
+- Avoid adding third-party dependencies, extra files, extra headers, etc.
+- Always consider cross-compatibility with other operating systems and architectures
+- Avoid fancy looking modern STL constructs, use basic `for` loops, avoid templates, keep it simple
+- There are no strict rules for the code style, but try to follow the patterns in the code (indentation, spaces, etc.). Vertical alignment makes things more readable and easier to batch edit
+- Clean-up any trailing whitespaces, use 4 spaces indentation, brackets on same line, `void * ptr`, `int & a`
+- See [good first issues](https://github.com/ggerganov/llama.cpp/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) for tasks suitable for first contributions
+
+### Misc
+
+- Practice your C++ typing skills: https://typing-battles.ggerganov.com

convert-pth-to-ggml.py

@@ -22,7 +22,6 @@ import json
 import struct
 import numpy as np
 import torch
-
 from sentencepiece import SentencePieceProcessor
 
 if len(sys.argv) < 3:
@@ -101,12 +100,28 @@ for p in range(n_parts):
 
     # Is this correct??
     for i in range(32000):
-        # TODO: this is probably wrong - not sure how this tokenizer works
-        text = tokenizer.decode([29889, i]).encode('utf-8')
-        # remove the first byte (it's always '.')
-        text = text[1:]
-        fout.write(struct.pack("i", len(text)))
-        fout.write(text)
+        if tokenizer.is_unknown(i):
+            # "<unk>" token (translated as ??)
+            text = " \u2047 ".encode("utf-8")
+            fout.write(struct.pack("i", len(text)))
+            fout.write(text)
+        elif tokenizer.is_control(i):
+            # "<s>"/"</s>" tokens
+            fout.write(struct.pack("i", 0))
+        elif tokenizer.is_byte(i):
+            # "<U+XX>" tokens (which may be invalid UTF-8)
+            piece = tokenizer.id_to_piece(i)
+            if len(piece) != 6:
+                print("Invalid token: " + piece)
+                sys.exit(1)
+            byte_value = int(piece[3:-1], 16)
+            fout.write(struct.pack("i", 1))
+            fout.write(struct.pack("B", byte_value))
+        else:
+            # normal token. Uses U+2581 (LOWER ONE EIGHTH BLOCK) to represent spaces.
+            text = tokenizer.id_to_piece(i).replace("\u2581", " ").encode("utf-8")
+            fout.write(struct.pack("i", len(text)))
+            fout.write(text)
 
     for k, v in model.items():
         name = k

ggml.c

@@ -1,8 +1,9 @@
+#define _GNU_SOURCE
 #include "ggml.h"
 
 #if defined(_MSC_VER) || defined(__MINGW32__)
-#include <malloc.h> // using malloc.h with MSC/MINGW
-#elif !defined(__FreeBSD__)
+//#include <malloc.h> // using malloc.h with MSC/MINGW
+#elif !defined(__FreeBSD__) && !defined(__NetBSD__)
 #include <alloca.h>
 #endif
 
@@ -1359,8 +1360,24 @@ inline static void ggml_vec_dot_q4_0(const int n, float * restrict s, const void
         const int8x16_t v0_1hs = vsubq_s8(v0_1h, s8b);
         const int8x16_t v1_1hs = vsubq_s8(v1_1h, s8b);
 
+#if defined(__ARM_FEATURE_DOTPROD)
         // dot product into int16x8_t
-        const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0ls), vget_low_s8 (v1_0ls));
+        int32x4_t p_0 = vdotq_s32(vdupq_n_s32(0), v0_0ls, v1_0ls);
+        int32x4_t p_1 = vdotq_s32(vdupq_n_s32(0), v0_1ls, v1_1ls);
+
+        p_0 = vdotq_s32(p_0, v0_0hs, v1_0hs);
+        p_1 = vdotq_s32(p_1, v0_1hs, v1_1hs);
+
+        // scalar
+#if defined(__ARM_FEATURE_QRDMX)
+        sum0 += d0_0*d1_0*vaddvq_s32(p_0);
+        sum1 += d0_1*d1_1*vaddvq_s32(p_1);
+#else
+        sum0 += d0_0*d1_0*(vgetq_lane_s32(p_0, 0) + vgetq_lane_s32(p_0, 1) + vgetq_lane_s32(p_0, 2) + vgetq_lane_s32(p_0, 3));
+        sum1 += d0_1*d1_1*(vgetq_lane_s32(p_1, 0) + vgetq_lane_s32(p_1, 1) + vgetq_lane_s32(p_1, 2) + vgetq_lane_s32(p_1, 3));
+#endif
+#else
+	    const int16x8_t pl0l = vmull_s8(vget_low_s8 (v0_0ls), vget_low_s8 (v1_0ls));
         const int16x8_t pl0h = vmull_s8(vget_high_s8(v0_0ls), vget_high_s8(v1_0ls));
 
         const int16x8_t ph0l = vmull_s8(vget_low_s8 (v0_0hs), vget_low_s8 (v1_0hs));
@@ -1388,6 +1405,7 @@ inline static void ggml_vec_dot_q4_0(const int n, float * restrict s, const void
 #else
         sum0 += d0_0*d1_0*(vgetq_lane_s16(p_0, 0) + vgetq_lane_s16(p_0, 1) + vgetq_lane_s16(p_0, 2) + vgetq_lane_s16(p_0, 3) + vgetq_lane_s16(p_0, 4) + vgetq_lane_s16(p_0, 5) + vgetq_lane_s16(p_0, 6) + vgetq_lane_s16(p_0, 7));
         sum1 += d0_1*d1_1*(vgetq_lane_s16(p_1, 0) + vgetq_lane_s16(p_1, 1) + vgetq_lane_s16(p_1, 2) + vgetq_lane_s16(p_1, 3) + vgetq_lane_s16(p_1, 4) + vgetq_lane_s16(p_1, 5) + vgetq_lane_s16(p_1, 6) + vgetq_lane_s16(p_1, 7));
+#endif
 #endif
     }
 

main.cpp

@@ -1,21 +1,41 @@
+#if defined(_MSC_VER) || defined(__MINGW32__)
+#define NOMINMAX
+#endif
+
 #include "ggml.h"
 
 #include "utils.h"
+#include "mmap.h"
 
 #include <cassert>
 #include <cmath>
 #include <cstdio>
+#include <cerrno>
 #include <cstring>
 #include <fstream>
 #include <map>
 #include <string>
 #include <vector>
+#include <atomic>
 
 #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
+#include <fcntl.h>
 #include <signal.h>
 #include <unistd.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#elif defined (_POSIX_MAPPED_FILES)
+#include <sys/mman.h>
 #endif
 
+#define ROUNDUP(X, K) (((X) + (K)-1) & -(K))
+#define IS2POW(X) (!((X) & ((X)-1)))
+
+#define MAGIC_PATH "magic.dat"
+#define MAGIC_ADDR (char *)0x330000000000
+#define MAGIC_GRAN 65536
+#define MAGIC_ALGN (sizeof(size_t) * 2)
+
 #define ANSI_COLOR_RED     "\x1b[31m"
 #define ANSI_COLOR_GREEN   "\x1b[32m"
 #define ANSI_COLOR_YELLOW  "\x1b[33m"
@@ -83,11 +103,188 @@ struct llama_model {
     std::map<std::string, struct ggml_tensor *> tensors;
 };
 
+struct magic {
+    uint32_t magic;
+    std::atomic<unsigned> lock;
+    int fd;
+    uint64_t commit;
+    uint64_t offset;
+    uint64_t capacity;
+    gpt_vocab *vocab;
+    llama_model *model;
+};
+
+static struct magic *mag;
+
+static inline void spin_lock(std::atomic<unsigned> &lock) {
+    while (lock.exchange(1, std::memory_order_acquire));
+}
+
+static inline void spin_unlock(std::atomic<unsigned> &lock) {
+    lock.store(0, std::memory_order_release);
+}
+
+static void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
+    void *res;
+    res = mmap(addr, length, prot, flags, fd, offset);
+    if (res != MAP_FAILED) return res;
+    perror("mmap");
+    exit(77);
+}
+
+static void magic_commit(void) {
+    mag->commit = ROUNDUP(mag->offset, MAGIC_GRAN);
+    mag->magic = 0xFEEDABEE;
+    if (msync(mag, mag->commit, MS_ASYNC) == -1) {
+        perror("msync");
+        exit(77);
+    }
+}
+
+static void magic_init(void) {
+    int fd;
+    size_t n;
+    int64_t size;
+    if (mag) return;
+    n = ROUNDUP(sizeof(struct magic), MAGIC_GRAN);
+    if ((fd = open(MAGIC_PATH, O_RDWR)) != -1) {
+        if ((size = lseek(fd, 0, SEEK_END)) == -1) {
+            perror("lseek");
+            exit(77);
+        }
+        if (size >= n) {
+            mag = (struct magic *)Mmap(MAGIC_ADDR, n,
+                                       PROT_READ | PROT_WRITE,
+                                       MAP_PRIVATE | MAP_FIXED, fd, 0);
+            if (mag->magic == 0xFEEDABEE) {
+                mag = (struct magic *)Mmap(MAGIC_ADDR, mag->commit,
+                                           PROT_READ | PROT_WRITE,
+                                           MAP_PRIVATE | MAP_FIXED, fd, 0);
+                madvise(MAGIC_ADDR, mag->capacity, MADV_WILLNEED);
+                mag->offset = mag->commit;
+                mag->capacity = mag->commit;
+                mag->fd = -1;
+                return;
+            }
+        }
+        if (ftruncate(fd, 0) == -1) {
+            perror("ftruncate");
+            exit(77);
+        }
+    } else if ((fd = open(MAGIC_PATH, O_RDWR | O_CREAT | O_TRUNC, 0644)) == -1) {
+        perror(MAGIC_PATH);
+        exit(77);
+    }
+    if (ftruncate(fd, n) == -1) {
+        perror("ftruncate");
+        exit(77);
+    }
+    mag = (struct magic *)Mmap(MAGIC_ADDR, n,
+                               PROT_READ | PROT_WRITE,
+                               MAP_SHARED | MAP_FIXED, fd, 0);
+    mag->offset = n;
+    mag->capacity = n;
+    mag->fd = fd;
+}
+
+void *magic_memalign(size_t a, size_t n) {
+    void *p;
+    static int count;
+    size_t i, j, k, m, c2;
+    magic_init();
+    if (a < MAGIC_ALGN) a = MAGIC_ALGN;
+    while (!IS2POW(a)) ++a;
+    m = n ? n : 1;
+    spin_lock(mag->lock);
+    i = mag->offset;
+    i = i + sizeof(size_t);
+    i = ROUNDUP(i, a);
+    j = ROUNDUP(i + m, MAGIC_GRAN);
+    if (j > mag->capacity) {
+        c2 = mag->capacity;
+        if (!c2) {
+            c2 = MAGIC_GRAN;
+        }
+        while (j > c2) {
+            c2 += c2 >> 4;
+            c2 = ROUNDUP(c2, MAGIC_GRAN);
+        }
+        if (!mag->magic) {
+            if (ftruncate(mag->fd, c2) == -1) {
+                perror("ftruncate");
+                spin_unlock(mag->lock);
+                return 0;
+            }
+            p = mmap(MAGIC_ADDR + mag->capacity,
+                     c2 - mag->capacity, PROT_READ | PROT_WRITE,
+                     MAP_SHARED | MAP_FIXED, mag->fd, mag->capacity);
+        } else {
+            p = mmap(MAGIC_ADDR + mag->capacity,
+                     c2 - mag->capacity, PROT_READ | PROT_WRITE,
+                     MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
+        }
+        if (p != MAP_FAILED) {
+            mag->capacity = c2;
+        } else {
+            perror("mmap");
+            spin_unlock(mag->lock);
+            return 0;
+        }
+    }
+    mag->offset = i + m;
+    spin_unlock(mag->lock);
+    p = MAGIC_ADDR + i;
+    ((size_t *)p)[-1] = n;
+    return p;
+}
+
+void *magic_malloc(size_t n) {
+    return magic_memalign(MAGIC_ALGN, n);
+}
+
+void *magic_calloc(size_t n, size_t z) {
+    void *p;
+    if ((p = magic_malloc((n *= z)))) {
+        memset(p, 0, n);
+    }
+    return p;
+}
+
+void magic_free(void *p) {
+    // do nothing
+}
+
+void *magic_realloc(void *p, size_t n) {
+    void *q;
+    if (!p) {
+        return magic_malloc(n);
+    }
+    if (!n) {
+        magic_free(p);
+        return 0;
+    }
+    if ((q = magic_malloc(n))) {
+        memcpy(q, p, ((const size_t *)p)[-1]);
+    }
+    return q;
+}
+
+void* operator new(size_t size) {
+    return magic_malloc(size);
+}
+
+void operator delete(void* p) {
+    magic_free(p);
+}
+
 // load the model's weights from a file
 bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab & vocab, int n_ctx) {
-    printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+    fprintf(stderr, "%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
+
+    std::vector<char> f_buf(1024*1024);
 
     auto fin = std::ifstream(fname, std::ios::binary);
+    fin.rdbuf()->pubsetbuf(f_buf.data(), f_buf.size());
     if (!fin) {
         fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
         return false;
@@ -124,16 +321,16 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
         n_ff = ((2*(4*hparams.n_embd)/3 + hparams.n_mult - 1)/hparams.n_mult)*hparams.n_mult;
         n_parts = LLAMA_N_PARTS.at(hparams.n_embd);
 
-        printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
-        printf("%s: n_ctx   = %d\n", __func__, hparams.n_ctx);
-        printf("%s: n_embd  = %d\n", __func__, hparams.n_embd);
-        printf("%s: n_mult  = %d\n", __func__, hparams.n_mult);
-        printf("%s: n_head  = %d\n", __func__, hparams.n_head);
-        printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
-        printf("%s: n_rot   = %d\n", __func__, hparams.n_rot);
-        printf("%s: f16     = %d\n", __func__, hparams.f16);
-        printf("%s: n_ff    = %d\n", __func__, n_ff);
-        printf("%s: n_parts = %d\n", __func__, n_parts);
+        fprintf(stderr, "%s: n_vocab = %d\n", __func__, hparams.n_vocab);
+        fprintf(stderr, "%s: n_ctx   = %d\n", __func__, hparams.n_ctx);
+        fprintf(stderr, "%s: n_embd  = %d\n", __func__, hparams.n_embd);
+        fprintf(stderr, "%s: n_mult  = %d\n", __func__, hparams.n_mult);
+        fprintf(stderr, "%s: n_head  = %d\n", __func__, hparams.n_head);
+        fprintf(stderr, "%s: n_layer = %d\n", __func__, hparams.n_layer);
+        fprintf(stderr, "%s: n_rot   = %d\n", __func__, hparams.n_rot);
+        fprintf(stderr, "%s: f16     = %d\n", __func__, hparams.f16);
+        fprintf(stderr, "%s: n_ff    = %d\n", __func__, n_ff);
+        fprintf(stderr, "%s: n_parts = %d\n", __func__, n_parts);
     }
 
     // load vocab
@@ -158,7 +355,7 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
             vocab.id_to_token[i] = word;
 
             //if (i < 30000) {
-            //    printf("%s: vocab[%d] = '%s'\n", __func__, i, word.c_str());
+            //    fprintf(stderr, "%s: vocab[%d] = '%s'\n", __func__, i, word.c_str());
             //}
         }
     }
@@ -217,14 +414,14 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
 
         ctx_size += (5 + 10*n_layer)*256; // object overhead
 
-        printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
+        fprintf(stderr, "%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
     }
 
     // create the ggml context
     {
         struct ggml_init_params params = {
             /*.mem_size   =*/ ctx_size,
-            /*.mem_buffer =*/ NULL,
+            /*.mem_buffer =*/ magic_malloc(ctx_size),
         };
 
         model.ctx = ggml_init(params);
@@ -304,7 +501,7 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
 
         const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);
 
-        printf("%s: memory_size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
+        fprintf(stderr, "%s: memory_size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
     }
 
     const size_t file_offset = fin.tellg();
@@ -322,9 +519,10 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
             fname_part += "." + std::to_string(i);
         }
 
-        printf("%s: loading model part %d/%d from '%s'\n", __func__, i+1, n_parts, fname_part.c_str());
+        fprintf(stderr, "%s: loading model part %d/%d from '%s'\n", __func__, i+1, n_parts, fname_part.c_str());
 
         fin = std::ifstream(fname_part, std::ios::binary);
+        fin.rdbuf()->pubsetbuf(f_buf.data(), f_buf.size());
         fin.seekg(file_offset);
 
         // load weights
@@ -332,7 +530,7 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
             int n_tensors = 0;
             size_t total_size = 0;
 
-            printf("%s: ", __func__);
+            fprintf(stderr, "%s: ", __func__);
 
             while (true) {
                 int32_t n_dims;
@@ -432,7 +630,7 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
 
                 if (0) {
                     static const char * ftype_str[] = { "f32", "f16", "q4_0", "q4_1", };
-                    printf("%24s - [%5d, %5d], type = %6s, split = %d\n", name.data(), ne[0], ne[1], ftype_str[ftype], split_type);
+                    fprintf(stderr, "%24s - [%5d, %5d], type = %6s, split = %d\n", name.data(), ne[0], ne[1], ftype_str[ftype], split_type);
                 }
 
                 size_t bpe = 0;
@@ -495,16 +693,16 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
                     total_size += ggml_nbytes(tensor)/n_parts;
                 }
 
-                //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
+                //fprintf(stderr, "%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.data(), ne[0], ne[1], ftype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
                 if (++n_tensors % 8 == 0) {
-                    printf(".");
-                    fflush(stdout);
+                    fprintf(stderr, ".");
+                    fflush(stderr);
                 }
             }
 
-            printf(" done\n");
+            fprintf(stderr, " done\n");
 
-            printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
+            fprintf(stderr, "%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
         }
 
         fin.close();
@@ -548,7 +746,7 @@ bool llama_eval(
 
     if (mem_per_token > 0 && mem_per_token*N > buf_size) {
         const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
-        //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
+        //fprintf(stderr, "\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
 
         // reallocate
         buf_size = buf_size_new;
@@ -740,7 +938,7 @@ bool llama_eval(
     if (mem_per_token == 0) {
         mem_per_token = ggml_used_mem(ctx0)/N;
     }
-    //printf("used_mem = %zu\n", ggml_used_mem(ctx0));
+    //fprintf(stderr, "used_mem = %zu\n", ggml_used_mem(ctx0));
 
     ggml_free(ctx0);
 
@@ -761,7 +959,29 @@ void sigint_handler(int signo) {
 }
 #endif
 
+const char * llama_print_system_info(void) {
+    static std::string s;
+
+    s  = "";
+    s += "AVX = "       + std::to_string(ggml_cpu_has_avx())       + " | ";
+    s += "AVX2 = "      + std::to_string(ggml_cpu_has_avx2())      + " | ";
+    s += "AVX512 = "    + std::to_string(ggml_cpu_has_avx512())    + " | ";
+    s += "FMA = "       + std::to_string(ggml_cpu_has_fma())       + " | ";
+    s += "NEON = "      + std::to_string(ggml_cpu_has_neon())      + " | ";
+    s += "ARM_FMA = "   + std::to_string(ggml_cpu_has_arm_fma())   + " | ";
+    s += "F16C = "      + std::to_string(ggml_cpu_has_f16c())      + " | ";
+    s += "FP16_VA = "   + std::to_string(ggml_cpu_has_fp16_va())   + " | ";
+    s += "WASM_SIMD = " + std::to_string(ggml_cpu_has_wasm_simd()) + " | ";
+    s += "BLAS = "      + std::to_string(ggml_cpu_has_blas())      + " | ";
+    s += "SSE3 = "      + std::to_string(ggml_cpu_has_sse3())      + " | ";
+    s += "VSX = "       + std::to_string(ggml_cpu_has_vsx())       + " | ";
+
+    return s.c_str();
+}
+
 int main(int argc, char ** argv) {
+    magic_init();
+
     ggml_time_init();
     const int64_t t_main_start_us = ggml_time_us();
 
@@ -776,7 +996,7 @@ int main(int argc, char ** argv) {
         params.seed = time(NULL);
     }
 
-    printf("%s: seed = %d\n", __func__, params.seed);
+    fprintf(stderr, "%s: seed = %d\n", __func__, params.seed);
 
     std::mt19937 rng(params.seed);
     if (params.prompt.empty()) {
@@ -788,19 +1008,31 @@ int main(int argc, char ** argv) {
 
     int64_t t_load_us = 0;
 
-    gpt_vocab vocab;
-    llama_model model;
-
     // load the model
-    {
+    gpt_vocab *vocab;
+    llama_model *model;
+    if (!mag->magic) {
+        vocab = new gpt_vocab;
+        model = new llama_model;
         const int64_t t_start_us = ggml_time_us();
-
-        if (!llama_model_load(params.model, model, vocab, 512)) {  // TODO: set context from user input ??
+        if (!llama_model_load(params.model, *model, *vocab, 512)) {  // TODO: set context from user input ??
             fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
             return 1;
         }
-
         t_load_us = ggml_time_us() - t_start_us;
+        mag->vocab = vocab;
+        mag->model = model;
+        magic_commit();
+    } else {
+        vocab = mag->vocab;
+        model = mag->model;
+    }
+
+    // print system information
+    {
+        fprintf(stderr, "\n");
+        fprintf(stderr, "system_info: n_threads = %d / %d | %s\n",
+                params.n_threads, std::thread::hardware_concurrency(), llama_print_system_info());
     }
 
     int n_past = 0;
@@ -811,48 +1043,48 @@ int main(int argc, char ** argv) {
     std::vector<float> logits;
 
     // tokenize the prompt
-    std::vector<gpt_vocab::id> embd_inp = ::llama_tokenize(vocab, params.prompt, true);
+    std::vector<gpt_vocab::id> embd_inp = ::llama_tokenize(*vocab, params.prompt, true);
 
-    params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
+    params.n_predict = std::min(params.n_predict, model->hparams.n_ctx - (int) embd_inp.size());
 
     // tokenize the reverse prompt
-    std::vector<gpt_vocab::id> antiprompt_inp = ::llama_tokenize(vocab, params.antiprompt, false);
+    std::vector<gpt_vocab::id> antiprompt_inp = ::llama_tokenize(*vocab, params.antiprompt, false);
 
-    printf("\n");
-    printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str());
-    printf("%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size());
+    fprintf(stderr, "\n");
+    fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str());
+    fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size());
     for (int i = 0; i < (int) embd_inp.size(); i++) {
-        printf("%6d -> '%s'\n", embd_inp[i], vocab.id_to_token.at(embd_inp[i]).c_str());
+        fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], vocab->id_to_token.at(embd_inp[i]).c_str());
     }
-    printf("\n");
+    fprintf(stderr, "\n");
     if (params.interactive) {
 #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
         struct sigaction sigint_action;
         sigint_action.sa_handler = sigint_handler;
         sigemptyset (&sigint_action.sa_mask);
-        sigint_action.sa_flags = 0; 
+        sigint_action.sa_flags = 0;
         sigaction(SIGINT, &sigint_action, NULL);
 #endif
 
-        printf("%s: interactive mode on.\n", __func__);
+        fprintf(stderr, "%s: interactive mode on.\n", __func__);
 
         if(antiprompt_inp.size()) {
-            printf("%s: reverse prompt: '%s'\n", __func__, params.antiprompt.c_str());
-            printf("%s: number of tokens in reverse prompt = %zu\n", __func__, antiprompt_inp.size());
+            fprintf(stderr, "%s: reverse prompt: '%s'\n", __func__, params.antiprompt.c_str());
+            fprintf(stderr, "%s: number of tokens in reverse prompt = %zu\n", __func__, antiprompt_inp.size());
             for (int i = 0; i < (int) antiprompt_inp.size(); i++) {
-                printf("%6d -> '%s'\n", antiprompt_inp[i], vocab.id_to_token.at(antiprompt_inp[i]).c_str());
+                fprintf(stderr, "%6d -> '%s'\n", antiprompt_inp[i], vocab->id_to_token.at(antiprompt_inp[i]).c_str());
             }
-            printf("\n");
+            fprintf(stderr, "\n");
         }
     }
-    printf("sampling parameters: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n", params.temp, params.top_k, params.top_p, params.repeat_last_n, params.repeat_penalty);
-    printf("\n\n");
+    fprintf(stderr, "sampling parameters: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n", params.temp, params.top_k, params.top_p, params.repeat_last_n, params.repeat_penalty);
+    fprintf(stderr, "\n\n");
 
     std::vector<gpt_vocab::id> embd;
 
     // determine the required inference memory per token:
     size_t mem_per_token = 0;
-    llama_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
+    llama_eval(*model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
 
     int last_n_size = params.repeat_last_n;
     std::vector<gpt_vocab::id> last_n_tokens(last_n_size);
@@ -860,7 +1092,7 @@ int main(int argc, char ** argv) {
 
 
     if (params.interactive) {
-        printf("== Running in interactive mode. ==\n"
+        fprintf(stderr, "== Running in interactive mode. ==\n"
 #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
                " - Press Ctrl+C to interject at any time.\n"
 #endif
@@ -887,8 +1119,8 @@ int main(int argc, char ** argv) {
         if (embd.size() > 0) {
             const int64_t t_start_us = ggml_time_us();
 
-            if (!llama_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) {
-                printf("Failed to predict\n");
+            if (!llama_eval(*model, params.n_threads, n_past, embd, logits, mem_per_token)) {
+                fprintf(stderr, "Failed to predict\n");
                 return 1;
             }
 
@@ -905,14 +1137,14 @@ int main(int argc, char ** argv) {
             const float temp  = params.temp;
             const float repeat_penalty = params.repeat_penalty;
 
-            const int n_vocab = model.hparams.n_vocab;
+            const int n_vocab = model->hparams.n_vocab;
 
             gpt_vocab::id id = 0;
 
             {
                 const int64_t t_start_sample_us = ggml_time_us();
 
-                id = llama_sample_top_p_top_k(vocab, logits.data() + (logits.size() - n_vocab), last_n_tokens, repeat_penalty, top_k, top_p, temp, rng);
+                id = llama_sample_top_p_top_k(*vocab, logits.data() + (logits.size() - n_vocab), last_n_tokens, repeat_penalty, top_k, top_p, temp, rng);
 
                 last_n_tokens.erase(last_n_tokens.begin());
                 last_n_tokens.push_back(id);
@@ -939,16 +1171,17 @@ int main(int argc, char ** argv) {
                     break;
                 }
             }
+
+            // reset color to default if we there is no pending user input
+            if (!input_noecho && params.use_color && embd_inp.size() == input_consumed) {
+                printf(ANSI_COLOR_RESET);
+            }
         }
 
         // display text
         if (!input_noecho) {
             for (auto id : embd) {
-                printf("%s", vocab.id_to_token[id].c_str());
-            }
-            // reset color to default if we there is no pending user input
-            if (params.use_color && embd_inp.size() <= input_consumed) {
-                printf(ANSI_COLOR_RESET);
+                printf("%s", vocab->id_to_token[id].c_str());
             }
             fflush(stdout);
         }
@@ -962,7 +1195,7 @@ int main(int argc, char ** argv) {
                 is_interacting = true;
             }
             if (is_interacting) {
-                // currently being interactive 
+                // currently being interactive
                 bool another_line=true;
                 while (another_line) {
                     fflush(stdout);
@@ -986,7 +1219,7 @@ int main(int argc, char ** argv) {
                         buf[n_read+1] = 0;
                     }
 
-                    std::vector<gpt_vocab::id> line_inp = ::llama_tokenize(vocab, buf, false);
+                    std::vector<gpt_vocab::id> line_inp = ::llama_tokenize(*vocab, buf, false);
                     embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end());
 
                     remaining_tokens -= line_inp.size();
@@ -994,13 +1227,13 @@ int main(int argc, char ** argv) {
                     input_noecho = true; // do not echo this again
                 }
 
-                is_interacting = false;            
+                is_interacting = false;
             }
         }
 
         // end of text token
         if (embd.back() == 2) {
-            printf(" [end of text]\n");
+            fprintf(stderr, " [end of text]\n");
             break;
         }
     }
@@ -1010,15 +1243,15 @@ int main(int argc, char ** argv) {
     {
         const int64_t t_main_end_us = ggml_time_us();
 
-        printf("\n\n");
-        printf("%s: mem per token = %8zu bytes\n", __func__, mem_per_token);
-        printf("%s:     load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
-        printf("%s:   sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
-        printf("%s:  predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
-        printf("%s:    total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
+        fprintf(stderr, "\n\n");
+        fprintf(stderr, "%s: mem per token = %8zu bytes\n", __func__, mem_per_token);
+        fprintf(stderr, "%s:     load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
+        fprintf(stderr, "%s:   sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
+        fprintf(stderr, "%s:  predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
+        fprintf(stderr, "%s:    total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
     }
 
-    ggml_free(model.ctx);
+    ggml_free(model->ctx);
 
     return 0;
 }

mmap.c

@@ -0,0 +1,570 @@
+// Lightweight Portable mmap() Polyfill
+//
+// 1. Supports POSIX.1
+//
+//    The baseline POSIX standard doesn't specify MAP_ANONYMOUS. This
+//    library makes sure, on the hypothetical UNIX systems that don't
+//    have it, or on the mainstream UNIX platforms where the user has
+//    chosen to define _POSIX_C_SOURCE that cause headers to undefine
+//    it, this implementation will fallback to creating a secure temp
+//    file, for each anonymous mapping.
+//
+// 2. Supports Windows w/ Visual Studio
+//
+//    On Windows Vista and later an API exists that's almost as good as
+//    mmap(). However code that uses this library should conform to the
+//    subset of behaviors Microsoft accommodates.
+//
+// Caveats
+//
+// - You should just assume the page size is 64kb. That's how it is on
+//   Windows and it usually goes faster to assume that elsewhere too.
+//
+// - Not designed to support mprotect() at the moment. In order to
+//   support this, we'd need to consider _open(O_ACCMODE) on Windows
+//   and then have mmap() be more greedy about permissions.
+//
+// - There's limited support for being clever with memory intervals.
+//   For example, you can't punch a hole in a memory map on Windows.
+//   This abstraction does aim to offer more flexibility than WIN32.
+//   There should also be good error reporting for unsupported uses.
+
+#include "mmap.h"
+
+#ifdef NEED_POSIX_MMAP
+#include <stdlib.h>
+
+void *PosixMmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
+    int tfd;
+    void* res;
+    char path[] = "/tmp/llama.dat.XXXXXX";
+    if (~flags & MAP_ANONYMOUS) {
+        res = mmap(addr, length, prot, flags, fd, offset);
+    } else if ((tfd = mkstemp(path)) != -1) {
+        unlink(path);
+        if (!ftruncate(tfd, length)) {
+            res = mmap(addr, length, prot, flags & ~MAP_ANONYMOUS, tfd, 0);
+        } else {
+            res = MAP_FAILED;
+        }
+        close(tfd);
+    } else {
+        res = MAP_FAILED;
+    }
+    return res;
+}
+
+#elif defined(NEED_WIN32_MMAP)
+#include <errno.h>
+#include <stdio.h>
+#include <assert.h>
+#include <inttypes.h>
+
+struct WinMap {        // O(n) no ordering no overlaps
+    HANDLE hand;       // zero means array slots empty
+    HANDLE fand;       // for the original file, or -1
+    uintptr_t addr;    // base address (64 kb aligned)
+    uintptr_t length;  // byte size (>0, rounded 64kb)
+};
+
+struct WinMaps {
+    int n;
+    struct WinMap *p;
+    volatile long lock;
+};
+
+static struct WinMaps g_winmaps;
+
+static inline uintptr_t Min(uintptr_t x, uintptr_t y) {
+    return y > x ? x : y;
+}
+
+static inline uintptr_t Max(uintptr_t x, uintptr_t y) {
+    return y < x ? x : y;
+}
+
+static inline uintptr_t Roundup(uintptr_t x, intptr_t a) {
+    assert(a > 0);
+    assert(!(a & (a - 1)));
+    return (x + (a - 1)) & -a;
+}
+
+static inline void Lock(void) {
+    long x;
+    for (;;) {
+        x = InterlockedExchange(&g_winmaps.lock, 1);
+        if (!x) break;
+        assert(x == 1);
+    }
+}
+
+static inline void Unlock(void) {
+    assert(g_winmaps.lock == 1);
+    g_winmaps.lock = 0;
+}
+
+static int WinStrerror(int err, char *buf, int size) {
+    return FormatMessageA(
+        FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+        NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+        buf, size, NULL);
+}
+
+#ifdef NDEBUG
+#define LogError(thing) (void)0
+#else
+static void LogError(const char* file, int line, const char* thing) {
+#define LogError(thing) LogError(__FILE__, __LINE__, thing)
+    fprintf(stderr, "%s:%d: error: %s\n", file, line, thing);
+}
+#endif
+
+#ifdef NDEBUG
+#define LogWindowsError(thing) (void)0
+#else
+static void LogWindowsError(const char* file, int line, const char* thing) {
+#define LogWindowsError(thing) LogWindowsError(__FILE__, __LINE__, thing)
+    char s[256];
+    int e = GetLastError();
+    WinStrerror(e, s, sizeof(s));
+    fprintf(stderr, "%s:%d: error[%#x]: %s failed: %s\n", file, line, e, thing, s);
+}
+#endif
+
+static void *Recalloc(void *ptr, uint64_t newSize) {
+    HANDLE heap = GetProcessHeap();
+    if (!ptr) {
+        return HeapAlloc(heap, HEAP_ZERO_MEMORY, newSize);
+    }
+    if (!newSize) {
+        HeapFree(heap, 0, ptr);
+        return 0;
+    }
+    return HeapReAlloc(heap, HEAP_ZERO_MEMORY, ptr, newSize);
+}
+
+uint64_t WinSeek(int fd, uint64_t offset, int whence) {
+    HANDLE hFile;
+    DWORD winwhence;
+    LARGE_INTEGER distanceToMove;
+    LARGE_INTEGER newFilePointer;
+    distanceToMove.QuadPart = offset;
+    switch (whence) {
+    case SEEK_SET:
+        winwhence = FILE_BEGIN;
+        break;
+    case SEEK_CUR:
+        winwhence = FILE_CURRENT;
+        break;
+    case SEEK_END:
+        winwhence = FILE_END;
+        break;
+    default:
+        LogError("bad lseek() whence");
+        errno = EINVAL;
+        return -1;
+    }
+    hFile = (HANDLE)_get_osfhandle(fd);
+    if (hFile == INVALID_HANDLE_VALUE) {
+        LogWindowsError("_get_osfhandle");
+        errno = EBADF;
+        return -1;
+    }
+    if (GetFileType(hFile) != FILE_TYPE_DISK) {
+        LogError("bad file type for lseek()");
+        errno = ESPIPE;
+        return -1;
+    }
+    if (!SetFilePointerEx(hFile, distanceToMove, &newFilePointer, winwhence)) {
+        LogWindowsError("SetFilePointerEx");
+        errno = EPERM;
+        return -1;
+    }
+    return newFilePointer.QuadPart;
+}
+
+int WinFtruncate(int fd, uint64_t length) {
+    HANDLE hFile;
+    LARGE_INTEGER old, neu;
+    hFile = (HANDLE)_get_osfhandle(fd);
+    if (hFile == INVALID_HANDLE_VALUE) {
+        LogWindowsError("_get_osfhandle");
+        errno = EBADF;
+        return -1;
+    }
+    // save current file position
+    old.QuadPart = 0;
+    neu.QuadPart = 0;
+    if (!SetFilePointerEx(hFile, neu, &old, FILE_CURRENT)) {
+        LogWindowsError("SetFilePointerEx#1");
+        return -1;
+    }
+    // set current position to new file size
+    neu.QuadPart = length;
+    if (!SetFilePointerEx(hFile, neu, NULL, FILE_BEGIN)) {
+        LogWindowsError("SetFilePointerEx#2");
+        return -1;
+    }
+    // change the file size
+    if (!SetEndOfFile(hFile)) {
+        LogWindowsError("SetEndOfFile");
+        SetFilePointerEx(hFile, old, NULL, FILE_BEGIN);
+        return -1;
+    }
+    // restore the original file position
+    // win32 allows this to exceed the end of file
+    if (!SetFilePointerEx(hFile, old, NULL, FILE_BEGIN)) {
+        LogWindowsError("SetFilePointerEx>3");
+        return -1;
+    }
+    return 0;
+}
+
+int WinMadvise(void *addr, uintptr_t length, int advice) {
+    switch (advice) {
+    case MADV_NORMAL:
+    case MADV_DONTNEED:
+    case MADV_SEQUENTIAL:
+        return 0;
+    case MADV_RANDOM:
+    case MADV_WILLNEED: {
+        HANDLE proc;
+        WIN32_MEMORY_RANGE_ENTRY entry;
+        proc = GetCurrentProcess();
+        entry.VirtualAddress = addr;
+        entry.NumberOfBytes = length;
+        if (!PrefetchVirtualMemory(proc, 1, &entry, 0)) {
+            LogWindowsError("PrefetchVirtualMemory");
+            errno = ENOMEM;
+            return -1;
+        }
+        return 0;
+    }
+    default:
+        errno = EINVAL;
+        return -1;
+    }
+}
+
+int WinUnmap(void *addr, uintptr_t length) {
+    void *view;
+    HANDLE hand;
+    HANDLE fand;
+    int i, err = 0;
+    uintptr_t a, b;
+    uintptr_t x, y;
+    // compute the requested interval
+    // 1. length can't be zero
+    // 2. length is rounded up to the page size
+    // 3. addr must be aligned to page boundary
+    a = (uintptr_t)addr;
+    b = a + Roundup(length, 65536);
+    if (!length) {
+        LogError("tried to munmap zero bytes");
+        errno = EINVAL;
+        return -1;
+    }
+    if (a & 65535) {
+        LogError("tried to munmap an address that's not 64kb aligned");
+        errno = EINVAL;
+        return -1;
+    }
+    // 1. we permit unmapping multiple maps in one call
+    // 2. we don't care if the matched mappings aren't contiguous
+    // 3. it's an error if a matched mapping only partially overlaps
+    // 4. similar to close() we release all resources possible on error
+    Lock();
+    for (i = 0; i < g_winmaps.n; ++i) {
+        if (!g_winmaps.p[i].hand) {
+            // this array slot is empty
+            continue;
+        }
+        // compute overlap between known mapping and requested interval
+        x = Max(a, g_winmaps.p[i].addr);
+        y = Min(b, g_winmaps.p[i].addr + g_winmaps.p[i].length);
+        if (x >= y) {
+            // there isn't any overlap
+            continue;
+        }
+        if (y - x != g_winmaps.p[i].length) {
+            // requested interval partially overlapped this mapping
+            // therefore we can't unmap it and must report an error
+            LogError("tried to partially unmap a mapping");
+            err = ENOMEM;
+            continue;
+        }
+        // save the information we care about
+        view = (void *)g_winmaps.p[i].addr;
+        hand = g_winmaps.p[i].hand;
+        fand = g_winmaps.p[i].fand;
+        // delete this mapping from the global array
+        g_winmaps.p[i].hand = 0;
+        // perform the systems operations
+        // safe to release lock since g_winmaps.n is monotonic
+        Unlock();
+        if (!UnmapViewOfFile(view)) {
+            LogWindowsError("UnmapViewOfFile");
+        }
+        if (!CloseHandle(hand)) {
+            LogWindowsError("CloseHandle#1");
+        }
+        if (fand != INVALID_HANDLE_VALUE) {
+            if (!CloseHandle(fand)) {
+                LogWindowsError("CloseHandle#2");
+            }
+        }
+        Lock();
+    }
+    Unlock();
+    if (err) {
+        errno = err;
+        return -1;
+    }
+    return 0;
+}
+
+void* WinMap(void *addr, uintptr_t length, int prot, int flags, int fd, uint64_t offset) {
+    int i;
+    LPVOID res;
+    HANDLE hand;
+    HANDLE hFile;
+    DWORD access;
+    DWORD wiprot;
+    uintptr_t fsize;
+    if (!length) {
+        LogError("mmap(length) was zero");
+        errno = EINVAL;
+        return MAP_FAILED;
+    }
+    length = Roundup(length, 65536);
+    if ((uintptr_t)addr & 65535) {
+        if (~flags & MAP_FIXED) {
+            addr = 0;
+        } else {
+            LogError("MAP_FIXED used with address that's not 64kb aligned");
+            errno = EINVAL;
+            return MAP_FAILED;
+        }
+    }
+    // these are the logical flag equivalents for creating mappings.  please
+    // note that any subsequent virtualprotect calls must be a subset of the
+    // permissions we're using here.  that's not a supported use case for us
+    if (flags & MAP_PRIVATE) {
+        // private mapping
+        if (prot & PROT_EXEC) {
+            if (prot & PROT_WRITE) {
+                if (flags & MAP_ANONYMOUS) {
+                    wiprot = PAGE_EXECUTE_READWRITE;
+                    access = FILE_MAP_READ | FILE_MAP_WRITE | FILE_MAP_EXECUTE;
+                } else {
+                    wiprot = PAGE_EXECUTE_WRITECOPY;
+                    access = FILE_MAP_COPY | FILE_MAP_EXECUTE;
+                }
+            } else {
+                wiprot = PAGE_EXECUTE_READ;
+                access = FILE_MAP_READ | FILE_MAP_EXECUTE;
+            }
+        } else if (prot & PROT_WRITE) {
+            if (flags & MAP_ANONYMOUS) {
+                wiprot = PAGE_READWRITE;
+                access = FILE_MAP_READ | FILE_MAP_WRITE;
+            } else {
+                wiprot = PAGE_WRITECOPY;
+                access = FILE_MAP_COPY;
+            }
+        } else {
+            wiprot = PAGE_READONLY;
+            access = FILE_MAP_READ;
+        }
+    } else {
+        // shared mapping
+        if (prot & PROT_EXEC) {
+            if (prot & PROT_WRITE) {
+                wiprot = PAGE_EXECUTE_READWRITE;
+                access = FILE_MAP_READ | FILE_MAP_WRITE | FILE_MAP_EXECUTE;
+            } else {
+                wiprot = PAGE_EXECUTE_READ;
+                access = FILE_MAP_READ | FILE_MAP_EXECUTE;
+            }
+        } else if (prot & PROT_WRITE) {
+            wiprot = PAGE_READWRITE;
+            access = FILE_MAP_READ | FILE_MAP_WRITE;
+        } else {
+            wiprot = PAGE_READONLY;
+            access = FILE_MAP_READ;
+        }
+    }
+    if (flags & MAP_ANONYMOUS) {
+        hFile = INVALID_HANDLE_VALUE;
+        fsize = length;
+        offset = 0;
+    } else {
+        fsize = 0;
+        hFile = (HANDLE)_get_osfhandle(fd);
+        if (hFile == INVALID_HANDLE_VALUE) {
+            LogWindowsError("_get_osfhandle");
+            errno = EBADF;
+            return MAP_FAILED;
+        }
+        if (!DuplicateHandle(GetCurrentProcess(), hFile,
+                             GetCurrentProcess(), &hFile,
+                             0, FALSE, DUPLICATE_SAME_ACCESS)) {
+            LogWindowsError("DuplicateHandle");
+            errno = EBADF;
+            return MAP_FAILED;
+        }
+    }
+    if (flags & MAP_FIXED) {
+        if (!addr) {
+            // zero chance of microsoft letting us map the null page
+            if (hFile != INVALID_HANDLE_VALUE) {
+                CloseHandle(hFile);
+            }
+            errno = EINVAL;
+            return MAP_FAILED;
+        } else {
+            // blow away any existing mappings on requested interval
+            if (WinUnmap(addr, length) == -1) {
+                // can only happen if we partially overlap an existing mapping
+                assert(errno == ENOMEM);
+                if (hFile != INVALID_HANDLE_VALUE) {
+                    CloseHandle(hFile);
+                }
+                return MAP_FAILED;
+            }
+        }
+    }
+    hand = CreateFileMapping(hFile, 0, wiprot,
+                             (DWORD)(fsize >> 32),
+                             (DWORD)fsize,
+                             0);
+    if (!hand) {
+        LogWindowsError("CreateFileMapping");
+        if (hFile != INVALID_HANDLE_VALUE) {
+            CloseHandle(hFile);
+        }
+        errno = EPERM;
+        return MAP_FAILED;
+    }
+    res = MapViewOfFileEx(hand, access,
+                          (DWORD)(offset >> 32),
+                          (DWORD)offset,
+                          length, addr);
+    if (!res) {
+        LogWindowsError("MapViewOfFileEx");
+        if (hFile != INVALID_HANDLE_VALUE) {
+            CloseHandle(hFile);
+        }
+        CloseHandle(hand);
+        errno = EPERM;
+        return MAP_FAILED;
+    }
+    if (flags & MAP_FIXED) {
+        // this assertion could legitimately fail if two threads engage in a
+        // race to create a MAP_FIXED mapping at the same address and that's
+        // certainly not the kind of use case we're designed to support here
+        assert(res == addr);
+    }
+    // record our new mapping in the global array
+    Lock();
+    for (i = 0; i < g_winmaps.n; ++i) {
+        if (!g_winmaps.p[i].hand) {
+            // we found an empty slot
+            break;
+        }
+    }
+    if (i == g_winmaps.n) {
+        // we need to grow the array
+        // it's important to use kernel32 memory
+        // our malloc implementation depends on this
+        int n2;
+        struct WinMap *p2;
+        p2 = g_winmaps.p;
+        n2 = g_winmaps.n;
+        if (n2) {
+            n2 += n2 >> 1;
+        } else {
+            n2 = 7;
+        }
+        if ((p2 = (struct WinMap*)Recalloc(p2, n2 * sizeof(*p2)))) {
+            g_winmaps.p = p2;
+            g_winmaps.n = n2;
+        } else {
+            Unlock();
+            LogError("recalloc failed");
+            UnmapViewOfFile(res);
+            CloseHandle(hand);
+            if (hFile != INVALID_HANDLE_VALUE) {
+                CloseHandle(hFile);
+            }
+            errno = ENOMEM;
+            return MAP_FAILED;
+        }
+    }
+    g_winmaps.p[i].hand = hand;
+    g_winmaps.p[i].fand = hFile;
+    g_winmaps.p[i].addr = (uintptr_t)res;
+    g_winmaps.p[i].length = length;
+    Unlock();
+    return res;
+}
+
+int WinMsync(void *addr, uintptr_t length, int flags) {
+    int i, err;
+    HANDLE hand;
+    uintptr_t x, y;
+    if (flags & ~(MS_ASYNC | MS_INVALIDATE | MS_SYNC)) {
+        LogError("bad msync flags");
+        errno = EINVAL;
+        return -1;
+    }
+    // 1. we do nothing if length is zero (unlike win32 api)
+    // 2. the requested interval may envelop multiple known mappings
+    // 3. we don't care if those mappings aren't contiguous or a hole exists
+    // 4. the requested interval may specify a subrange of any given mapping
+    Lock();
+    for (err = i = 0; i < g_winmaps.n; ++i) {
+        if (!g_winmaps.p[i].hand) {
+            // this array slot is empty
+            continue;
+        }
+        // compute overlap between known mapping and requested interval
+        x = Max((uintptr_t)addr, g_winmaps.p[i].addr);
+        y = Min((uintptr_t)addr + length, g_winmaps.p[i].addr + g_winmaps.p[i].length);
+        if (x >= y) {
+            // there isn't any overlap
+            continue;
+        }
+        // it's safe to release lock temporarily, since g_winmaps.n is monotonic
+        // any race conditions in handle being deleted should be caught by win32
+        hand = g_winmaps.p[i].fand;
+        Unlock();
+        // ensure coherency and that filesystem flush *will* happen
+        if (!FlushViewOfFile((void*)x, y - x)) {
+            LogWindowsError("FlushViewOfFile");
+            err = EPERM;
+        }
+        if (flags & MS_SYNC) {
+            // ensure that filesystem flush *has* happened
+            if (!FlushFileBuffers(hand)) {
+                LogWindowsError("FlushFileBuffers");
+                err = EPERM;
+            }
+        }
+        Lock();
+    }
+    Unlock();
+    if (err) {
+        errno = err;
+        return -1;
+    }
+    return 0;
+}
+
+#else // NEED_*_MAP
+
+// this is a normal unix platform
+// add some content to this object so the apple linker doesn't whine
+int justine_mmap_module;
+
+#endif // NEED_*_MMAP

mmap.h

@@ -0,0 +1,150 @@
+#pragma once
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
+#include <fcntl.h>
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if defined (_MSC_VER) && !(defined (_POSIX_MAPPED_FILES))
+#define NEED_WIN32_MMAP
+#include <Windows.h>
+#include <io.h>
+
+#ifndef PROT_READ
+#define PROT_READ 1
+#endif
+#ifndef PROT_WRITE
+#define PROT_WRITE 2
+#endif
+#ifndef PROT_EXEC
+#define PROT_EXEC 4
+#endif
+
+#ifndef MAP_SHARED
+#define MAP_SHARED 1
+#endif
+#ifndef MAP_PRIVATE
+#define MAP_PRIVATE 2
+#endif
+#ifndef MAP_FIXED
+#define MAP_FIXED 16
+#endif
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS 32
+#endif
+#ifndef MAP_FAILED
+#define MAP_FAILED ((void*)-1)
+#endif
+
+#ifndef O_RDONLY
+#define O_RDONLY _O_RDWR  // intentional smudge for mmap()
+#endif
+#ifndef O_WRONLY
+#define O_WRONLY _O_WRONLY
+#endif
+#ifndef O_RDWR
+#define O_RDWR _O_RDWR
+#endif
+#ifndef O_CREAT
+#define O_CREAT _O_CREAT
+#endif
+#ifndef O_TRUNC
+#define O_TRUNC _O_TRUNC
+#endif
+#ifndef O_EXCL
+#define O_EXCL _O_EXCL
+#endif
+
+#ifndef MADV_NORMAL
+#define MADV_NORMAL 0
+#endif
+#ifndef MADV_DONTNEED
+#define MADV_DONTNEED 4
+#endif
+#ifndef MADV_RANDOM
+#define MADV_RANDOM 1
+#endif
+#ifndef MADV_SEQUENTIAL
+#define MADV_SEQUENTIAL 2
+#endif
+#ifndef MADV_WILLNEED
+#define MADV_WILLNEED 3
+#endif
+
+#ifndef MS_ASYNC
+#define MS_ASYNC 1
+#endif
+#ifndef MS_INVALIDATE
+#define MS_INVALIDATE 2
+#endif
+#ifndef MS_SYNC
+#define MS_SYNC 4
+#endif
+
+#ifndef SEEK_SET
+#define SEEK_SET 0
+#endif
+#ifndef SEEK_CUR
+#define SEEK_CUR 1
+#endif
+#ifndef SEEK_END
+#define SEEK_END 2
+#endif
+
+#ifndef mmap
+#define mmap WinMap
+#endif
+#ifndef munmap
+#define munmap WinUnmap
+#endif
+#ifndef open
+#define open _open
+#endif
+#ifndef close
+#define close _close
+#endif
+#ifndef lseek
+#define lseek WinSeek
+#endif
+#ifndef msync
+#define msync WinMsync
+#endif
+#ifndef madvise
+#define madvise WinMadvise
+#endif
+#ifndef ftruncate
+#define ftruncate WinFtruncate
+#endif
+
+uint64_t WinSeek(int, uint64_t, int);
+int WinMsync(void *, uintptr_t, int);
+int WinMadvise(void *, uintptr_t, int);
+int WinFtruncate(int, uint64_t);
+int WinUnmap(void *, uintptr_t);
+void *WinMap(void *, uintptr_t, int, int, int, uint64_t);
+
+#else // _MSC_VER
+
+#include <unistd.h>
+#include <sys/mman.h>
+
+#ifndef MAP_ANONYMOUS
+#define NEED_POSIX_MMAP
+#define mmap PosixMmap
+#define MAP_ANONYMOUS 0x10000000
+void *PosixMmap(void*, size_t, int, int, int, off_t);
+#endif // MAP_ANONYMOUS
+
+#endif // _MSC_VER
+
+#ifdef __cplusplus
+}
+#endif

quantize.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+if ! [[ "$1" =~ ^[0-9]{1,2}B$ ]]; then
+    echo
+    echo "Usage: quantize.sh 7B|13B|30B|65B [--remove-f16]"
+    echo
+    exit 1
+fi
+
+for i in `ls models/$1/ggml-model-f16.bin*`; do
+    ./quantize "$i" "${i/f16/q4_0}" 2
+    if [[ "$2" == "--remove-f16" ]]; then
+        rm "$i"
+    fi
+done

utils.cpp

@@ -11,7 +11,7 @@
 
  #if defined(_MSC_VER) || defined(__MINGW32__)
  #include <malloc.h> // using malloc.h with MSC/MINGW
- #elif !defined(__FreeBSD__)
+ #elif !defined(__FreeBSD__) && !defined(__NetBSD__)
  #include <alloca.h>
  #endif