Merge pull request #1680 from dhiltgen/better_patching

Refactor how we augment llama.cpp and refine windows native build

gpu/gpu.go

@@ -13,6 +13,7 @@ import "C"
 import (
 	"fmt"
 	"log"
+	"runtime"
 	"sync"
 	"unsafe"
 
@@ -65,15 +66,14 @@ func GetGPUInfo() GpuInfo {
 	}
 
 	var memInfo C.mem_info_t
-	resp := GpuInfo{"", "", 0, 0}
+	resp := GpuInfo{"", 0, 0}
 	if gpuHandles.cuda != nil {
 		C.cuda_check_vram(*gpuHandles.cuda, &memInfo)
 		if memInfo.err != nil {
 			log.Printf("error looking up CUDA GPU memory: %s", C.GoString(memInfo.err))
 			C.free(unsafe.Pointer(memInfo.err))
 		} else {
-			resp.Driver = "CUDA"
+			resp.Library = "cuda"
-			resp.Library = "cuda_server"
 		}
 	} else if gpuHandles.rocm != nil {
 		C.rocm_check_vram(*gpuHandles.rocm, &memInfo)
@@ -81,15 +81,17 @@ func GetGPUInfo() GpuInfo {
 			log.Printf("error looking up ROCm GPU memory: %s", C.GoString(memInfo.err))
 			C.free(unsafe.Pointer(memInfo.err))
 		} else {
-			resp.Driver = "ROCM"
+			resp.Library = "rocm"
-			resp.Library = "rocm_server"
 		}
 	}
-	if resp.Driver == "" {
+	if resp.Library == "" {
 		C.cpu_check_ram(&memInfo)
-		resp.Driver = "CPU"
 		// In the future we may offer multiple CPU variants to tune CPU features
-		resp.Library = "default"
+		if runtime.GOOS == "windows" {
+			resp.Library = "cpu"
+		} else {
+			resp.Library = "default"
+		}
 	}
 	if memInfo.err != nil {
 		log.Printf("error looking up CPU memory: %s", C.GoString(memInfo.err))
@@ -103,7 +105,7 @@ func GetGPUInfo() GpuInfo {
 
 func CheckVRAM() (int64, error) {
 	gpuInfo := GetGPUInfo()
-	if gpuInfo.FreeMemory > 0 && gpuInfo.Driver != "CPU" {
+	if gpuInfo.FreeMemory > 0 && (gpuInfo.Library == "cuda" || gpuInfo.Library == "rocm") {
 		return int64(gpuInfo.FreeMemory), nil
 	}
 	return 0, fmt.Errorf("no GPU detected") // TODO - better handling of CPU based memory determiniation
@@ -114,7 +116,7 @@ func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int {
 		return opts.NumGPU
 	}
 	info := GetGPUInfo()
-	if info.Driver == "CPU" {
+	if info.Library == "cpu" || info.Library == "default" {
 		return 0
 	}
 
@@ -128,7 +130,7 @@ func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int {
 	// 75% of the absolute max number of layers we can fit in available VRAM, off-loading too many layers to the GPU can cause OOM errors
 	layers := int(info.FreeMemory/bytesPerLayer) * 3 / 4
 
-	log.Printf("%d MB VRAM available, loading up to %d %s GPU layers out of %d", info.FreeMemory/(1024*1024), layers, info.Driver, numLayer)
+	log.Printf("%d MB VRAM available, loading up to %d %s GPU layers out of %d", info.FreeMemory/(1024*1024), layers, info.Library, numLayer)
 
 	return layers
 }

gpu/gpu_darwin.go

@@ -20,7 +20,6 @@ func GetGPUInfo() GpuInfo {
 	// TODO - Metal vs. x86 macs...
 
 	return GpuInfo{
-		Driver:      "METAL",
 		Library:     "default",
 		TotalMemory: 0,
 		FreeMemory:  0,

gpu/gpu_test.go

@@ -9,7 +9,7 @@ import (
 
 func TestBasicGetGPUInfo(t *testing.T) {
 	info := GetGPUInfo()
-	assert.Contains(t, "CUDA ROCM CPU METAL", info.Driver)
+	assert.Contains(t, "cuda rocm cpu default", info.Library)
 
 	switch runtime.GOOS {
 	case "darwin":

gpu/types.go

@@ -2,7 +2,6 @@ package gpu
 
 // Beginning of an `ollama info` command
 type GpuInfo struct {
-	Driver      string `json:"driver,omitempty"`
 	Library     string `json:"library,omitempty"`
 	TotalMemory uint64 `json:"total_memory,omitempty"`
 	FreeMemory  uint64 `json:"free_memory,omitempty"`

llm/dynamic_shim.c

@@ -7,24 +7,29 @@
 #include <dlfcn.h>
 #define LOAD_LIBRARY(lib, flags) dlopen(lib, flags | RTLD_DEEPBIND)
 #define LOAD_SYMBOL(handle, sym) dlsym(handle, sym)
-#define LOAD_ERR() dlerror()
+#define LOAD_ERR() strdup(dlerror())
 #define UNLOAD_LIBRARY(handle) dlclose(handle)
 #elif _WIN32
 #include <windows.h>
 #define LOAD_LIBRARY(lib, flags) LoadLibrary(lib)
 #define LOAD_SYMBOL(handle, sym) GetProcAddress(handle, sym)
 #define UNLOAD_LIBRARY(handle) FreeLibrary(handle)
-// TODO - refactor this with proper error message handling on windows
+inline char *LOAD_ERR() {
-inline static char *LOAD_ERR() {
+  LPSTR messageBuffer = NULL;
-  static char errbuf[8];
+  size_t size = FormatMessageA(
-  snprintf(errbuf, 8, "0x%lx", GetLastError());
+      FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
-  return errbuf;
+          FORMAT_MESSAGE_IGNORE_INSERTS,
+      NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+      (LPSTR)&messageBuffer, 0, NULL);
+  char *resp = strdup(messageBuffer);
+  LocalFree(messageBuffer);
+  return resp;
 }
 #else
 #include <dlfcn.h>
 #define LOAD_LIBRARY(lib, flags) dlopen(lib, flags)
 #define LOAD_SYMBOL(handle, sym) dlsym(handle, sym)
-#define LOAD_ERR() dlerror()
+#define LOAD_ERR() strdup(dlerror())
 #define UNLOAD_LIBRARY(handle) dlclose(handle)
 #endif
 
@@ -57,8 +62,10 @@ void dynamic_shim_init(const char *libPath, struct dynamic_llama_server *s,
   s->handle = LOAD_LIBRARY(libPath, RTLD_NOW);
   if (!s->handle) {
     err->id = -1;
+    char *msg = LOAD_ERR();
     snprintf(err->msg, err->msg_len,
-             "Unable to load dynamic server library: %s", LOAD_ERR());
+             "Unable to load dynamic server library: %s", msg);
+    free(msg);
     return;
   }
 
@@ -67,8 +74,10 @@ void dynamic_shim_init(const char *libPath, struct dynamic_llama_server *s,
     if (!l[i].p) {
       UNLOAD_LIBRARY(s->handle);
       err->id = -1;
+      char *msg = LOAD_ERR();
       snprintf(err->msg, err->msg_len, "symbol lookup for %s failed: %s",
-               l[i].s, LOAD_ERR());
+               l[i].s, msg);
+      free(msg);
       return;
     }
   }

llm/dynamic_shim.h

@@ -1,6 +1,6 @@
 #include <stdlib.h>
 
-#include "server.h"
+#include "ext_server.h"
 
 #ifdef __cplusplus
 extern "C" {

llm/ext_server_common.go

@@ -1,7 +1,7 @@
 package llm
 
 /*
-#cgo CFLAGS: -I${SRCDIR}/llama.cpp/gguf -I${SRCDIR}/llama.cpp/gguf/common -I${SRCDIR}/llama.cpp/gguf/examples/server
+#cgo CFLAGS: -I${SRCDIR}/llama.cpp -I${SRCDIR}/llama.cpp/gguf -I${SRCDIR}/llama.cpp/gguf/common -I${SRCDIR}/llama.cpp/gguf/examples/server
 #cgo CFLAGS: -DNDEBUG -DLLAMA_SERVER_LIBRARY=1 -D_XOPEN_SOURCE=600 -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64
 #cgo CFLAGS: -Wmissing-noreturn -Wall -Wextra -Wcast-qual -Wno-unused-function -Wno-array-bounds
 #cgo CPPFLAGS: -Ofast -Wall -Wextra -Wno-unused-function -Wno-unused-variable -Wno-deprecated-declarations -Wno-unused-but-set-variable
@@ -10,23 +10,22 @@ package llm
 #cgo darwin CPPFLAGS: -DGGML_USE_METAL -DGGML_METAL_NDEBUG
 #cgo darwin LDFLAGS: -lc++ -framework Accelerate
 #cgo darwin LDFLAGS: -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders
-#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/common/libcommon.a
+#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/darwin/metal/lib/libcommon.a
-#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/examples/server/libext_server.a
+#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/darwin/metal/lib/libext_server.a
-#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/libllama.a
+#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/darwin/metal/lib/libllama.a
-#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/libggml_static.a
+#cgo darwin LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/darwin/metal/lib/libggml_static.a
 #cgo linux CFLAGS: -D_GNU_SOURCE
 #cgo linux windows CFLAGS: -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_MMV_Y=1 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_USE_CUBLAS
 #cgo linux LDFLAGS: -L/usr/local/cuda/targets/x86_64-linux/lib -L/usr/local/cuda/lib64 -L/usr/local/cuda/targets/x86_64-linux/lib/stubs
-#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/examples/server/libext_server.a
+#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/linux/cpu/lib/libext_server.a
-#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/common/libcommon.a
+#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/linux/cpu/lib/libcommon.a
-#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/libllama.a
+#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/linux/cpu/lib/libllama.a
-#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/libggml_static.a
+#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/linux/cpu/lib/libggml_static.a
-#cgo linux LDFLAGS: -lrt -lpthread -ldl -lstdc++ -lm
+#cgo linux LDFLAGS: -lrt -ldl -lstdc++ -lm
-#cgo windows LDFLAGS: -L${SRCDIR}/llama.cpp/gguf/build/wincpu/dist/lib
+#cgo linux windows LDFLAGS: -lpthread
-#cgo windows LDFLAGS: -lcpu_server -lpthread
 
 #include <stdlib.h>
-#include "server.h"
+#include "ext_server.h"
 
 */
 import "C"
@@ -46,6 +45,24 @@ import (
 	"github.com/jmorganca/ollama/gpu"
 )
 
+type extServer interface {
+	LLM
+	llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t)
+	llama_server_start()
+	llama_server_stop()
+	llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t)
+	llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t)
+	llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t)
+	llama_server_release_task_result(result *C.ext_server_task_result_t)
+	llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t)
+	llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t)
+	llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t)
+	llama_server_release_json_resp(json_resp **C.char)
+}
+
+// Note: current implementation does not support concurrent instantiations
+var mutex sync.Mutex
+
 func newExtServerResp(len C.size_t) C.ext_server_resp_t {
 	var resp C.ext_server_resp_t
 	resp.msg_len = len
@@ -65,69 +82,6 @@ func extServerResponseToErr(resp C.ext_server_resp_t) error {
 	return fmt.Errorf(C.GoString(resp.msg))
 }
 
-type extServer interface {
-	LLM
-	llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t)
-	llama_server_start()
-	llama_server_stop()
-	llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t)
-	llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t)
-	llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t)
-	llama_server_release_task_result(result *C.ext_server_task_result_t)
-	llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t)
-	llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t)
-	llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t)
-	llama_server_release_json_resp(json_resp **C.char)
-}
-
-type llamaExtServer struct {
-	api.Options
-}
-
-// Note: current implementation does not support concurrent instantiations
-var mutex sync.Mutex
-
-func (llm *llamaExtServer) llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) {
-	C.llama_server_init(sparams, err)
-}
-func (llm *llamaExtServer) llama_server_start() {
-	C.llama_server_start()
-}
-func (llm *llamaExtServer) llama_server_stop() {
-	C.llama_server_stop()
-}
-
-func (llm *llamaExtServer) llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) {
-	C.llama_server_completion(json_req, resp)
-}
-func (llm *llamaExtServer) llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) {
-	C.llama_server_completion_next_result(task_id, resp)
-}
-func (llm *llamaExtServer) llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) {
-	C.llama_server_completion_cancel(task_id, err)
-}
-func (llm *llamaExtServer) llama_server_release_task_result(result *C.ext_server_task_result_t) {
-	C.llama_server_release_task_result(result)
-}
-
-func (llm *llamaExtServer) llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
-	C.llama_server_tokenize(json_req, json_resp, err)
-}
-func (llm *llamaExtServer) llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
-	C.llama_server_detokenize(json_req, json_resp, err)
-}
-func (llm *llamaExtServer) llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
-	C.llama_server_embedding(json_req, json_resp, err)
-}
-func (llm *llamaExtServer) llama_server_release_json_resp(json_resp **C.char) {
-	C.llama_server_release_json_resp(json_resp)
-}
-
-func newDefaultExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
-	server := &llamaExtServer{opts}
-	return newExtServer(server, model, adapters, projectors, numLayers, opts)
-}
-
 func newExtServer(server extServer, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
 	if !mutex.TryLock() {
 		log.Printf("concurrent llm servers not yet supported, waiting for prior server to complete")
@@ -199,10 +153,6 @@ func newExtServer(server extServer, model string, adapters, projectors []string,
 	return server, nil
 }
 
-func (llm *llamaExtServer) Predict(ctx context.Context, pred PredictOpts, fn func(PredictResult)) error {
-	return predict(llm, llm.Options, ctx, pred, fn)
-}
-
 func predict(llm extServer, opts api.Options, ctx context.Context, predict PredictOpts, fn func(PredictResult)) error {
 	resp := newExtServerResp(128)
 	defer freeExtServerResp(resp)
@@ -326,9 +276,6 @@ func predict(llm extServer, opts api.Options, ctx context.Context, predict Predi
 	// should never reach here ideally
 	return fmt.Errorf("max retries exceeded")
 }
-func (llm *llamaExtServer) Encode(ctx context.Context, prompt string) ([]int, error) {
-	return encode(llm, ctx, prompt)
-}
 
 func encode(llm extServer, ctx context.Context, prompt string) ([]int, error) {
 	data, err := json.Marshal(TokenizeRequest{Content: prompt})
@@ -354,10 +301,6 @@ func encode(llm extServer, ctx context.Context, prompt string) ([]int, error) {
 	return encoded.Tokens, err
 }
 
-func (llm *llamaExtServer) Decode(ctx context.Context, tokens []int) (string, error) {
-	return decode(llm, ctx, tokens)
-}
-
 func decode(llm extServer, ctx context.Context, tokens []int) (string, error) {
 	if len(tokens) == 0 {
 		return "", nil
@@ -386,9 +329,6 @@ func decode(llm extServer, ctx context.Context, tokens []int) (string, error) {
 	return decoded.Content, err
 }
 
-func (llm *llamaExtServer) Embedding(ctx context.Context, input string) ([]float64, error) {
-	return embedding(llm, ctx, input)
-}
 func embedding(llm extServer, ctx context.Context, input string) ([]float64, error) {
 	data, err := json.Marshal(TokenizeRequest{Content: input})
 	if err != nil {
@@ -414,10 +354,6 @@ func embedding(llm extServer, ctx context.Context, input string) ([]float64, err
 	return embedding.Embedding, nil
 }
 
-func (llm *llamaExtServer) Close() {
-	close(llm)
-}
-
 func close(llm extServer) {
 	llm.llama_server_stop()
 	mutex.Unlock()

llm/ext_server_default.go

@@ -0,0 +1,80 @@
+//go:build !windows
+
+package llm
+
+/*
+#include <stdlib.h>
+#include "ext_server.h"
+
+*/
+import "C"
+import (
+	"context"
+
+	"github.com/jmorganca/ollama/api"
+)
+
+type llamaExtServer struct {
+	api.Options
+}
+
+func (llm *llamaExtServer) llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) {
+	C.llama_server_init(sparams, err)
+}
+func (llm *llamaExtServer) llama_server_start() {
+	C.llama_server_start()
+}
+func (llm *llamaExtServer) llama_server_stop() {
+	C.llama_server_stop()
+}
+
+func (llm *llamaExtServer) llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) {
+	C.llama_server_completion(json_req, resp)
+}
+func (llm *llamaExtServer) llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) {
+	C.llama_server_completion_next_result(task_id, resp)
+}
+func (llm *llamaExtServer) llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) {
+	C.llama_server_completion_cancel(task_id, err)
+}
+func (llm *llamaExtServer) llama_server_release_task_result(result *C.ext_server_task_result_t) {
+	C.llama_server_release_task_result(result)
+}
+
+func (llm *llamaExtServer) llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.llama_server_tokenize(json_req, json_resp, err)
+}
+func (llm *llamaExtServer) llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.llama_server_detokenize(json_req, json_resp, err)
+}
+func (llm *llamaExtServer) llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.llama_server_embedding(json_req, json_resp, err)
+}
+func (llm *llamaExtServer) llama_server_release_json_resp(json_resp **C.char) {
+	C.llama_server_release_json_resp(json_resp)
+}
+
+func newDefaultExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
+	server := &llamaExtServer{opts}
+	return newExtServer(server, model, adapters, projectors, numLayers, opts)
+}
+
+func (llm *llamaExtServer) Predict(ctx context.Context, pred PredictOpts, fn func(PredictResult)) error {
+	return predict(llm, llm.Options, ctx, pred, fn)
+}
+
+func (llm *llamaExtServer) Encode(ctx context.Context, prompt string) ([]int, error) {
+	return encode(llm, ctx, prompt)
+}
+
+func (llm *llamaExtServer) Decode(ctx context.Context, tokens []int) (string, error) {
+	return decode(llm, ctx, tokens)
+}
+
+func (llm *llamaExtServer) Embedding(ctx context.Context, input string) ([]float64, error) {
+	return embedding(llm, ctx, input)
+}
+
+func (llm *llamaExtServer) Close() {
+	close(llm)
+}

llm/ext_server_windows.go

@@ -0,0 +1,15 @@
+package llm
+
+import (
+	"fmt"
+
+	"github.com/jmorganca/ollama/api"
+)
+
+func newDefaultExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
+	// On windows we always load the llama.cpp libraries dynamically to avoid startup DLL dependencies
+	// This ensures we can update the PATH at runtime to get everything loaded
+
+	// Should not happen
+	return nil, fmt.Errorf("no default impl on windows - all dynamic")
+}

llm/llama.cpp/CMakeLists.txt

@@ -0,0 +1,29 @@
+# Ollama specific CMakefile to include in llama.cpp/examples/server
+
+set(TARGET ext_server)
+option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON)
+add_library(${TARGET} STATIC ../../../ext_server.cpp)
+target_include_directories(${TARGET} PRIVATE ../../common)
+target_include_directories(${TARGET} PRIVATE ../..)
+target_include_directories(${TARGET} PRIVATE ../../..)
+target_compile_features(${TARGET} PRIVATE cxx_std_11)
+target_compile_definitions(${TARGET} PUBLIC LLAMA_SERVER_LIBRARY=1)
+target_link_libraries(${TARGET} PRIVATE common llama llava ${CMAKE_THREAD_LIBS_INIT})
+target_compile_definitions(${TARGET} PRIVATE
+    SERVER_VERBOSE=$<BOOL:${LLAMA_SERVER_VERBOSE}>
+)
+
+if (BUILD_SHARED_LIBS)
+    set_target_properties(ext_server PROPERTIES POSITION_INDEPENDENT_CODE ON)
+    target_compile_definitions(ext_server PRIVATE LLAMA_SHARED LLAMA_BUILD)
+    add_library(ext_server_shared SHARED $<TARGET_OBJECTS:ext_server>)
+    target_link_libraries(ext_server_shared PRIVATE ggml llama llava common ${CMAKE_THREAD_LIBS_INIT})
+    install(TARGETS ext_server_shared LIBRARY)
+endif()
+
+if (CUDAToolkit_FOUND)
+    target_include_directories(${TARGET} PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+    if (WIN32)
+        target_link_libraries(ext_server_shared PRIVATE nvml)
+    endif()
+endif()

llm/llama.cpp/ext_server.cpp

@@ -0,0 +1,281 @@
+#include "ext_server.h"
+
+// Necessary evil since the server types are not defined in a header
+#include "server.cpp"
+
+// Expose the llama server as a callable extern "C" API
+llama_server_context *llama = NULL;
+std::atomic<bool> ext_server_running(false);
+std::thread ext_server_thread;
+
+void llama_server_init(ext_server_params *sparams, ext_server_resp_t *err) {
+#if SERVER_VERBOSE != 1
+  log_disable();
+#endif
+  assert(err != NULL && sparams != NULL);
+  err->id = 0;
+  err->msg[0] = '\0';
+  try {
+    llama = new llama_server_context;
+    log_set_target(stdout);
+    gpt_params params;
+    params.n_ctx = sparams->n_ctx;
+    params.n_batch = sparams->n_batch;
+    if (sparams->n_threads > 0) {
+      params.n_threads = sparams->n_threads;
+    }
+    params.n_parallel = sparams->n_parallel;
+    params.rope_freq_base = sparams->rope_freq_base;
+    params.rope_freq_scale = sparams->rope_freq_scale;
+
+    if (sparams->memory_f16) {
+      params.cache_type_k = "f16";
+      params.cache_type_v = "f16";
+    } else {
+      params.cache_type_k = "f32";
+      params.cache_type_v = "f32";
+    }
+
+    params.n_gpu_layers = sparams->n_gpu_layers;
+    params.main_gpu = sparams->main_gpu;
+    params.use_mlock = sparams->use_mlock;
+    params.use_mmap = sparams->use_mmap;
+    params.numa = sparams->numa;
+    params.embedding = sparams->embedding;
+    if (sparams->model != NULL) {
+      params.model = sparams->model;
+    }
+
+    for (ext_server_lora_adapter *la = sparams->lora_adapters; la != NULL;
+         la = la->next) {
+      params.lora_adapter.push_back(std::make_tuple(la->adapter, la->scale));
+    }
+
+    if (sparams->mmproj != NULL) {
+      params.mmproj = std::string(sparams->mmproj);
+    }
+
+    llama_backend_init(params.numa);
+
+    // load the model
+    if (!llama->load_model(params)) {
+      // TODO - consider modifying the logging logic or patching load_model so
+      // we can capture more detailed error messages and pass them back to the
+      // caller for better UX
+      err->id = -1;
+      snprintf(err->msg, err->msg_len, "error loading model %s",
+               params.model.c_str());
+      return;
+    }
+
+    llama->initialize();
+  } catch (std::exception &e) {
+    err->id = -1;
+    snprintf(err->msg, err->msg_len, "exception %s", e.what());
+  } catch (...) {
+    err->id = -1;
+    snprintf(err->msg, err->msg_len,
+             "Unknown exception initializing llama server");
+  }
+}
+
+void llama_server_start() {
+  assert(llama != NULL);
+  // TODO mutex to protect thread creation
+  ext_server_thread = std::thread([&]() {
+    ext_server_running = true;
+    try {
+      LOG_TEE("llama server main loop starting\n");
+      ggml_time_init();
+      while (ext_server_running.load()) {
+        if (!llama->update_slots()) {
+          LOG_TEE(
+              "unexpected error in llama server update_slots - exiting main "
+              "loop\n");
+          break;
+        }
+      }
+    } catch (std::exception &e) {
+      LOG_TEE("caught exception in llama server main loop: %s\n", e.what());
+    } catch (...) {
+      LOG_TEE("caught unknown exception in llama server main loop\n");
+    }
+    LOG_TEE("\nllama server shutting down\n");
+    llama_backend_free();
+  });
+}
+
+void llama_server_stop() {
+  assert(llama != NULL);
+  // TODO - too verbose, remove once things are solid
+  LOG_TEE("requesting llama server shutdown\n");
+  ext_server_running = false;
+  ext_server_thread.join();
+  delete llama;
+  llama = NULL;
+  LOG_TEE("llama server shutdown complete\n");
+}
+
+void llama_server_completion(const char *json_req, ext_server_resp_t *resp) {
+  assert(llama != NULL && json_req != NULL && resp != NULL);
+  resp->id = -1;
+  resp->msg[0] = '\0';
+  try {
+    json data = json::parse(json_req);
+    resp->id = llama->request_completion(data, false, false, -1);
+  } catch (std::exception &e) {
+    snprintf(resp->msg, resp->msg_len, "exception %s", e.what());
+  } catch (...) {
+    snprintf(resp->msg, resp->msg_len, "Unknown exception during completion");
+  }
+}
+
+void llama_server_completion_next_result(const int task_id,
+                                         ext_server_task_result_t *resp) {
+  assert(llama != NULL && resp != NULL);
+  std::string msg;
+  resp->id = -1;
+  resp->stop = false;
+  resp->error = false;
+  resp->json_resp = NULL;
+  std::string result_json;
+  try {
+    task_result result = llama->next_result(task_id);
+    result_json =
+        result.result_json.dump(-1, ' ', false, json::error_handler_t::replace);
+    resp->id = result.id;
+    resp->stop = result.stop;
+    resp->error = result.error;
+    if (result.error) {
+      llama->request_cancel(task_id);
+    } else if (result.stop) {
+      llama->request_cancel(task_id);
+    }
+  } catch (std::exception &e) {
+    resp->error = true;
+    resp->id = -1;
+    result_json = "{\"error\":\"exception " + std::string(e.what()) + "\"}";
+    LOG_TEE("llama server completion exception %s\n", e.what());
+  } catch (...) {
+    resp->error = true;
+    resp->id = -1;
+    result_json = "{\"error\":\"Unknown exception during completion\"}";
+    LOG_TEE("llama server completion unknown exception\n");
+  }
+  const std::string::size_type size = result_json.size() + 1;
+  resp->json_resp = new char[size];
+  snprintf(resp->json_resp, size, "%s", result_json.c_str());
+}
+
+void llama_server_release_task_result(ext_server_task_result_t *result) {
+  if (result == NULL || result->json_resp == NULL) {
+    return;
+  }
+  delete[] result->json_resp;
+}
+
+void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err) {
+  assert(llama != NULL && err != NULL);
+  err->id = 0;
+  err->msg[0] = '\0';
+  try {
+    llama->request_cancel(task_id);
+  } catch (std::exception &e) {
+    err->id = -1;
+    snprintf(err->msg, err->msg_len, "exception %s", e.what());
+  } catch (...) {
+    err->id = -1;
+    snprintf(err->msg, err->msg_len,
+             "Unknown exception completion cancel in llama server");
+  }
+}
+
+void llama_server_tokenize(const char *json_req, char **json_resp,
+                           ext_server_resp_t *err) {
+  assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
+  *json_resp = NULL;
+  err->id = 0;
+  err->msg[0] = '\0';
+  try {
+    const json body = json::parse(json_req);
+    std::vector<llama_token> tokens;
+    if (body.count("content") != 0) {
+      tokens = llama->tokenize(body["content"], false);
+    }
+    const json data = format_tokenizer_response(tokens);
+    std::string result_json = data.dump();
+    const std::string::size_type size = result_json.size() + 1;
+    *json_resp = new char[size];
+    snprintf(*json_resp, size, "%s", result_json.c_str());
+  } catch (std::exception &e) {
+    err->id = -1;
+    snprintf(err->msg, err->msg_len, "exception %s", e.what());
+  } catch (...) {
+    err->id = -1;
+    snprintf(err->msg, err->msg_len, "Unknown exception during tokenize");
+  }
+}
+
+void llama_server_release_json_resp(char **json_resp) {
+  if (json_resp == NULL || *json_resp == NULL) {
+    return;
+  }
+  delete[] *json_resp;
+}
+
+void llama_server_detokenize(const char *json_req, char **json_resp,
+                             ext_server_resp_t *err) {
+  assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
+  *json_resp = NULL;
+  err->id = 0;
+  err->msg[0] = '\0';
+  try {
+    const json body = json::parse(json_req);
+    std::string content;
+    if (body.count("tokens") != 0) {
+      const std::vector<llama_token> tokens = body["tokens"];
+      content = tokens_to_str(llama->ctx, tokens.cbegin(), tokens.cend());
+    }
+    const json data = format_detokenized_response(content);
+    std::string result_json = data.dump();
+    const std::string::size_type size = result_json.size() + 1;
+    *json_resp = new char[size];
+    snprintf(*json_resp, size, "%s", result_json.c_str());
+  } catch (std::exception &e) {
+    err->id = -1;
+    snprintf(err->msg, err->msg_len, "exception %s", e.what());
+  } catch (...) {
+    err->id = -1;
+    snprintf(err->msg, err->msg_len, "Unknown exception during detokenize");
+  }
+}
+
+void llama_server_embedding(const char *json_req, char **json_resp,
+                            ext_server_resp_t *err) {
+  assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
+  *json_resp = NULL;
+  err->id = 0;
+  err->msg[0] = '\0';
+  try {
+    const json body = json::parse(json_req);
+    json prompt;
+    if (body.count("content") != 0) {
+      prompt = body["content"];
+    } else {
+      prompt = "";
+    }
+    const int task_id = llama->request_completion(
+        {{"prompt", prompt}, {"n_predict", 0}}, false, true, -1);
+    task_result result = llama->next_result(task_id);
+    std::string result_json = result.result_json.dump();
+    const std::string::size_type size = result_json.size() + 1;
+    *json_resp = new char[size];
+    snprintf(*json_resp, size, "%s", result_json.c_str());
+  } catch (std::exception &e) {
+    err->id = -1;
+    snprintf(err->msg, err->msg_len, "exception %s", e.what());
+  } catch (...) {
+    err->id = -1;
+    snprintf(err->msg, err->msg_len, "Unknown exception during embedding");
+  }
+}

llm/llama.cpp/ext_server.h

@@ -0,0 +1,94 @@
+#if defined(LLAMA_SERVER_LIBRARY)
+#ifndef LLAMA_SERVER_H
+#define LLAMA_SERVER_H
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+int __main(int argc, char **argv);
+
+// This exposes extern C entrypoints into the llama_server
+// To enable the server compile with LLAMA_SERVER_LIBRARY
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+typedef struct ext_server_resp {
+  int id;          // < 0 on error
+  size_t msg_len;  // caller must allocate msg and set msg_len
+  char *msg;
+} ext_server_resp_t;
+
+// Allocated and freed by caller
+typedef struct ext_server_lora_adapter {
+  char *adapter;
+  float scale;
+  struct ext_server_lora_adapter *next;
+} ext_server_lora_adapter_t;
+
+// Allocated and freed by caller
+typedef struct ext_server_params {
+  char *model;
+  uint32_t n_ctx;         // token context window, 0 = from model
+  uint32_t n_batch;       // prompt processing maximum batch size
+  uint32_t n_threads;     // number of threads to use for generation
+  int32_t n_parallel;     // number of parallel sequences to decodewra
+  float rope_freq_base;   // RoPE base frequency, 0 = from model
+  float rope_freq_scale;  // RoPE frequency scaling factor, 0 = from model
+  bool memory_f16;        // use f16 instead of f32 for memory kv
+  int32_t n_gpu_layers;  // number of layers to store in VRAM (-1 - use default)
+  int32_t main_gpu;      // the GPU that is used for scratch and small tensors
+  bool use_mlock;        // force system to keep model in RAM
+  bool use_mmap;         // use mmap if possible
+  bool numa;             // attempt optimizations that help on some NUMA systems
+  bool embedding;        // get only sentence embedding
+  ext_server_lora_adapter_t *lora_adapters;
+  char *mmproj;
+} ext_server_params_t;
+
+typedef struct ext_server_task_result {
+  int id;
+  bool stop;
+  bool error;
+  char *json_resp;  // null terminated, memory managed by ext_server
+} ext_server_task_result_t;
+
+// Initialize the server once per process
+// err->id = 0 for success and err->msg[0] = NULL
+// err->id != 0 for failure, and err->msg contains error message
+void llama_server_init(ext_server_params_t *sparams, ext_server_resp_t *err);
+
+// Run the main loop, called once per init
+void llama_server_start();
+// Stop the main loop and free up resources allocated in init and start.  Init
+// must be called again to reuse
+void llama_server_stop();
+
+// json_req null terminated string, memory managed by caller
+// resp->id >= 0 on success (task ID)
+// resp->id < 0 on error, and resp->msg contains error message
+void llama_server_completion(const char *json_req, ext_server_resp_t *resp);
+
+// Caller must call llama_server_release_task_result to free resp->json_resp
+void llama_server_completion_next_result(const int task_id,
+                                         ext_server_task_result_t *result);
+void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err);
+void llama_server_release_task_result(ext_server_task_result_t *result);
+
+// Caller must call llama_server_releaes_json_resp to free json_resp if err.id <
+// 0
+void llama_server_tokenize(const char *json_req, char **json_resp,
+                           ext_server_resp_t *err);
+void llama_server_detokenize(const char *json_req, char **json_resp,
+                             ext_server_resp_t *err);
+void llama_server_embedding(const char *json_req, char **json_resp,
+                            ext_server_resp_t *err);
+void llama_server_release_json_resp(char **json_resp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+#endif  // LLAMA_SERVER_LIBRARY

llm/llama.cpp/gen_common.sh

@@ -25,18 +25,30 @@ git_module_setup() {
 }
 
 apply_patches() {
-    if [ -n "${OLLAMA_SKIP_PATCHING}" ]; then
+    # Wire up our CMakefile
-        echo "Skipping submodule patching"
+    if ! grep ollama gguf/examples/server/CMakeLists.txt; then
-        return
+        echo 'include (../../../CMakeLists.txt) # ollama' >>gguf/examples/server/CMakeLists.txt
     fi
-    # Workaround git apply not handling creation well for iteration
+    # Avoid duplicate main symbols when we link into the cgo binary
-    rm -f gguf/examples/server/server.h
+    sed -e 's/int main(/int __main(/g' <./gguf/examples/server/server.cpp >./gguf/examples/server/server.cpp.tmp &&
-    for patch in ${PATCHES}; do
+        mv ./gguf/examples/server/server.cpp.tmp ./gguf/examples/server/server.cpp
-        git -C gguf apply ../patches/${patch}
-    done
 }
 
 build() {
     cmake -S ${LLAMACPP_DIR} -B ${BUILD_DIR} ${CMAKE_DEFS}
     cmake --build ${BUILD_DIR} ${CMAKE_TARGETS} -j8
 }
+
+install() {
+    rm -rf ${BUILD_DIR}/lib
+    mkdir -p ${BUILD_DIR}/lib
+    cp ${BUILD_DIR}/examples/server/libext_server.a ${BUILD_DIR}/lib
+    cp ${BUILD_DIR}/common/libcommon.a ${BUILD_DIR}/lib
+    cp ${BUILD_DIR}/libllama.a ${BUILD_DIR}/lib
+    cp ${BUILD_DIR}/libggml_static.a ${BUILD_DIR}/lib
+}
+
+# Keep the local tree clean after we're done with the build
+cleanup() {
+    (cd gguf/examples/server/ && git checkout CMakeLists.txt server.cpp)
+}

llm/llama.cpp/gen_darwin.sh

@@ -10,21 +10,23 @@ echo "Starting darwin generate script"
 source $(dirname $0)/gen_common.sh
 init_vars
 CMAKE_DEFS="-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 -DLLAMA_METAL=on ${CMAKE_DEFS}"
-BUILD_DIR="gguf/build/metal"
+BUILD_DIR="gguf/build/darwin/metal"
 case "${GOARCH}" in
-    "amd64")
+"amd64")
-        CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 ${CMAKE_DEFS}"
+    CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 ${CMAKE_DEFS}"
-        ;;
+    ;;
-     "arm64")
+"arm64")
-        CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_OSX_ARCHITECTURES=arm64 ${CMAKE_DEFS}"
+    CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_OSX_ARCHITECTURES=arm64 ${CMAKE_DEFS}"
-        ;;
+    ;;
-    *)
+*)
-        echo "GOARCH must be set"
+    echo "GOARCH must be set"
-        echo "this script is meant to be run from within go generate"
+    echo "this script is meant to be run from within go generate"
-        exit 1
+    exit 1
-        ;;
+    ;;
 esac
 
 git_module_setup
 apply_patches
 build
+install
+cleanup

llm/llama.cpp/gen_linux.sh

@@ -21,34 +21,33 @@ if [ -z "${CUDACXX}" -a -x /usr/local/cuda/bin/nvcc ]; then
     export CUDACXX=/usr/local/cuda/bin/nvcc
 fi
 COMMON_CMAKE_DEFS="-DCMAKE_POSITION_INDEPENDENT_CODE=on -DLLAMA_ACCELERATE=on -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off"
-OLLAMA_DYN_LIB_DIR="gguf/build/lib"
 source $(dirname $0)/gen_common.sh
 init_vars
 git_module_setup
 apply_patches
 
-mkdir -p ${OLLAMA_DYN_LIB_DIR}
-touch ${OLLAMA_DYN_LIB_DIR}/.generated
-
 #
 # CPU first for the default library
 #
 CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS}"
-BUILD_DIR="gguf/build/cpu"
+BUILD_DIR="gguf/build/linux/cpu"
+
 build
+install
 
 if [ -d /usr/local/cuda/lib64/ ]; then
     echo "CUDA libraries detected - building dynamic CUDA library"
     init_vars
     CMAKE_DEFS="-DLLAMA_CUBLAS=on ${COMMON_CMAKE_DEFS} ${CMAKE_DEFS}"
-    BUILD_DIR="gguf/build/cuda"
+    BUILD_DIR="gguf/build/linux/cuda"
     CUDA_LIB_DIR=/usr/local/cuda/lib64
     build
-    gcc -fPIC -g -shared -o ${OLLAMA_DYN_LIB_DIR}/libcuda_server.so \
+    install
+    gcc -fPIC -g -shared -o ${BUILD_DIR}/lib/libext_server.so \
         -Wl,--whole-archive \
-        ${BUILD_DIR}/examples/server/libext_server.a \
+        ${BUILD_DIR}/lib/libext_server.a \
-        ${BUILD_DIR}/common/libcommon.a \
+        ${BUILD_DIR}/lib/libcommon.a \
-        ${BUILD_DIR}/libllama.a \
+        ${BUILD_DIR}/lib/libllama.a \
         -Wl,--no-whole-archive \
         ${CUDA_LIB_DIR}/libcudart_static.a \
         ${CUDA_LIB_DIR}/libcublas_static.a \
@@ -74,16 +73,19 @@ if [ -d "${ROCM_PATH}" ]; then
     echo "ROCm libraries detected - building dynamic ROCm library"
     init_vars
     CMAKE_DEFS="${COMMON_CMAKE_DEFS} ${CMAKE_DEFS} -DLLAMA_HIPBLAS=on -DCMAKE_C_COMPILER=$ROCM_PATH/llvm/bin/clang -DCMAKE_CXX_COMPILER=$ROCM_PATH/llvm/bin/clang++ -DAMDGPU_TARGETS='gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102' -DGPU_TARGETS='gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102'"
-    BUILD_DIR="gguf/build/rocm"
+    BUILD_DIR="gguf/build/linux/rocm"
     build
-    gcc -fPIC -g -shared -o ${OLLAMA_DYN_LIB_DIR}/librocm_server.so \
+    install
+    gcc -fPIC -g -shared -o ${BUILD_DIR}/lib/libext_server.so \
         -Wl,--whole-archive \
-        ${BUILD_DIR}/examples/server/libext_server.a \
+        ${BUILD_DIR}/lib/libext_server.a \
-        ${BUILD_DIR}/common/libcommon.a \
+        ${BUILD_DIR}/lib/libcommon.a \
-        ${BUILD_DIR}/libllama.a \
+        ${BUILD_DIR}/lib/libllama.a \
         -Wl,--no-whole-archive \
         -lrt -lpthread -ldl -lstdc++ -lm \
         -L/opt/rocm/lib -L/opt/amdgpu/lib/x86_64-linux-gnu/ \
         -Wl,-rpath,/opt/rocm/lib,-rpath,/opt/amdgpu/lib/x86_64-linux-gnu/ \
         -lhipblas -lrocblas -lamdhip64 -lrocsolver -lamd_comgr -lhsa-runtime64 -lrocsparse -ldrm -ldrm_amdgpu
 fi
+
+cleanup

llm/llama.cpp/gen_windows.ps1

@@ -5,7 +5,7 @@ $ErrorActionPreference = "Stop"
 function init_vars {
     $script:patches = @("0001-Expose-callable-API-for-server.patch")
     $script:cmakeDefs = @("-DBUILD_SHARED_LIBS=on", "-DLLAMA_NATIVE=off", "-DLLAMA_F16C=off", "-DLLAMA_FMA=off", "-DLLAMA_AVX512=off", "-DLLAMA_AVX2=off", "-DLLAMA_AVX=on", "-DLLAMA_K_QUANTS=on", "-DLLAMA_ACCELERATE=on", "-A","x64")
-
+    $script:cmakeTargets = @("ggml", "ggml_static", "llama", "build_info", "common", "ext_server_shared", "llava_static")
     if ($env:CGO_CFLAGS -contains "-g") {
         $script:cmakeDefs += @("-DCMAKE_VERBOSE_MAKEFILE=on", "-DLLAMA_SERVER_VERBOSE=on")
         $script:config = "RelWithDebInfo"
@@ -24,12 +24,14 @@ function git_module_setup {
 }
 
 function apply_patches {
-    rm -erroraction ignore -path "gguf/examples/server/server.h"
+    # Wire up our CMakefile
-    foreach ($patch in $script:patches) {
+    if (!(Select-String -Path "gguf/examples/server/CMakeLists.txt" -Pattern 'ollama')) {
-        write-host "Applying patch $patch"
+        Add-Content -Path "gguf/examples/server/CMakeLists.txt" -Value 'include (../../../CMakeLists.txt) # ollama'
-        & git -C gguf apply ../patches/$patch
-        if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
     }
+    # Avoid duplicate main symbols when we link into the cgo binary
+    $content = Get-Content -Path "./gguf/examples/server/server.cpp"
+    $content = $content -replace 'int main\(', 'int __main('
+    Set-Content -Path "./gguf/examples/server/server.cpp" -Value $content
 }
 
 function build {
@@ -37,16 +39,21 @@ function build {
     & cmake --version
     & cmake -S gguf -B $script:buildDir $script:cmakeDefs
     if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-    write-host "building with: cmake --build $script:buildDir --config $script:config"
+    write-host "building with: cmake --build $script:buildDir --config $script:config ($script:cmakeTargets | ForEach-Object { "--target", $_ })"
-    & cmake --build $script:buildDir --config $script:config
+    & cmake --build $script:buildDir --config $script:config ($script:cmakeTargets | ForEach-Object { "--target", $_ })
     if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
 }
 
 function install {
-    rm -erroraction ignore -recurse -force -path $script:installDir
+    rm -ea 0 -recurse -force -path "${script:buildDir}/lib"
-    & cmake --install $script:buildDir --prefix $script:installDir --config $script:config
+    md "${script:buildDir}/lib" -ea 0 > $null
-    if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
+    cp "${script:buildDir}/bin/${script:config}/ext_server_shared.dll" "${script:buildDir}/lib"
+    cp "${script:buildDir}/bin/${script:config}/llama.dll" "${script:buildDir}/lib"
+}
 
+function cleanup {
+    Set-Location "gguf/examples/server"
+    git checkout CMakeLists.txt server.cpp
 }
 
 init_vars
@@ -54,40 +61,24 @@ git_module_setup
 apply_patches
 
 # first build CPU based
-$script:buildDir="gguf/build/wincpu"
+$script:buildDir="gguf/build/windows/cpu"
-$script:installDir="gguf/build/wincpu/dist"
 
 build
-# install
+install
-
-md gguf/build/lib -ea 0
-md gguf/build/wincpu/dist/lib -ea 0
-mv gguf/build/wincpu/bin/$script:config/ext_server_shared.dll gguf/build/wincpu/dist/lib/cpu_server.dll
-
-
-# Nope, this barfs on lots of symbol problems
-#mv gguf/build/wincpu/examples/server/$script:config/ext_server_shared.dll gguf/build/wincpu/dist/lib/cpu_server.lib
-# Nope: this needs lots of include paths to pull in things like msvcprt.lib and other deps
-# & cl.exe `
-#     gguf/build/wincpu/examples/server/$script:config/ext_server.lib `
-#     gguf/build/wincpu/common/$script:config/common.lib `
-#     gguf/build/wincpu/$script:config/llama.lib `
-#     gguf/build/wincpu/$script:config/ggml_static.lib `
-#     /link /DLL /DEF:cpu_server.def /NOENTRY /MACHINE:X64  /OUT:gguf/build/wincpu/dist/lib/cpu_server.dll
-# if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
 
 # Then build cuda as a dynamically loaded library
 init_vars
-$script:buildDir="gguf/build/wincuda"
+$script:buildDir="gguf/build/windows/cuda"
-$script:installDir="gguf/build/wincuda/dist"
+$script:cmakeDefs += @("-DLLAMA_CUBLAS=ON")
-$script:cmakeDefs += @("-DLLAMA_CUBLAS=ON", "-DBUILD_SHARED_LIBS=on")
 build
 install
-cp gguf/build/wincuda/dist/bin/ext_server_shared.dll gguf/build/lib/cuda_server.dll
 
-# TODO - more to do here to create a usable dll
+# TODO - actually implement ROCm support on windows
+$script:buildDir="gguf/build/windows/rocm"
 
+rm -ea 0 -recurse -force -path "${script:buildDir}/lib"
+md "${script:buildDir}/lib" -ea 0 > $null
+echo $null >> "${script:buildDir}/lib/.generated"
 
-# TODO - implement ROCm support on windows
+cleanup
-md gguf/build/winrocm/lib -ea 0
+write-host "`ngo generate completed"
-echo $null >> gguf/build/winrocm/lib/.generated

llm/llama.cpp/patches/0001-Expose-callable-API-for-server.patch

@@ -1,464 +0,0 @@
-From 90c332fe2ef61149b38561d02836e66715df214d Mon Sep 17 00:00:00 2001
-From: Daniel Hiltgen <daniel@ollama.com>
-Date: Mon, 13 Nov 2023 12:25:58 -0800
-Subject: [PATCH] Expose callable API for server
-
-This adds an extern "C" interface within the example server
----
- examples/server/CMakeLists.txt |  27 ++++
- examples/server/server.cpp     | 280 +++++++++++++++++++++++++++++++++
- examples/server/server.h       |  89 +++++++++++
- ggml-cuda.cu                   |   1 +
- 4 files changed, 397 insertions(+)
- create mode 100644 examples/server/server.h
-
-diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt
-index 859cd12..da2b9bf 100644
---- a/examples/server/CMakeLists.txt
-+++ b/examples/server/CMakeLists.txt
-@@ -11,3 +11,30 @@ if (WIN32)
-     TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ws2_32)
- endif()
- target_compile_features(${TARGET} PRIVATE cxx_std_11)
-+
-+set(TARGET ext_server)
-+option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON)
-+add_library(${TARGET} STATIC server.cpp)
-+target_include_directories(${TARGET} PRIVATE ../../common)
-+target_include_directories(${TARGET} PRIVATE ../..)
-+target_compile_features(${TARGET} PRIVATE cxx_std_11)
-+target_compile_definitions(${TARGET} PUBLIC LLAMA_SERVER_LIBRARY=1)
-+target_link_libraries(${TARGET} PRIVATE common llama llava ${CMAKE_THREAD_LIBS_INIT})
-+target_compile_definitions(${TARGET} PRIVATE
-+    SERVER_VERBOSE=$<BOOL:${LLAMA_SERVER_VERBOSE}>
-+)
-+
-+if (BUILD_SHARED_LIBS)
-+    set_target_properties(ext_server PROPERTIES POSITION_INDEPENDENT_CODE ON)
-+    target_compile_definitions(ext_server PRIVATE LLAMA_SHARED LLAMA_BUILD)
-+    add_library(ext_server_shared SHARED $<TARGET_OBJECTS:ext_server>)
-+    target_link_libraries(ext_server_shared PRIVATE ggml llama llava common ${CMAKE_THREAD_LIBS_INIT})
-+    install(TARGETS ext_server_shared LIBRARY)
-+endif()
-+
-+if (CUDAToolkit_FOUND)
-+    target_include_directories(${TARGET} PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-+    if (WIN32)
-+        target_link_libraries(ext_server_shared PRIVATE nvml)
-+    endif()
-+endif()
-\ No newline at end of file
-diff --git a/examples/server/server.cpp b/examples/server/server.cpp
-index 0403853..07fb05c 100644
---- a/examples/server/server.cpp
-+++ b/examples/server/server.cpp
-@@ -5,6 +5,9 @@
- #include "../llava/clip.h"
- 
- #include "stb_image.h"
-+#if defined(LLAMA_SERVER_LIBRARY)
-+#include "server.h"
-+#endif
- 
- #ifndef NDEBUG
- // crash the server in debug mode, otherwise send an http 500 error
-@@ -2643,6 +2646,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con
-     }
- }
- 
-+#ifndef LLAMA_SERVER_LIBRARY
- int main(int argc, char **argv)
- {
- #if SERVER_VERBOSE != 1
-@@ -3123,3 +3127,279 @@ int main(int argc, char **argv)
-     llama_backend_free();
-     return 0;
- }
-+
-+#else // LLAMA_SERVER_LIBRARY
-+// Expose the llama server as a callable extern "C" API
-+llama_server_context *llama = NULL;
-+std::atomic<bool> ext_server_running(false);
-+std::thread ext_server_thread;
-+
-+void llama_server_init(ext_server_params *sparams, ext_server_resp_t *err)
-+{
-+#if SERVER_VERBOSE != 1
-+    LOG_TEE("disabling verbose llm logging\n");
-+    log_disable();
-+#endif
-+    assert(err != NULL && sparams != NULL);
-+    err->id = 0;
-+    err->msg[0] = '\0';
-+    try {
-+        llama = new llama_server_context;
-+        log_set_target(stdout);
-+        gpt_params params;
-+        params.n_ctx = sparams->n_ctx;
-+        params.n_batch = sparams->n_batch;
-+        if (sparams->n_threads > 0) {
-+            params.n_threads = sparams->n_threads;
-+        }
-+        params.n_parallel = sparams->n_parallel;
-+        params.rope_freq_base = sparams->rope_freq_base;
-+        params.rope_freq_scale = sparams->rope_freq_scale;
-+
-+        if (sparams->memory_f16)  {
-+            params.cache_type_k = "f16";
-+            params.cache_type_v = "f16";
-+        } else {
-+            params.cache_type_k = "f32";
-+            params.cache_type_v = "f32";
-+        }
-+
-+        params.n_gpu_layers = sparams->n_gpu_layers;
-+        params.main_gpu = sparams->main_gpu;
-+        params.use_mlock = sparams->use_mlock;
-+        params.use_mmap = sparams->use_mmap;
-+        params.numa = sparams->numa;
-+        params.embedding = sparams->embedding;
-+        if (sparams->model != NULL) {
-+            params.model = sparams->model;
-+        }
-+
-+        for (ext_server_lora_adapter *la = sparams->lora_adapters; la != NULL; la = la->next) {
-+            params.lora_adapter.push_back(std::make_tuple(la->adapter, la->scale));
-+        }
-+
-+        if (sparams->mmproj != NULL) {
-+            params.mmproj = std::string(sparams->mmproj);
-+        }
-+           
-+        llama_backend_init(params.numa);
-+
-+        // load the model
-+        if (!llama->load_model(params))
-+        {
-+            // TODO - consider modifying the logging logic or patching load_model so we can capture more detailed error messages
-+            // and pass them back to the caller for better UX
-+            err->id = -1;
-+            snprintf(err->msg, err->msg_len, "error loading model %s", params.model.c_str());
-+            return;
-+        }
-+
-+        llama->initialize();
-+    } catch (std::exception &e) {
-+        err->id = -1;
-+        snprintf(err->msg, err->msg_len, "exception %s", e.what());
-+    } catch (...) {
-+        err->id = -1;
-+        snprintf(err->msg, err->msg_len, "Unknown exception initializing llama server");
-+    }
-+}
-+
-+void llama_server_start()
-+{
-+    assert(llama != NULL);
-+     // TODO mutex to protect thread creation
-+    ext_server_thread = std::thread([&]()
-+    {
-+        ext_server_running = true;
-+        try {
-+            LOG_TEE("llama server main loop starting\n");
-+            ggml_time_init();
-+            while (ext_server_running.load())
-+            {
-+                if (!llama->update_slots()) {
-+                    LOG_TEE("unexpected error in llama server update_slots - exiting main loop\n");
-+                    break;
-+                }
-+            }
-+        } catch (std::exception &e) {
-+            LOG_TEE("caught exception in llama server main loop: %s\n", e.what());
-+        } catch (...) {
-+            LOG_TEE("caught unknown exception in llama server main loop\n");
-+        }
-+        LOG_TEE("\nllama server shutting down\n");
-+        llama_backend_free();
-+    });
-+}
-+
-+void llama_server_stop() {
-+    assert(llama != NULL);
-+    // TODO - too verbose, remove once things are solid
-+    LOG_TEE("requesting llama server shutdown\n");
-+    ext_server_running = false;
-+    ext_server_thread.join();
-+    delete llama;
-+    llama = NULL;
-+    LOG_TEE("llama server shutdown complete\n");
-+}
-+
-+void llama_server_completion(const char *json_req, ext_server_resp_t *resp) {
-+    assert(llama != NULL && json_req != NULL && resp != NULL);
-+    resp->id = -1;
-+    resp->msg[0] = '\0';
-+    try {
-+        json data = json::parse(json_req);
-+        resp->id = llama->request_completion(data, false, false, -1);
-+    } catch (std::exception &e) {
-+        snprintf(resp->msg, resp->msg_len, "exception %s", e.what());
-+    } catch (...) {
-+        snprintf(resp->msg, resp->msg_len, "Unknown exception during completion");
-+    }
-+}
-+
-+void llama_server_completion_next_result(const int task_id, ext_server_task_result_t *resp) {
-+    assert(llama != NULL && resp != NULL);
-+    std::string msg;
-+    resp->id = -1;
-+    resp->stop = false;
-+    resp->error = false;
-+    resp->json_resp = NULL;
-+    std::string result_json;
-+    try {
-+        task_result result = llama->next_result(task_id);
-+        result_json = result.result_json.dump(-1, ' ', false, json::error_handler_t::replace);
-+        resp->id = result.id;
-+        resp->stop = result.stop;
-+        resp->error = result.error;
-+        if (result.error) {
-+            llama->request_cancel(task_id);
-+        } else if (result.stop) {
-+            llama->request_cancel(task_id);
-+        }
-+    } catch (std::exception &e) {
-+        resp->error = true;
-+        resp->id = -1;
-+        result_json = "{\"error\":\"exception " + std::string(e.what()) + "\"}";
-+    } catch (...) {
-+        resp->error = true;
-+        resp->id = -1;
-+        result_json = "{\"error\":\"Unknown exception during completion\"}";
-+    }
-+    const std::string::size_type size = result_json.size() + 1;
-+    resp->json_resp = new char[size];
-+    snprintf(resp->json_resp, size, "%s", result_json.c_str());
-+}
-+
-+void llama_server_release_task_result(ext_server_task_result_t *result) {
-+    if (result == NULL || result->json_resp == NULL) {
-+        return;
-+    }
-+    delete[] result->json_resp;
-+}
-+
-+void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err) {
-+    assert(llama != NULL && err != NULL);
-+    err->id = 0;
-+    err->msg[0] = '\0';
-+    try {
-+        llama->request_cancel(task_id);
-+    } catch (std::exception &e) {
-+        err->id = -1;
-+        snprintf(err->msg, err->msg_len, "exception %s", e.what());
-+    } catch (...) {
-+        err->id = -1;
-+        snprintf(err->msg, err->msg_len, "Unknown exception completion cancel in llama server");
-+    }
-+}
-+
-+void llama_server_tokenize(const char *json_req, char **json_resp, ext_server_resp_t *err) {
-+    assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
-+    *json_resp = NULL;
-+    err->id = 0;
-+    err->msg[0] = '\0';
-+    try {
-+        const json body = json::parse(json_req);
-+        std::vector<llama_token> tokens;
-+        if (body.count("content") != 0)
-+        {
-+            tokens = llama->tokenize(body["content"], false);
-+        }
-+        const json data = format_tokenizer_response(tokens);
-+        std::string result_json = data.dump();
-+        const std::string::size_type size = result_json.size() + 1;
-+        *json_resp = new char[size];
-+        snprintf(*json_resp, size, "%s", result_json.c_str());
-+    } catch (std::exception &e) {
-+        err->id = -1;
-+        snprintf(err->msg, err->msg_len, "exception %s", e.what());
-+    } catch (...) {
-+        err->id = -1;
-+        snprintf(err->msg, err->msg_len, "Unknown exception during tokenize");
-+    }
-+}
-+
-+void llama_server_release_json_resp(char **json_resp) {
-+    if (json_resp == NULL || *json_resp == NULL) {
-+        return;
-+    }
-+    delete[] *json_resp;
-+}
-+
-+void llama_server_detokenize(const char *json_req, char **json_resp, ext_server_resp_t *err) {
-+    assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
-+    *json_resp = NULL;
-+    err->id = 0;
-+    err->msg[0] = '\0';
-+    try {
-+        const json body = json::parse(json_req);
-+        std::string content;
-+        if (body.count("tokens") != 0)
-+        {
-+            const std::vector<llama_token> tokens = body["tokens"];
-+            content = tokens_to_str(llama->ctx, tokens.cbegin(), tokens.cend());
-+        }
-+        const json data = format_detokenized_response(content);
-+        std::string result_json = data.dump();
-+        const std::string::size_type size = result_json.size() + 1;
-+        *json_resp = new char[size];
-+        snprintf(*json_resp, size, "%s", result_json.c_str());
-+    } catch (std::exception &e) {
-+        err->id = -1;
-+        snprintf(err->msg, err->msg_len, "exception %s", e.what());
-+    } catch (...) {
-+        err->id = -1;
-+        snprintf(err->msg, err->msg_len, "Unknown exception during detokenize");
-+    }
-+}
-+
-+void llama_server_embedding(const char *json_req, char** json_resp, ext_server_resp_t *err) {
-+    assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
-+    *json_resp = NULL;
-+    err->id = 0;
-+    err->msg[0] = '\0';
-+    try {
-+        const json body = json::parse(json_req);
-+        json prompt;
-+        if (body.count("content") != 0)
-+        {
-+            prompt = body["content"];
-+        }
-+        else
-+        {
-+            prompt = "";
-+        }
-+        const int task_id = llama->request_completion({ {"prompt", prompt}, { "n_predict", 0} }, false, true, -1);
-+        task_result result = llama->next_result(task_id);
-+        std::string result_json = result.result_json.dump();
-+        const std::string::size_type size = result_json.size() + 1;
-+        *json_resp = new char[size];
-+        snprintf(*json_resp, size, "%s", result_json.c_str());
-+    } catch (std::exception &e) {
-+        err->id = -1;
-+        snprintf(err->msg, err->msg_len, "exception %s", e.what());
-+    } catch (...) {
-+        err->id = -1;
-+        snprintf(err->msg, err->msg_len, "Unknown exception during embedding");
-+    }
-+}
-+
-+#endif // LLAMA_SERVER_LIBRARY
-\ No newline at end of file
-diff --git a/examples/server/server.h b/examples/server/server.h
-new file mode 100644
-index 0000000..d22f1b6
---- /dev/null
-+++ b/examples/server/server.h
-@@ -0,0 +1,89 @@
-+#if defined(LLAMA_SERVER_LIBRARY)
-+#ifndef LLAMA_SERVER_H
-+#define LLAMA_SERVER_H
-+#include <stddef.h>
-+#include <stdint.h>
-+#include <stdio.h>
-+#include <stdbool.h>
-+
-+// This exposes extern C entrypoints into the llama_server 
-+// To enable the server compile with LLAMA_SERVER_LIBRARY
-+
-+#ifdef __cplusplus
-+extern "C"
-+{
-+#endif
-+    typedef struct ext_server_resp {
-+        int id; // < 0 on error
-+        size_t msg_len; // caller must allocate msg and set msg_len
-+        char *msg;
-+    } ext_server_resp_t;
-+
-+    // Allocated and freed by caller
-+    typedef struct ext_server_lora_adapter {
-+        char *adapter;
-+        float scale;
-+        struct ext_server_lora_adapter *next;
-+    } ext_server_lora_adapter_t;
-+
-+    // Allocated and freed by caller
-+    typedef struct ext_server_params
-+    {
-+        char *model;            
-+        uint32_t n_ctx;         // text context, 0 = from model
-+        uint32_t n_batch;       // prompt processing maximum batch size
-+        uint32_t n_threads;     // number of threads to use for generation
-+        int32_t n_parallel;     // number of parallel sequences to decodewra
-+        float rope_freq_base;   // RoPE base frequency, 0 = from model
-+        float rope_freq_scale;  // RoPE frequency scaling factor, 0 = from model
-+        bool memory_f16;        // use f16 instead of f32 for memory kv
-+        int32_t n_gpu_layers;   // number of layers to store in VRAM (-1 - use default)
-+        int32_t main_gpu;       // the GPU that is used for scratch and small tensors
-+        bool use_mlock;         // force system to keep model in RAM
-+        bool use_mmap;          // use mmap if possible
-+        bool numa;              // attempt optimizations that help on some NUMA systems
-+        bool embedding;         // get only sentence embedding
-+        ext_server_lora_adapter_t* lora_adapters;
-+        char *mmproj;
-+    } ext_server_params_t;
-+
-+    typedef struct ext_server_task_result
-+    {
-+        int id;
-+        bool stop;
-+        bool error;
-+        char* json_resp; // null terminated, memory managed by ext_server
-+    } ext_server_task_result_t;
-+
-+    // Initialize the server once per process
-+    // err->id = 0 for success and err->msg[0] = NULL
-+    // err->id != 0 for failure, and err->msg contains error message
-+    void llama_server_init(ext_server_params_t *sparams, ext_server_resp_t *err);
-+
-+    // Run the main loop, called once per init
-+    void llama_server_start();
-+    // Stop the main loop and free up resources allocated in init and start.  Init must be called again to reuse
-+    void llama_server_stop();
-+
-+    // json_req null terminated string, memory managed by caller
-+    // resp->id >= 0 on success (task ID)
-+    // resp->id < 0 on error, and resp->msg contains error message
-+    void llama_server_completion(const char *json_req, ext_server_resp_t *resp);
-+
-+    // Caller must call llama_server_release_task_result to free resp->json_resp
-+    void llama_server_completion_next_result(const int task_id, ext_server_task_result_t *result);
-+    void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err);
-+    void llama_server_release_task_result(ext_server_task_result_t *result);
-+
-+    // Caller must call llama_server_releaes_json_resp to free json_resp if err.id < 0
-+    void llama_server_tokenize(const char *json_req, char **json_resp, ext_server_resp_t *err);
-+    void llama_server_detokenize(const char *json_req, char **json_resp, ext_server_resp_t *err);
-+    void llama_server_embedding(const char *json_req, char** json_resp, ext_server_resp_t *err);
-+    void llama_server_release_json_resp(char **json_resp);
-+
-+#ifdef __cplusplus
-+}
-+#endif
-+
-+#endif
-+#endif // LLAMA_SERVER_LIBRARY
-\ No newline at end of file
-diff --git a/ggml-cuda.cu b/ggml-cuda.cu
-index f20846f..9640cf3 100644
---- a/ggml-cuda.cu
-+++ b/ggml-cuda.cu
-@@ -6757,6 +6757,7 @@ static cudaError_t ggml_cuda_cpy_tensor_2d(
-         CUDA_CHECK(cudaGetDevice(&id));
-         src_ptr = (char *) extra->data_device[id];
-     } else {
-+        fprintf(stderr, "ggml_cuda_cpy_tensor_2d assert: backend: %d\n", src->backend);
-         GGML_ASSERT(false);
-     }
-     char * dst_ptr = (char *) dst;
--- 
-2.39.3 (Apple Git-145)
-

llm/llama.go

@@ -6,11 +6,8 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
-	"io"
-	"io/fs"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"sync"
 	"time"
 
@@ -206,41 +203,3 @@ type EmbeddingRequest struct {
 type EmbeddingResponse struct {
 	Embedding []float64 `json:"embedding"`
 }
-
-func extractDynamicLibs(workDir, glob string) ([]string, error) {
-	files, err := fs.Glob(libEmbed, glob)
-	if err != nil || len(files) == 0 {
-		return nil, payloadMissing
-	}
-	libs := make([]string, len(files))
-
-	for i, file := range files {
-		srcFile, err := libEmbed.Open(file)
-		if err != nil {
-			return nil, fmt.Errorf("read payload %s: %v", file, err)
-		}
-		defer srcFile.Close()
-		if err := os.MkdirAll(workDir, 0o755); err != nil {
-			return nil, fmt.Errorf("create payload temp dir %s: %v", workDir, err)
-		}
-
-		destFile := filepath.Join(workDir, filepath.Base(file))
-		libs[i] = destFile
-
-		_, err = os.Stat(destFile)
-		switch {
-		case errors.Is(err, os.ErrNotExist):
-			destFile, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
-			if err != nil {
-				return nil, fmt.Errorf("write payload %s: %v", file, err)
-			}
-			defer destFile.Close()
-			if _, err := io.Copy(destFile, srcFile); err != nil {
-				return nil, fmt.Errorf("copy payload %s: %v", file, err)
-			}
-		case err != nil:
-			return nil, fmt.Errorf("stat payload %s: %v", file, err)
-		}
-	}
-	return libs, nil
-}

llm/shim_darwin.go

@@ -2,9 +2,13 @@ package llm
 
 import (
 	"embed"
+	"errors"
 	"fmt"
+	"io"
+	"io/fs"
 	"log"
 	"os"
+	"path/filepath"
 
 	"github.com/jmorganca/ollama/api"
 )
@@ -18,7 +22,7 @@ func newDynamicShimExtServer(library, model string, adapters, projectors []strin
 }
 
 func nativeInit(workdir string) error {
-	_, err := extractDynamicLibs(workdir, "llama.cpp/gguf/ggml-metal.metal")
+	err := extractPayloadFiles(workdir, "llama.cpp/gguf/ggml-metal.metal")
 	if err != nil {
 		if err == payloadMissing {
 			// TODO perhaps consider this a hard failure on arm macs?
@@ -30,3 +34,38 @@ func nativeInit(workdir string) error {
 	os.Setenv("GGML_METAL_PATH_RESOURCES", workdir)
 	return nil
 }
+
+func extractPayloadFiles(workDir, glob string) error {
+	files, err := fs.Glob(libEmbed, glob)
+	if err != nil || len(files) == 0 {
+		return payloadMissing
+	}
+
+	for _, file := range files {
+		srcFile, err := libEmbed.Open(file)
+		if err != nil {
+			return fmt.Errorf("read payload %s: %v", file, err)
+		}
+		defer srcFile.Close()
+		if err := os.MkdirAll(workDir, 0o755); err != nil {
+			return fmt.Errorf("create payload temp dir %s: %v", workDir, err)
+		}
+
+		destFile := filepath.Join(workDir, filepath.Base(file))
+		_, err = os.Stat(destFile)
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+			destFile, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
+			if err != nil {
+				return fmt.Errorf("write payload %s: %v", file, err)
+			}
+			defer destFile.Close()
+			if _, err := io.Copy(destFile, srcFile); err != nil {
+				return fmt.Errorf("copy payload %s: %v", file, err)
+			}
+		case err != nil:
+			return fmt.Errorf("stat payload %s: %v", file, err)
+		}
+	}
+	return nil
+}

llm/shim_ext_server.go

@@ -11,9 +11,9 @@ package llm
 import "C"
 import (
 	"context"
-	"embed"
 	"errors"
 	"fmt"
+	"io"
 	"io/fs"
 	"log"
 	"os"
@@ -25,11 +25,6 @@ import (
 	"github.com/jmorganca/ollama/api"
 )
 
-//go:embed llama.cpp/gguf/build/lib/*
-var libEmbed embed.FS
-
-var RocmShimMissing = fmt.Errorf("ROCm shim library not included in this build of ollama. Radeon GPUs are not supported")
-
 type shimExtServer struct {
 	s       C.struct_dynamic_llama_server
 	options api.Options
@@ -78,6 +73,7 @@ func (llm *shimExtServer) llama_server_release_json_resp(json_resp **C.char) {
 func newDynamicShimExtServer(library, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
 	shimMutex.Lock()
 	defer shimMutex.Unlock()
+	updatePath(filepath.Dir(library))
 	libPath := C.CString(library)
 	defer C.free(unsafe.Pointer(libPath))
 	resp := newExtServerResp(128)
@@ -116,7 +112,7 @@ func (llm *shimExtServer) Close() {
 }
 
 func nativeInit(workdir string) error {
-	libs, err := extractDynamicLibs(workdir, "llama.cpp/gguf/build/lib/*server*")
+	libs, err := extractDynamicLibs(workdir, "llama.cpp/gguf/build/*/*/lib/*")
 	if err != nil {
 		if err == payloadMissing {
 			log.Printf("%s", payloadMissing)
@@ -125,28 +121,71 @@ func nativeInit(workdir string) error {
 		return err
 	}
 	for _, lib := range libs {
-		libName := strings.Split(strings.TrimPrefix(filepath.Base(lib), "lib"), ".")[0]
+		// The last dir component is the variant name
-		AvailableShims[libName] = lib
+		variant := filepath.Base(filepath.Dir(lib))
+		AvailableShims[variant] = lib
 	}
 
-	// Only check ROCm access if we have the dynamic lib loaded
+	if err := verifyDriverAccess(); err != nil {
-	if _, rocmPresent := AvailableShims["rocm_server"]; rocmPresent {
+		return err
-		// Verify we have permissions - either running as root, or we have group access to the driver
-		fd, err := os.OpenFile("/dev/kfd", os.O_RDWR, 0666)
-		if err != nil {
-			if errors.Is(err, fs.ErrPermission) {
-				log.Fatalf("Radeon card detected, but permissions not set up properly.  Either run ollama as root, or add you user account to the render group.")
-				return err
-			} else if errors.Is(err, fs.ErrNotExist) {
-				// expected behavior without a radeon card
-				return nil
-			}
-
-			return fmt.Errorf("failed to check permission on /dev/kfd: %w", err)
-		}
-		fd.Close()
-
 	}
 
+	// Report which dynamic libraries we have loaded to assist troubleshooting
+	variants := make([]string, len(AvailableShims))
+	i := 0
+	for variant := range AvailableShims {
+		variants[i] = variant
+		i++
+	}
+	log.Printf("Dynamic LLM variants %v", variants)
+
 	return nil
 }
+
+func extractDynamicLibs(workDir, glob string) ([]string, error) {
+	files, err := fs.Glob(libEmbed, glob)
+	if err != nil || len(files) == 0 {
+		return nil, payloadMissing
+	}
+	libs := make([]string, len(files))
+
+	for i, file := range files {
+		pathComps := strings.Split(file, "/")
+		if len(pathComps) != 7 {
+			log.Printf("unexpected payload components: %v", pathComps)
+			continue
+		}
+		// llama.cpp/gguf/build/$OS/$VARIANT/lib/$LIBRARY
+		// Include the variant in the path to avoid conflicts between multiple server libs
+		targetDir := filepath.Join(workDir, pathComps[4])
+		srcFile, err := libEmbed.Open(file)
+		if err != nil {
+			return nil, fmt.Errorf("read payload %s: %v", file, err)
+		}
+		defer srcFile.Close()
+		if err := os.MkdirAll(targetDir, 0o755); err != nil {
+			return nil, fmt.Errorf("create payload temp dir %s: %v", workDir, err)
+		}
+
+		destFile := filepath.Join(targetDir, filepath.Base(file))
+		if strings.Contains(destFile, "server") {
+			libs[i] = destFile
+		}
+
+		_, err = os.Stat(destFile)
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+			destFile, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
+			if err != nil {
+				return nil, fmt.Errorf("write payload %s: %v", file, err)
+			}
+			defer destFile.Close()
+			if _, err := io.Copy(destFile, srcFile); err != nil {
+				return nil, fmt.Errorf("copy payload %s: %v", file, err)
+			}
+		case err != nil:
+			return nil, fmt.Errorf("stat payload %s: %v", file, err)
+		}
+	}
+	return libs, nil
+}

llm/shim_ext_server_linux.go

@@ -0,0 +1,46 @@
+package llm
+
+import (
+	"embed"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"strings"
+)
+
+//go:embed llama.cpp/gguf/build/*/*/lib/*.so
+var libEmbed embed.FS
+
+func updatePath(dir string) {
+	pathComponents := strings.Split(os.Getenv("PATH"), ":")
+	for _, comp := range pathComponents {
+		if comp == dir {
+			return
+		}
+	}
+	newPath := strings.Join(append(pathComponents, dir), ":")
+	log.Printf("Updating PATH to %s", newPath)
+	os.Setenv("PATH", newPath)
+}
+
+func verifyDriverAccess() error {
+	// Only check ROCm access if we have the dynamic lib loaded
+	if _, rocmPresent := AvailableShims["rocm"]; rocmPresent {
+		// Verify we have permissions - either running as root, or we have group access to the driver
+		fd, err := os.OpenFile("/dev/kfd", os.O_RDWR, 0666)
+		if err != nil {
+			if errors.Is(err, fs.ErrPermission) {
+				return fmt.Errorf("Radeon card detected, but permissions not set up properly.  Either run ollama as root, or add you user account to the render group.")
+			} else if errors.Is(err, fs.ErrNotExist) {
+				// expected behavior without a radeon card
+				return nil
+			}
+
+			return fmt.Errorf("failed to check permission on /dev/kfd: %w", err)
+		}
+		fd.Close()
+	}
+	return nil
+}

llm/shim_ext_server_windows.go

@@ -0,0 +1,29 @@
+package llm
+
+import (
+	"embed"
+	"log"
+	"os"
+	"strings"
+)
+
+//go:embed llama.cpp/gguf/build/windows/*/lib/*.dll
+var libEmbed embed.FS
+
+func updatePath(dir string) {
+	pathComponents := strings.Split(os.Getenv("PATH"), ";")
+	for _, comp := range pathComponents {
+		// Case incensitive
+		if strings.ToLower(comp) == strings.ToLower(dir) {
+			return
+		}
+	}
+	newPath := strings.Join(append(pathComponents, dir), ";")
+	log.Printf("Updating PATH to %s", newPath)
+	os.Setenv("PATH", newPath)
+}
+
+func verifyDriverAccess() error {
+	// TODO if applicable
+	return nil
+}