ml/backend/ggml: fix library loading on macOS amd64 (#8827)

CMakeLists.txt

@@ -29,6 +29,11 @@ if((NOT CMAKE_OSX_ARCHITECTURES MATCHES "arm64")
     set(GGML_CPU_ALL_VARIANTS ON)
 endif()
 
+if (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64")
+    set(CMAKE_BUILD_RPATH "@loader_path")
+    set(CMAKE_INSTALL_RPATH "@loader_path")
+endif()
+
 set(OLLAMA_BUILD_DIR ${CMAKE_BINARY_DIR}/lib/ollama)
 set(OLLAMA_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/ollama)
 

llm/server.go

@@ -281,9 +281,14 @@ func NewLlamaServer(gpus discover.GpuInfoList, model string, ggml *GGML, adapter
 		finalParams = append(finalParams, params...)
 		finalParams = append(finalParams, "--port", strconv.Itoa(port))
 
-		pathEnv := "LD_LIBRARY_PATH"
+		var pathEnv string
-		if runtime.GOOS == "windows" {
+		switch runtime.GOOS {
+		case "windows":
 			pathEnv = "PATH"
+		case "darwin":
+			pathEnv = "DYLD_LIBRARY_PATH"
+		default:
+			pathEnv = "LD_LIBRARY_PATH"
 		}
 
 		var libraryPaths []string
@@ -385,7 +390,8 @@ func NewLlamaServer(gpus discover.GpuInfoList, model string, ggml *GGML, adapter
 					strings.HasPrefix(ev, "HSA_") ||
 					strings.HasPrefix(ev, "GGML_") ||
 					strings.HasPrefix(ev, "PATH=") ||
-					strings.HasPrefix(ev, "LD_LIBRARY_PATH=") {
+					strings.HasPrefix(ev, "LD_LIBRARY_PATH=") ||
+					strings.HasPrefix(ev, "DYLD_LIBRARY_PATH=") {
 					filteredEnv = append(filteredEnv, ev)
 				}
 			}

ml/backend/ggml/ggml/src/ggml.go

@@ -41,36 +41,48 @@ func sink(level C.int, text *C.char, _ unsafe.Pointer) {
 }
 
 var OnceLoad = sync.OnceFunc(func() {
-	var lib struct{ name, defaultValue string }
+	exe, err := os.Executable()
+	if err != nil {
+		slog.Warn("failed to get executable path", "error", err)
+		exe = "."
+	}
+
+	// PATH, LD_LIBRARY_PATH, and DYLD_LIBRARY_PATH are often
+	// set by the parent process, however, use a default value
+	// if the environment variable is not set.
+	var name, value string
 	switch runtime.GOOS {
-	case "darwin", "linux":
+	case "darwin":
-		lib.name = "LD_LIBRARY_PATH"
+		// On macOS, DYLD_LIBRARY_PATH is often not set, so
-		lib.defaultValue = "/usr/local/lib:/usr/lib"
+		// we use the directory of the executable as the default.
+		name = "DYLD_LIBRARY_PATH"
+		value = filepath.Dir(exe)
 	case "windows":
-		lib.name = "PATH"
+		name = "PATH"
-		lib.defaultValue = "."
+		value = filepath.Join(filepath.Dir(exe), "lib", "ollama")
 	default:
-		return
+		name = "LD_LIBRARY_PATH"
+		value = filepath.Join(filepath.Dir(exe), "..", "lib", "ollama")
 	}
 
-	paths, ok := os.LookupEnv(lib.name)
+	paths, ok := os.LookupEnv(name)
 	if !ok {
-		paths = lib.defaultValue
+		paths = value
-	}
-
-	if runtime.GOOS == "darwin" {
-		if _, ok := os.LookupEnv("DYLD_LIBRARY_PATH"); !ok {
-			os.Setenv("DYLD_LIBRARY_PATH", paths)
-		}
 	}
 
 	split := filepath.SplitList(paths)
 	visited := make(map[string]struct{}, len(split))
 	for _, path := range split {
-		abspath, _ := filepath.Abs(path)
+		abspath, err := filepath.Abs(path)
+		if err != nil {
+			slog.Error("failed to get absolute path", "error", err)
+			continue
+		}
+
 		if _, ok := visited[abspath]; !ok {
 			func() {
-				cpath := C.CString(path)
+				slog.Debug("ggml backend load all from path", "path", abspath)
+				cpath := C.CString(abspath)
 				defer C.free(unsafe.Pointer(cpath))
 				C.ggml_backend_load_all_from_path(cpath)
 			}()

scripts/build_darwin.sh

@@ -32,10 +32,10 @@ _build_darwin() {
             status "Building darwin $ARCH dynamic backends"
             cmake -B build/darwin-$ARCH \
                 -DCMAKE_OSX_ARCHITECTURES=x86_64 \
-                -DCMAKE_OSX_DEPLOYMENT_TARGET=11.3
+                -DCMAKE_OSX_DEPLOYMENT_TARGET=11.3 \
+                -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX
             cmake --build build/darwin-$ARCH --target ggml-cpu -j
-            install -d $INSTALL_PREFIX/lib/ollama
+            cmake --install build/darwin-$ARCH --component CPU
-            install build/darwin-$ARCH/lib/ollama/*.{dylib,so} $INSTALL_PREFIX/lib/ollama
         fi
     done
 }