diff --git a/CMakeLists.txt b/CMakeLists.txt
index 19d9bd8f9..58cb8b449 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -29,6 +29,11 @@ if((NOT CMAKE_OSX_ARCHITECTURES MATCHES "arm64")
     set(GGML_CPU_ALL_VARIANTS ON)
 endif()
 
+if (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64")
+    set(CMAKE_BUILD_RPATH "@loader_path")
+    set(CMAKE_INSTALL_RPATH "@loader_path")
+endif()
+
 set(OLLAMA_BUILD_DIR ${CMAKE_BINARY_DIR}/lib/ollama)
 set(OLLAMA_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/ollama)
 
diff --git a/llm/server.go b/llm/server.go
index 9daec344b..881209b39 100644
--- a/llm/server.go
+++ b/llm/server.go
@@ -281,9 +281,14 @@ func NewLlamaServer(gpus discover.GpuInfoList, model string, ggml *GGML, adapter
 		finalParams = append(finalParams, params...)
 		finalParams = append(finalParams, "--port", strconv.Itoa(port))
 
-		pathEnv := "LD_LIBRARY_PATH"
-		if runtime.GOOS == "windows" {
+		var pathEnv string
+		switch runtime.GOOS {
+		case "windows":
 			pathEnv = "PATH"
+		case "darwin":
+			pathEnv = "DYLD_LIBRARY_PATH"
+		default:
+			pathEnv = "LD_LIBRARY_PATH"
 		}
 
 		var libraryPaths []string
@@ -385,7 +390,8 @@ func NewLlamaServer(gpus discover.GpuInfoList, model string, ggml *GGML, adapter
 					strings.HasPrefix(ev, "HSA_") ||
 					strings.HasPrefix(ev, "GGML_") ||
 					strings.HasPrefix(ev, "PATH=") ||
-					strings.HasPrefix(ev, "LD_LIBRARY_PATH=") {
+					strings.HasPrefix(ev, "LD_LIBRARY_PATH=") ||
+					strings.HasPrefix(ev, "DYLD_LIBRARY_PATH=") {
 					filteredEnv = append(filteredEnv, ev)
 				}
 			}
diff --git a/ml/backend/ggml/ggml/src/ggml.go b/ml/backend/ggml/ggml/src/ggml.go
index 7cf40e705..94b0d1853 100644
--- a/ml/backend/ggml/ggml/src/ggml.go
+++ b/ml/backend/ggml/ggml/src/ggml.go
@@ -41,36 +41,48 @@ func sink(level C.int, text *C.char, _ unsafe.Pointer) {
 }
 
 var OnceLoad = sync.OnceFunc(func() {
-	var lib struct{ name, defaultValue string }
+	exe, err := os.Executable()
+	if err != nil {
+		slog.Warn("failed to get executable path", "error", err)
+		exe = "."
+	}
+
+	// PATH, LD_LIBRARY_PATH, and DYLD_LIBRARY_PATH are often
+	// set by the parent process, however, use a default value
+	// if the environment variable is not set.
+	var name, value string
 	switch runtime.GOOS {
-	case "darwin", "linux":
-		lib.name = "LD_LIBRARY_PATH"
-		lib.defaultValue = "/usr/local/lib:/usr/lib"
+	case "darwin":
+		// On macOS, DYLD_LIBRARY_PATH is often not set, so
+		// we use the directory of the executable as the default.
+		name = "DYLD_LIBRARY_PATH"
+		value = filepath.Dir(exe)
 	case "windows":
-		lib.name = "PATH"
-		lib.defaultValue = "."
+		name = "PATH"
+		value = filepath.Join(filepath.Dir(exe), "lib", "ollama")
 	default:
-		return
+		name = "LD_LIBRARY_PATH"
+		value = filepath.Join(filepath.Dir(exe), "..", "lib", "ollama")
 	}
 
-	paths, ok := os.LookupEnv(lib.name)
+	paths, ok := os.LookupEnv(name)
 	if !ok {
-		paths = lib.defaultValue
-	}
-
-	if runtime.GOOS == "darwin" {
-		if _, ok := os.LookupEnv("DYLD_LIBRARY_PATH"); !ok {
-			os.Setenv("DYLD_LIBRARY_PATH", paths)
-		}
+		paths = value
 	}
 
 	split := filepath.SplitList(paths)
 	visited := make(map[string]struct{}, len(split))
 	for _, path := range split {
-		abspath, _ := filepath.Abs(path)
+		abspath, err := filepath.Abs(path)
+		if err != nil {
+			slog.Error("failed to get absolute path", "error", err)
+			continue
+		}
+
 		if _, ok := visited[abspath]; !ok {
 			func() {
-				cpath := C.CString(path)
+				slog.Debug("ggml backend load all from path", "path", abspath)
+				cpath := C.CString(abspath)
 				defer C.free(unsafe.Pointer(cpath))
 				C.ggml_backend_load_all_from_path(cpath)
 			}()
diff --git a/scripts/build_darwin.sh b/scripts/build_darwin.sh
index baba3c1fd..76d0a6c2b 100755
--- a/scripts/build_darwin.sh
+++ b/scripts/build_darwin.sh
@@ -32,10 +32,10 @@ _build_darwin() {
             status "Building darwin $ARCH dynamic backends"
             cmake -B build/darwin-$ARCH \
                 -DCMAKE_OSX_ARCHITECTURES=x86_64 \
-                -DCMAKE_OSX_DEPLOYMENT_TARGET=11.3
+                -DCMAKE_OSX_DEPLOYMENT_TARGET=11.3 \
+                -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX
             cmake --build build/darwin-$ARCH --target ggml-cpu -j
-            install -d $INSTALL_PREFIX/lib/ollama
-            install build/darwin-$ARCH/lib/ollama/*.{dylib,so} $INSTALL_PREFIX/lib/ollama
+            cmake --install build/darwin-$ARCH --component CPU
         fi
     done
 }