mirror of
https://github.com/ollama/ollama.git
synced 2025-11-10 15:37:27 +01:00
3258a89b6e
* DRY out the runner lifecycle code Now that discovery uses the runners as well, this unifies the runner spawning code into a single place. This also unifies GPU discovery types with the newer ml.DeviceInfo * win: make incremental builds better Place build artifacts in discrete directories so incremental builds don't have to start fresh * Adjust sort order to consider iGPUs * handle cpu inference oom scenarios * review comments
74 lines
1.9 KiB
Go
74 lines
1.9 KiB
Go
package discover
|
|
|
|
import (
|
|
"log/slog"
|
|
"os"
|
|
"regexp"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/ollama/ollama/ml"
|
|
)
|
|
|
|
// Jetson devices have JETSON_JETPACK="x.y.z" factory set to the Jetpack version installed.
|
|
// Included to drive logic for reducing Ollama-allocated overhead on L4T/Jetson devices.
|
|
var CudaTegra string = os.Getenv("JETSON_JETPACK")
|
|
|
|
// GetSystemInfo returns the last cached state of the GPUs on the system
|
|
func GetSystemInfo() ml.SystemInfo {
|
|
memInfo, err := GetCPUMem()
|
|
if err != nil {
|
|
slog.Warn("error looking up system memory", "error", err)
|
|
}
|
|
var threadCount int
|
|
cpus := GetCPUDetails()
|
|
for _, c := range cpus {
|
|
threadCount += c.CoreCount - c.EfficiencyCoreCount
|
|
}
|
|
|
|
if threadCount == 0 {
|
|
// Fall back to Go's num CPU
|
|
threadCount = runtime.NumCPU()
|
|
}
|
|
|
|
return ml.SystemInfo{
|
|
ThreadCount: threadCount,
|
|
TotalMemory: memInfo.TotalMemory,
|
|
FreeMemory: memInfo.FreeMemory,
|
|
FreeSwap: memInfo.FreeSwap,
|
|
}
|
|
}
|
|
|
|
func cudaJetpack() string {
|
|
if runtime.GOARCH == "arm64" && runtime.GOOS == "linux" {
|
|
if CudaTegra != "" {
|
|
ver := strings.Split(CudaTegra, ".")
|
|
if len(ver) > 0 {
|
|
return "jetpack" + ver[0]
|
|
}
|
|
} else if data, err := os.ReadFile("/etc/nv_tegra_release"); err == nil {
|
|
r := regexp.MustCompile(` R(\d+) `)
|
|
m := r.FindSubmatch(data)
|
|
if len(m) != 2 {
|
|
slog.Info("Unexpected format for /etc/nv_tegra_release. Set JETSON_JETPACK to select version")
|
|
} else {
|
|
if l4t, err := strconv.Atoi(string(m[1])); err == nil {
|
|
// Note: mapping from L4t -> JP is inconsistent (can't just subtract 30)
|
|
// https://developer.nvidia.com/embedded/jetpack-archive
|
|
switch l4t {
|
|
case 35:
|
|
return "jetpack5"
|
|
case 36:
|
|
return "jetpack6"
|
|
default:
|
|
// Newer Jetson systems use the SBSU runtime
|
|
slog.Debug("unrecognized L4T version", "nv_tegra_release", string(data))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|