package main import ( "bytes" "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" "time" ) // A fake machine: an Ollama store, a vLLM folder and any number of external disks, all in temp folders. type fakeBox struct { t *testing.T root string cfg nodeConfig plugged map[string]bool } func newBox(t *testing.T, diskIDs ...string) *fakeBox { t.Helper() root := t.TempDir() if err := os.Symlink(root, filepath.Join(t.TempDir(), "probe")); err != nil { t.Skip("symbolic links are not available here:", err) } if len(diskIDs) == 0 { diskIDs = []string{"d1"} } b := &fakeBox{t: t, root: root, plugged: map[string]bool{}} b.cfg = nodeConfig{ ModelsDir: filepath.Join(root, "models", "ollama"), VLLMDir: filepath.Join(root, "models", "vllm"), ExternalRoot: filepath.Join(root, "ext"), MinFreeGB: 0, ExternalRuntimes: []string{"ollama", "vllm"}, MainModels: []string{"starter:1b", "org/starter"}, } for _, d := range []string{b.cfg.ModelsDir + "/blobs", b.cfg.ModelsDir + "/manifests", b.cfg.VLLMDir} { os.MkdirAll(d, 0755) } for _, id := range diskIDs { b.cfg.ExternalDisks = append(b.cfg.ExternalDisks, extDiskCfg{ID: id, UUID: "uuid-" + id, Label: "Disk " + id}) os.MkdirAll(b.cfg.extStore(id), 0755) os.WriteFile(filepath.Join(b.cfg.extStore(id), extMarker), nil, 0644) b.plugged[id] = true } cfgOverride = &b.cfg runtimeLoaded = func(string) (map[string]int64, error) { return nil, fmt.Errorf("no runtime in tests") } extDeviceExists = func(uuid string) bool { return b.plugged[strings.TrimPrefix(uuid, "uuid-")] } extIsMount = func(dir string) bool { return b.plugged[filepath.Base(dir)] } b.rescan() t.Cleanup(func() { cfgOverride = nil runtimeLoaded = func(id string) (map[string]int64, error) { rt := runtimeByID(id) if rt == nil { return nil, fmt.Errorf("unknown runtime") } return rt.Loaded() } extDeviceExists = func(uuid string) bool { _, err := os.Stat("/dev/disk/by-uuid/" + uuid); return err == nil } extIsMount = isMountPoint setExternalDisks(nil) }) return b } // rescan is what the watcher does every few seconds. func (b *fakeBox) rescan() { setExternalDisks(probeAll(b.cfg)) } // unplug takes the disk away (its folder disappears, so links to it dangle); plug brings it back. func (b *fakeBox) unplug(id string) { b.plugged[id] = false os.Rename(b.cfg.extDiskDir(id), b.cfg.extDiskDir(id)+".gone") b.rescan() } func (b *fakeBox) plug(id string) { os.Rename(b.cfg.extDiskDir(id)+".gone", b.cfg.extDiskDir(id)) b.plugged[id] = true b.rescan() } // setFree pretends a disk has this much room. func (b *fakeBox) setFree(id string, bytes uint64) { var list []extDisk for _, d := range externalDisks() { if d.ID == id { d.Free = bytes } list = append(list, d) } setExternalDisks(list) } // ollamaModel creates a model "name:tag" with one blob per seed; the same seed gives the same (shared) blob. func (b *fakeBox) ollamaModel(name string, seeds ...string) []string { b.t.Helper() var digests []string var layers []map[string]any for _, seed := range seeds { data := bytes.Repeat([]byte(seed), 1000) hex := fmt.Sprintf("%064x", []byte(seed)) d := "sha256:" + hex os.WriteFile(ollamaBlobPath(b.cfg.ModelsDir, d), data, 0644) digests = append(digests, d) layers = append(layers, map[string]any{"digest": d, "size": len(data)}) } mp, err := ollamaManifestPath(b.cfg.ModelsDir, name) if err != nil { b.t.Fatal(err) } os.MkdirAll(filepath.Dir(mp), 0755) m, _ := json.Marshal(map[string]any{"config": layers[0], "layers": layers[1:]}) os.WriteFile(mp, m, 0644) return digests } func (b *fakeBox) vllmModel(name string, files map[string]string) { dir := filepath.Join(b.cfg.VLLMDir, name) os.MkdirAll(dir, 0755) os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0644) for f, c := range files { os.WriteFile(filepath.Join(dir, f), []byte(c), 0644) } } // copyOllamaToDisk puts a full second copy of a model on a disk (a disk that came from another machine). func (b *fakeBox) copyOllamaToDisk(name, id string, digests []string) { os.MkdirAll(filepath.Join(ollamaStore(b.cfg, id), "blobs"), 0755) for _, d := range digests { data, _ := os.ReadFile(ollamaBlobPath(b.cfg.ModelsDir, d)) os.WriteFile(ollamaExtBlob(b.cfg, id, d), data, 0644) } idx := loadOllamaIndex(b.cfg, id) idx.Models[name] = digests saveOllamaIndex(b.cfg, id, idx) } func noProgress(string, int64, int64) {} func isLink(p string) bool { fi, err := os.Lstat(p) return err == nil && fi.Mode()&os.ModeSymlink != 0 } func exists(p string) bool { _, err := os.Stat(p); return err == nil } func locOf(b *fakeBox, name string) string { loc, disk := ollamaLocation(b.cfg, name) if disk != "" { return loc + ":" + disk } return loc } func TestProbeDiskStates(t *testing.T) { root := t.TempDir() cfg := nodeConfig{ExternalRoot: root} c := extDiskCfg{ID: "d1", UUID: "u1"} os.MkdirAll(cfg.extStore("d1"), 0755) defer func(a func(string) bool, m func(string) bool) { extDeviceExists, extIsMount = a, m }(extDeviceExists, extIsMount) extDeviceExists, extIsMount = func(string) bool { return false }, func(string) bool { return false } if d := probeDisk(cfg, c); d.Present || d.Why != "not_plugged" || d.Label != "d1" { t.Fatalf("%+v", d) } extDeviceExists, extIsMount = func(string) bool { return true }, func(string) bool { return true } if d := probeDisk(cfg, c); d.Present || d.Why != "not_prepared" { t.Fatalf("plugged but no marker: %+v", d) } os.WriteFile(filepath.Join(cfg.extStore("d1"), extMarker), nil, 0644) if d := probeDisk(cfg, c); !d.Present { t.Fatalf("%+v", d) } // a marker in a plain folder (nothing mounted there) is not a disk extIsMount = func(string) bool { return false } if d := probeDisk(cfg, c); d.Present { t.Fatalf("an unmounted folder must not count: %+v", d) } if got := probeAll(nodeConfig{}); len(got) != 0 { t.Fatalf("no disks configured: %+v", got) } } func TestDiskOfPathKnowsOnlyConfiguredDisks(t *testing.T) { cfg := nodeConfig{ExternalRoot: "/mnt/x", ExternalDisks: []extDiskCfg{{ID: "d1"}, {ID: "d2"}}} for p, want := range map[string]string{"/mnt/x/d1/wxch-models/a": "d1", "/mnt/x/d2": "", "/mnt/x/d2/f": "d2", "/mnt/x/d3/f": "", "/mnt/models/f": "", "/mnt/x/d1x/f": ""} { if got := diskOfPath(cfg, p); got != want { t.Errorf("%s: got %q want %q", p, got, want) } } } func TestOllamaMoveToDiskAndBack(t *testing.T) { b := newBox(t) digests := b.ollamaModel("m1:1b", "config", "weights") if got := locOf(b, "m1:1b"); got != "internal" { t.Fatal(got) } if err := moveModel(b.cfg, "ollama", "m1:1b", "external", "", noProgress); err != nil { t.Fatal(err) } for _, d := range digests { if !isLink(ollamaBlobPath(b.cfg.ModelsDir, d)) || !exists(ollamaExtBlob(b.cfg, "d1", d)) { t.Fatalf("%s should be a link to a file on the disk", d) } } if got := locOf(b, "m1:1b"); got != "external:d1" { t.Fatal(got) } if idx := loadOllamaIndex(b.cfg, "d1"); len(idx.Models["m1:1b"]) != 2 { t.Fatalf("%+v", idx) } if got, _ := os.ReadFile(ollamaBlobPath(b.cfg.ModelsDir, digests[1])); !bytes.Equal(got, bytes.Repeat([]byte("weights"), 1000)) { t.Fatal("content changed (Ollama reads it through the link)") } if err := moveModel(b.cfg, "ollama", "m1:1b", "internal", "", noProgress); err != nil { t.Fatal(err) } for _, d := range digests { if isLink(ollamaBlobPath(b.cfg.ModelsDir, d)) || exists(ollamaExtBlob(b.cfg, "d1", d)) { t.Fatalf("%s should be a real file on the machine only", d) } } if got := locOf(b, "m1:1b"); got != "internal" || len(loadOllamaIndex(b.cfg, "d1").Models) != 0 { t.Fatal(got) } } func TestSeveralDisksTheBestOneWinsAndModelsMoveBetweenThem(t *testing.T) { b := newBox(t, "d1", "d2", "d3") digests := b.ollamaModel("m1:1b", "config", "weights") b.setFree("d1", 50e9) b.setFree("d2", 500e9) b.setFree("d3", 5) // no room at all if err := moveModel(b.cfg, "ollama", "m1:1b", "external", "", noProgress); err != nil { t.Fatal(err) } if got := locOf(b, "m1:1b"); got != "external:d2" { t.Fatalf("the disk with the most room should win: %s", got) } // asking again with no particular disk leaves it where it is if err := moveModel(b.cfg, "ollama", "m1:1b", "external", "", noProgress); err != nil || locOf(b, "m1:1b") != "external:d2" { t.Fatal("must stay on d2") } // moving it to a chosen disk copies it there and removes the old copy if err := moveModel(b.cfg, "ollama", "m1:1b", "external", "d1", noProgress); err != nil { t.Fatal(err) } if got := locOf(b, "m1:1b"); got != "external:d1" { t.Fatal(got) } for _, d := range digests { if exists(ollamaExtBlob(b.cfg, "d2", d)) { t.Fatal("the copy on d2 should be gone") } } if len(loadOllamaIndex(b.cfg, "d2").Models) != 0 || len(loadOllamaIndex(b.cfg, "d1").Models["m1:1b"]) != 2 { t.Fatal("indexes must follow the model") } // a disk that is not there cannot be chosen b.unplug("d3") if err := moveModel(b.cfg, "ollama", "m1:1b", "external", "d3", noProgress); err == nil || !strings.Contains(err.Error(), "not connected") { t.Fatalf("%v", err) } } func TestNoDiskHasRoomIsReportedAndNothingChanges(t *testing.T) { b := newBox(t, "d1", "d2") digests := b.ollamaModel("m1:1b", "config", "weights") b.setFree("d1", 100) b.setFree("d2", 100) err := moveModel(b.cfg, "ollama", "m1:1b", "external", "", noProgress) if err == nil || !strings.Contains(err.Error(), "not enough room") { t.Fatalf("%v", err) } for _, d := range digests { if isLink(ollamaBlobPath(b.cfg.ModelsDir, d)) { t.Fatal("nothing may have changed") } } } func TestSharedBlobsStayOnTheDiskWhileAnotherModelNeedsThem(t *testing.T) { b := newBox(t) shared := b.ollamaModel("a:1b", "base", "only-a") b.ollamaModel("b:1b", "base", "only-b") for _, n := range []string{"a:1b", "b:1b"} { if err := moveModel(b.cfg, "ollama", n, "external", "", noProgress); err != nil { t.Fatal(err) } } if err := moveModel(b.cfg, "ollama", "a:1b", "internal", "", noProgress); err != nil { t.Fatal(err) } if !exists(ollamaExtBlob(b.cfg, "d1", shared[0])) { t.Fatal("the shared blob must stay on the disk for model b") } if exists(ollamaExtBlob(b.cfg, "d1", shared[1])) { t.Fatal("a's own blob should have left the disk") } } func TestModelOnAnUnpluggedDiskIsHiddenAndComesBack(t *testing.T) { b := newBox(t, "d1", "d2") b.ollamaModel("m1:1b", "config", "weights") b.ollamaModel("keep:1b", "c2", "w2") if err := moveModel(b.cfg, "ollama", "m1:1b", "external", "d1", noProgress); err != nil { t.Fatal(err) } b.unplug("d1") if got := locOf(b, "m1:1b"); got != "offline:d1" { t.Fatal(got) } inst := ollamaDropOffline(b.cfg, map[string]int64{"m1:1b": 1, "keep:1b": 1}) if _, still := inst["m1:1b"]; still || len(inst) != 1 { t.Fatalf("%v", inst) } if err := moveModel(b.cfg, "ollama", "m1:1b", "internal", "", noProgress); err == nil { t.Fatal("moving a model whose disk is away must fail with a clear message") } b.plug("d1") if got := locOf(b, "m1:1b"); got != "external:d1" { t.Fatalf("back with the disk: %s", got) } } func TestOllamaDuplicatesAcrossMachineAndDisksAndTheirResolution(t *testing.T) { for _, keep := range []string{"internal", "d1", "d2"} { t.Run("keep "+keep, func(t *testing.T) { b := newBox(t, "d1", "d2") digests := b.ollamaModel("m1:1b", "config", "weights") b.copyOllamaToDisk("m1:1b", "d1", digests) b.copyOllamaToDisk("m1:1b", "d2", digests) d := storageDuplicates(b.cfg) if len(d) != 1 || d[0].Name != "m1:1b" || len(d[0].Places) != 3 || d[0].Size == 0 { t.Fatalf("%+v", d) } if err := ollamaDedupe(b.cfg, "m1:1b", keep); err != nil { t.Fatal(err) } if d := storageDuplicates(b.cfg); len(d) != 0 { t.Fatalf("still a duplicate: %+v", d) } want := map[string]string{"internal": "internal", "d1": "external:d1", "d2": "external:d2"}[keep] if got := locOf(b, "m1:1b"); got != want { t.Fatalf("location %s, want %s", got, want) } for _, id := range []string{"d1", "d2"} { if has := exists(ollamaExtBlob(b.cfg, id, digests[1])); has != (id == keep) { t.Fatalf("keep %s: copy on %s exists=%v", keep, id, has) } } if got, _ := os.ReadFile(ollamaBlobPath(b.cfg.ModelsDir, digests[1])); len(got) != 7000 { t.Fatal("the kept copy must be readable") } }) } } func TestAdoptingModelsFromEachDiskAfterAReinstall(t *testing.T) { b := newBox(t, "d1", "d2") a := b.ollamaModel("a:1b", "ca", "wa") c := b.ollamaModel("c:1b", "cc", "wc") moveModel(b.cfg, "ollama", "a:1b", "external", "d1", noProgress) moveModel(b.cfg, "ollama", "c:1b", "external", "d2", noProgress) os.RemoveAll(b.cfg.ModelsDir) // "reinstall": the machine's Ollama store is empty again os.MkdirAll(b.cfg.ModelsDir+"/blobs", 0755) for _, d := range presentDisks() { ollamaAdoptDisk(b.cfg, d) // Ollama is not running here, so only the links are made } for _, dg := range append(a, c...) { if !isLink(ollamaBlobPath(b.cfg.ModelsDir, dg)) || !exists(ollamaBlobPath(b.cfg.ModelsDir, dg)) { t.Fatalf("%s was not linked", dg) } } } func TestDeletingAnExternalModelRemovesItsDiskCopy(t *testing.T) { b := newBox(t) digests := b.ollamaModel("m1:1b", "config", "weights") blobs, _ := ollamaBlobs(b.cfg.ModelsDir, "m1:1b") moveModel(b.cfg, "ollama", "m1:1b", "external", "", noProgress) for _, d := range digests { // what Ollama's own delete does: it removes the blob names (the links) os.Remove(ollamaBlobPath(b.cfg.ModelsDir, d)) } ollamaForgetAfterDelete(b.cfg, "m1:1b", blobs) for _, d := range digests { if exists(ollamaExtBlob(b.cfg, "d1", d)) { t.Fatal("the copy on the disk should be removed") } } if idx := loadOllamaIndex(b.cfg, "d1"); len(idx.Models) != 0 { t.Fatalf("%+v", idx) } } func TestVLLMMoveBackAndForthAndBetweenDisks(t *testing.T) { b := newBox(t, "d1", "d2") b.vllmModel("org/model", map[string]string{"w.safetensors": strings.Repeat("x", 5000)}) in := filepath.Join(b.cfg.VLLMDir, "org", "model") if err := moveModel(b.cfg, "vllm", "org/model", "external", "d1", noProgress); err != nil { t.Fatal(err) } if loc, disk := vllmLocation(b.cfg, "org/model"); loc != "external" || disk != "d1" || !isLink(in) || !exists(filepath.Join(in, "config.json")) { t.Fatalf("%s %s", loc, disk) } inst, _ := newVLLMRuntime().Installed() if inst["org/model"] < 5000 { t.Fatalf("size must come through the link: %v", inst) } if err := moveModel(b.cfg, "vllm", "org/model", "external", "d2", noProgress); err != nil { t.Fatal(err) } if loc, disk := vllmLocation(b.cfg, "org/model"); loc != "external" || disk != "d2" { t.Fatalf("%s %s", loc, disk) } if exists(vllmDiskUnit(b.cfg, "d1", "org/model")) { t.Fatal("the copy on d1 should be gone") } if err := moveModel(b.cfg, "vllm", "org/model", "internal", "", noProgress); err != nil { t.Fatal(err) } if loc, _ := vllmLocation(b.cfg, "org/model"); loc != "internal" || isLink(in) || exists(vllmDiskUnit(b.cfg, "d2", "org/model")) { t.Fatal("should be a real folder on the machine only") } } func TestVLLMUnfinishedDownloadIsNotMoved(t *testing.T) { b := newBox(t) b.vllmModel("org/model", map[string]string{"w.safetensors.part": "half"}) if err := moveModel(b.cfg, "vllm", "org/model", "external", "", noProgress); err == nil { t.Fatal("a download that is not finished must not be moved") } } func TestVLLMDuplicatesAdoptAndDelete(t *testing.T) { b := newBox(t, "d1", "d2") b.vllmModel("org/dup", map[string]string{"w": "1"}) for _, id := range []string{"d1", "d2"} { u := vllmDiskUnit(b.cfg, id, "org/dup") os.MkdirAll(u, 0755) os.WriteFile(filepath.Join(u, "config.json"), []byte("{}"), 0644) os.WriteFile(filepath.Join(u, "w"), []byte("1"), 0644) } if d := vllmDuplicates(b.cfg); len(d) != 1 || d[0].Name != "org/dup" || len(d[0].Places) != 3 { t.Fatalf("%+v", d) } if err := vllmDedupe(b.cfg, "org/dup", "d2"); err != nil { t.Fatal(err) } if loc, disk := vllmLocation(b.cfg, "org/dup"); loc != "external" || disk != "d2" || len(vllmDuplicates(b.cfg)) != 0 || exists(vllmDiskUnit(b.cfg, "d1", "org/dup")) { t.Fatalf("expected one copy, on d2, as a link: %s %s", loc, disk) } // a model that only exists on a disk is linked in when the disk is connected only := vllmDiskUnit(b.cfg, "d1", "org/only") os.MkdirAll(only, 0755) os.WriteFile(filepath.Join(only, "config.json"), []byte("{}"), 0644) d1, _ := diskByID("d1") vllmAdoptDisk(b.cfg, d1) if loc, disk := vllmLocation(b.cfg, "org/only"); loc != "external" || disk != "d1" { t.Fatal("not adopted") } // deleting a model on a disk removes the link and the data if err := vllmRemoveUnit(b.cfg, filepath.Join(b.cfg.VLLMDir, "org", "only")); err != nil { t.Fatal(err) } if exists(only) { t.Fatal("the data on the disk should be gone") } // keep internal: the machine's copy stays, the disks' copies go b2 := newBox(t, "d1") b2.vllmModel("org/x", map[string]string{"w": "1"}) u := vllmDiskUnit(b2.cfg, "d1", "org/x") os.MkdirAll(u, 0755) os.WriteFile(filepath.Join(u, "config.json"), []byte("{}"), 0644) if err := vllmDedupe(b2.cfg, "org/x", "internal"); err != nil || exists(u) || isLink(filepath.Join(b2.cfg.VLLMDir, "org", "x")) { t.Fatalf("keep internal failed: %v", err) } } func TestMovingNeedsAConnectedDiskAndAnUnloadedModel(t *testing.T) { b := newBox(t) b.ollamaModel("m1:1b", "config", "weights") runtimeLoaded = func(string) (map[string]int64, error) { return map[string]int64{"m1:1b": 1}, nil } if err := moveModel(b.cfg, "ollama", "m1:1b", "external", "", noProgress); err == nil || !strings.Contains(err.Error(), "unload") { t.Fatalf("a loaded model must not be moved: %v", err) } runtimeLoaded = func(string) (map[string]int64, error) { return nil, nil } b.unplug("d1") if err := moveModel(b.cfg, "ollama", "m1:1b", "external", "", noProgress); err == nil || !strings.Contains(err.Error(), "no external disk") { t.Fatalf("%v", err) } } func TestPlacementPolicy(t *testing.T) { b := newBox(t, "d1", "d2") b.setFree("d1", 10e9) b.setFree("d2", 900e9) b.ollamaModel("plain:1b", "c1", "w1") b.ollamaModel("starter:1b", "c2", "w2") var said []string say := func(s string, _, _ int64) { said = append(said, s) } // an ordinary model goes to the disk with the most room, without a question if err := placeAfterPull("ollama", "plain:1b", "auto", say); err != nil { t.Fatal(err) } if got := locOf(b, "plain:1b"); got != "external:d2" { t.Fatal(got) } // a starter model asks, and with no answer it stays on the machine placeAskWait = 50 * time.Millisecond defer func() { placeAskWait = 20 * time.Second }() if err := placeAfterPull("ollama", "starter:1b", "auto", say); err != nil { t.Fatal(err) } if got := locOf(b, "starter:1b"); got != "internal" { t.Fatal("no answer must keep it on the machine: " + got) } // ...but the owner can answer in time placeAskWait = 5 * time.Second done := make(chan error, 1) go func() { done <- placeAfterPull("ollama", "starter:1b", "auto", say) }() var id string for i := 0; i < 100 && id == ""; i++ { if p := pendingPlacements(); len(p) == 1 { id = p[0].ID } time.Sleep(10 * time.Millisecond) } if id == "" { t.Fatal("the question never appeared") } if err := decidePlacement(id, "external"); err != nil { t.Fatal(err) } if err := <-done; err != nil { t.Fatal(err) } if got := locOf(b, "starter:1b"); got != "external:d2" { t.Fatal("the answer was ignored: " + got) } if err := decidePlacement(id, "external"); err == nil { t.Fatal("a question can only be answered once") } if err := decidePlacement("x", "sideways"); err == nil { t.Fatal("bad choices are refused") } // explicit modes, and no disk connected if err := placeAfterPull("ollama", "plain:1b", "internal", say); err != nil || locOf(b, "plain:1b") != "external:d2" { t.Fatal("internal means: do nothing") } b.unplug("d1") b.unplug("d2") if err := placeAfterPull("ollama", "plain:1b", "auto", say); err != nil { t.Fatalf("auto without a disk is not an error: %v", err) } if err := placeAfterPull("ollama", "plain:1b", "external", say); err == nil { t.Fatal("explicit external without a disk must fail") } if err := placeAfterPull("lmstudio", "x/y", "external", say); err == nil { t.Fatal("LM Studio is not supported yet") } }