mirror of
https://github.com/multica-ai/multica.git
synced 2026-07-26 12:35:35 +02:00
fix(repocache): repair promisor config on isolated checkouts of a partial cache (#5772)
The isolated-checkout path used by Linux Codex builds a task-local repository with `git clone --local` from the workspace's bare cache. That command has two properties that combine badly when the cache is a partial clone: it does not carry the promisor remote configuration across, and it does not treat an incomplete source object store as an error. The result is a checkout that exits 0 with every tracked file reported as deleted, so an agent starts work in what looks like a repository someone emptied. Swap origin to the real remote and restore `remote.origin.promisor` / `remote.origin.partialclonefilter` before the first checkout, so git can lazily fetch the blobs it needs. Do the same on the reuse path, where a workdir created against a complete cache can later be resumed against a partial one. No cache is created as a partial clone today, so this changes nothing for existing installs; it is a prerequisite for the on-demand clone mode in MUL-4983 and hardens a path that fails silently rather than loudly. Co-authored-by: Bohan-J <bohan@devv.ai> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
@@ -644,6 +644,14 @@ func (c *Cache) createOrUpdateIsolatedCheckout(barePath, repoURL, checkoutPath,
|
||||
if err := setIsolatedCheckoutOrigin(checkoutPath, repoURL); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Idempotent, and required for a workdir that was first created while
|
||||
// the cache was still a full clone: without it, a checkout backed by a
|
||||
// blobless cache resolves missing blobs to nothing instead of fetching.
|
||||
if isPartialClone(barePath) {
|
||||
if err := configurePromisorRemote(checkoutPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if err := syncIsolatedCheckoutRefs(barePath, checkoutPath, baseRef); err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -731,18 +739,31 @@ func createIsolatedCheckout(barePath, repoURL, checkoutPath, branchName, baseRef
|
||||
}
|
||||
}()
|
||||
|
||||
if out, err := runGitCombinedOutput("-C", checkoutPath, "checkout", "--detach", baseCommit); err != nil {
|
||||
return "", fmt.Errorf("git checkout --detach: %s: %w", strings.TrimSpace(string(out)), err)
|
||||
}
|
||||
// The origin swap has to happen before the first checkout when the cache
|
||||
// is a partial clone. `git clone --local` hardlinks the objects it can see
|
||||
// and does NOT inherit the promisor configuration, so a blobless cache
|
||||
// yields a checkout whose blobs are unreachable — and git reports that as
|
||||
// success with every file "deleted" rather than as an error. Pointing
|
||||
// origin at the real remote and restoring the promisor config first lets
|
||||
// the checkout below lazily fetch what it needs.
|
||||
if out, err := runGitCombinedOutput("-C", checkoutPath, "remote", "remove", isolatedCacheRemoteName); err != nil {
|
||||
return "", fmt.Errorf("remove cache remote: %s: %w", strings.TrimSpace(string(out)), err)
|
||||
}
|
||||
if err := deleteAllLocalBranches(checkoutPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if out, err := runGitCombinedOutput("-C", checkoutPath, "remote", "add", "origin", repoURL); err != nil {
|
||||
return "", fmt.Errorf("add origin remote: %s: %w", strings.TrimSpace(string(out)), err)
|
||||
}
|
||||
if isPartialClone(barePath) {
|
||||
if err := configurePromisorRemote(checkoutPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
if out, err := runGitCombinedOutput("-C", checkoutPath, "checkout", "--detach", baseCommit); err != nil {
|
||||
return "", fmt.Errorf("git checkout --detach: %s: %w", strings.TrimSpace(string(out)), err)
|
||||
}
|
||||
if err := deleteAllLocalBranches(checkoutPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := syncIsolatedCheckoutRefs(barePath, checkoutPath, baseRef); err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -779,6 +800,39 @@ func isIsolatedCheckout(path string) bool {
|
||||
return err == nil && strings.TrimSpace(string(out)) == isolatedCheckoutConfigValue
|
||||
}
|
||||
|
||||
// partialCloneFilter is the object filter a blobless partial clone is created
|
||||
// with, and the value that has to be restored on any repository that inherits
|
||||
// such a clone's incomplete object store.
|
||||
const partialCloneFilter = "blob:none"
|
||||
|
||||
// isPartialClone reports whether a repository was created as a partial clone,
|
||||
// i.e. whether git will lazily fetch missing objects from its promisor remote.
|
||||
func isPartialClone(repoPath string) bool {
|
||||
out, err := runGitOutputWithTimeout(30*time.Second, "-C", repoPath, "config", "--get", "remote.origin.promisor")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(string(out)) == "true"
|
||||
}
|
||||
|
||||
// configurePromisorRemote marks origin as the promisor remote for a repository
|
||||
// whose object store is incomplete, so git lazily fetches missing blobs from
|
||||
// the real remote instead of failing. It mirrors the two config keys
|
||||
// `git clone --filter=blob:none` writes; `git clone --local` does not copy
|
||||
// them across, which is why they have to be restored by hand.
|
||||
func configurePromisorRemote(repoPath string) error {
|
||||
settings := [][2]string{
|
||||
{"remote.origin.promisor", "true"},
|
||||
{"remote.origin.partialclonefilter", partialCloneFilter},
|
||||
}
|
||||
for _, kv := range settings {
|
||||
if out, err := runGitCombinedOutput("-C", repoPath, "config", kv[0], kv[1]); err != nil {
|
||||
return fmt.Errorf("set %s: %s: %w", kv[0], strings.TrimSpace(string(out)), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func setIsolatedCheckoutOrigin(path, repoURL string) error {
|
||||
out, err := runGitCombinedOutput("-C", path, "remote", "set-url", "origin", repoURL)
|
||||
if err != nil {
|
||||
|
||||
211
server/internal/daemon/repocache/partial_clone_test.go
Normal file
211
server/internal/daemon/repocache/partial_clone_test.go
Normal file
@@ -0,0 +1,211 @@
|
||||
package repocache
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// createFilterableTestRepo builds a source repository that can actually serve
|
||||
// partial clones, and returns the URL to clone it from.
|
||||
//
|
||||
// Two details matter. First, `git clone --filter` is silently ignored for a
|
||||
// plain filesystem path — git takes the local hardlink shortcut instead of
|
||||
// speaking the pack protocol — so the URL has to be file://. Second, the
|
||||
// server side must opt into filtering via uploadpack.allowFilter, exactly as
|
||||
// GitHub and GitLab do.
|
||||
//
|
||||
// The commits carry real file contents, because a blobless clone only differs
|
||||
// from a full one in whether those blobs were transferred.
|
||||
func createFilterableTestRepo(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
run := func(args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Env = append(os.Environ(),
|
||||
"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@test.com",
|
||||
"GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@test.com",
|
||||
)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Skipf("git setup failed: %s: %v", out, err)
|
||||
}
|
||||
}
|
||||
run("init", dir)
|
||||
run("-C", dir, "config", "uploadpack.allowFilter", "true")
|
||||
for _, name := range partialCloneTestFiles {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte("contents of "+name+"\n"), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
run("-C", dir, "add", name)
|
||||
run("-C", dir, "commit", "-m", "add "+name)
|
||||
}
|
||||
return "file://" + dir
|
||||
}
|
||||
|
||||
var partialCloneTestFiles = []string{"a.txt", "b.txt"}
|
||||
|
||||
// seedBloblessCache creates the workspace's bare cache for sourceRepo as a
|
||||
// blobless partial clone, using the same two steps Cache.Sync performs for a
|
||||
// cold miss. Returns the bare repo path.
|
||||
func seedBloblessCache(t *testing.T, cache *Cache, workspaceID, sourceRepo string) string {
|
||||
t.Helper()
|
||||
barePath := filepath.Join(cache.root, workspaceID, bareDirName(sourceRepo))
|
||||
if err := os.MkdirAll(filepath.Dir(barePath), 0o755); err != nil {
|
||||
t.Fatalf("create workspace cache dir: %v", err)
|
||||
}
|
||||
if out, err := runGitCombinedOutput("clone", "--bare", "--filter=blob:none", sourceRepo, barePath); err != nil {
|
||||
t.Fatalf("seed blobless cache: %s: %v", out, err)
|
||||
}
|
||||
if err := ensureRemoteTrackingLayout(barePath); err != nil {
|
||||
t.Fatalf("ensure refspec: %v", err)
|
||||
}
|
||||
if !isPartialClone(barePath) {
|
||||
t.Fatal("seeded cache is not a partial clone; the filter was ignored")
|
||||
}
|
||||
return barePath
|
||||
}
|
||||
|
||||
// assertCheckoutIsComplete fails if the working tree is missing any tracked
|
||||
// file. `git status --porcelain` reports missing blobs as deletions, which is
|
||||
// how a silently-broken partial checkout shows up.
|
||||
func assertCheckoutIsComplete(t *testing.T, checkoutPath string) {
|
||||
t.Helper()
|
||||
out, err := runGitOutput("-C", checkoutPath, "status", "--porcelain")
|
||||
if err != nil {
|
||||
t.Fatalf("git status: %v", err)
|
||||
}
|
||||
if status := strings.TrimSpace(string(out)); status != "" {
|
||||
t.Fatalf("checkout is not clean, tracked files are missing:\n%s", status)
|
||||
}
|
||||
for _, name := range partialCloneTestFiles {
|
||||
data, err := os.ReadFile(filepath.Join(checkoutPath, name))
|
||||
if err != nil {
|
||||
t.Fatalf("read %s from checkout: %v", name, err)
|
||||
}
|
||||
if got, want := string(data), "contents of "+name+"\n"; got != want {
|
||||
t.Fatalf("%s = %q, want %q", name, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The isolated-checkout path (Linux Codex) clones the cache with
|
||||
// `git clone --local`. That command neither inherits a promisor remote's
|
||||
// configuration nor refuses to run against an incomplete object store: it
|
||||
// exits 0 and leaves every tracked file missing from the working tree. Without
|
||||
// the promisor config restored on the new checkout, an agent starts work in
|
||||
// what looks like a repository where someone deleted all the files.
|
||||
func TestCreateIsolatedCheckoutFromPartialCacheHasFileContents(t *testing.T) {
|
||||
t.Parallel()
|
||||
sourceRepo := createFilterableTestRepo(t)
|
||||
cache := New(t.TempDir(), testLogger())
|
||||
seedBloblessCache(t, cache, "ws-1", sourceRepo)
|
||||
|
||||
result, err := cache.CreateWorktree(WorktreeParams{
|
||||
WorkspaceID: "ws-1",
|
||||
RepoURL: sourceRepo,
|
||||
WorkDir: t.TempDir(),
|
||||
AgentName: "Linux Codex",
|
||||
TaskID: "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
|
||||
IsolatedGitMetadata: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateWorktree failed: %v", err)
|
||||
}
|
||||
assertCheckoutIsComplete(t, result.Path)
|
||||
|
||||
// The checkout must keep resolving blobs on its own once the cache is out
|
||||
// of the picture, so origin stays the real remote and carries the
|
||||
// promisor marker.
|
||||
origin, err := runGitOutput("-C", result.Path, "remote", "get-url", "origin")
|
||||
if err != nil {
|
||||
t.Fatalf("get origin URL: %v", err)
|
||||
}
|
||||
if got := strings.TrimSpace(string(origin)); got != sourceRepo {
|
||||
t.Fatalf("origin URL = %q, want %q", got, sourceRepo)
|
||||
}
|
||||
if !isPartialClone(result.Path) {
|
||||
t.Fatal("isolated checkout of a partial cache must be marked as a partial clone")
|
||||
}
|
||||
}
|
||||
|
||||
// A workdir created while the cache was still complete, then reused after the
|
||||
// cache became partial, must be repaired on reuse rather than left
|
||||
// half-populated.
|
||||
func TestReusedIsolatedCheckoutRepairsPromisorConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
sourceRepo := createFilterableTestRepo(t)
|
||||
cache := New(t.TempDir(), testLogger())
|
||||
seedBloblessCache(t, cache, "ws-1", sourceRepo)
|
||||
|
||||
params := WorktreeParams{
|
||||
WorkspaceID: "ws-1",
|
||||
RepoURL: sourceRepo,
|
||||
WorkDir: t.TempDir(),
|
||||
AgentName: "Linux Codex",
|
||||
TaskID: "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
|
||||
IsolatedGitMetadata: true,
|
||||
}
|
||||
first, err := cache.CreateWorktree(params)
|
||||
if err != nil {
|
||||
t.Fatalf("first CreateWorktree failed: %v", err)
|
||||
}
|
||||
if err := runGit("-C", first.Path, "config", "--unset", "remote.origin.promisor"); err != nil {
|
||||
t.Fatalf("unset promisor: %v", err)
|
||||
}
|
||||
|
||||
second, err := cache.CreateWorktree(params)
|
||||
if err != nil {
|
||||
t.Fatalf("second CreateWorktree failed: %v", err)
|
||||
}
|
||||
if !isPartialClone(second.Path) {
|
||||
t.Fatal("reused isolated checkout must have its promisor config restored")
|
||||
}
|
||||
assertCheckoutIsComplete(t, second.Path)
|
||||
}
|
||||
|
||||
// The linked-worktree path used by every other runtime shares the cache's own
|
||||
// object store and config, so it lazily fetches without any extra wiring. This
|
||||
// pins that difference so the promisor handling is not "fixed" by pushing it
|
||||
// into the shared cache later.
|
||||
func TestCreateWorktreeFromPartialCacheHasFileContents(t *testing.T) {
|
||||
t.Parallel()
|
||||
sourceRepo := createFilterableTestRepo(t)
|
||||
cache := New(t.TempDir(), testLogger())
|
||||
seedBloblessCache(t, cache, "ws-1", sourceRepo)
|
||||
|
||||
result, err := cache.CreateWorktree(WorktreeParams{
|
||||
WorkspaceID: "ws-1",
|
||||
RepoURL: sourceRepo,
|
||||
WorkDir: t.TempDir(),
|
||||
AgentName: "tester",
|
||||
TaskID: "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateWorktree failed: %v", err)
|
||||
}
|
||||
assertCheckoutIsComplete(t, result.Path)
|
||||
}
|
||||
|
||||
func TestIsPartialClone(t *testing.T) {
|
||||
t.Parallel()
|
||||
sourceRepo := createFilterableTestRepo(t)
|
||||
cache := New(t.TempDir(), testLogger())
|
||||
|
||||
if err := cache.Sync("ws-1", []RepoInfo{{URL: sourceRepo}}); err != nil {
|
||||
t.Fatalf("sync failed: %v", err)
|
||||
}
|
||||
if isPartialClone(cache.Lookup("ws-1", sourceRepo)) {
|
||||
t.Fatal("an ordinary full cache must not be reported as a partial clone")
|
||||
}
|
||||
|
||||
blobless := seedBloblessCache(t, New(t.TempDir(), testLogger()), "ws-2", sourceRepo)
|
||||
if !isPartialClone(blobless) {
|
||||
t.Fatal("a blobless cache must be reported as a partial clone")
|
||||
}
|
||||
if isPartialClone(filepath.Join(t.TempDir(), "does-not-exist")) {
|
||||
t.Fatal("a missing repository must not be reported as a partial clone")
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@
|
||||
- `repo checkout` requires `MULTICA_DAEMON_PORT`, sends `workspace_id`, `workdir`, `ref`, `agent_name`, `task_id`, and the daemon-managed optional `checkout_mode` to local daemon `/repo/checkout`, then prints the checked-out path.
|
||||
- `server/internal/daemon/health.go` resolves the checkout ref: request `ref` wins; otherwise it asks `server/internal/daemon/daemon.go` for the current task's project repo default ref. It forwards the validated isolated-checkout mode into `repocache.WorktreeParams`.
|
||||
- `server/internal/daemon/daemon.go` injects `MULTICA_REPO_CHECKOUT_MODE=isolated` only for Linux Codex tasks. `server/internal/daemon/repocache/cache.go` implements that mode as a same-filesystem local clone with task-local Git metadata and the real repository as `origin`; other runtimes keep the linked-worktree path.
|
||||
- When the bare cache is a partial clone, that isolated checkout must have `remote.origin.promisor` / `partialclonefilter` restored before its first `checkout`: `git clone --local` neither inherits them nor errors on the missing objects it leaves behind, so the checkout would otherwise succeed with an empty working tree. The linked-worktree path shares the cache's own config and needs no such repair.
|
||||
- `server/cmd/server/router.go` registers daemon APIs under `/api/daemon`, including workspace repos and task claim.
|
||||
- `server/internal/daemon/daemon.go` claims tasks, prepares workdirs, launches provider CLIs, and reports completion.
|
||||
- `server/internal/daemon/execenv/runtime_config.go` injects task/project/repo context into agent workdirs.
|
||||
|
||||
Reference in New Issue
Block a user