Add batched planning backend sensitivity test · Entire

Add batched planning backend sensitivity test

b718e82→main·

Soph·3mo ago·2 files·+91 added/-0 removed

Sessions

671334abfa33View transcript

[?
Can you take a look at the go code (wasm) in /Users/soph/Work/entire/devenv/entire-io-worktree1 based a bit on that I wonder if something like this can be build:Codex·GPT-5.4·1 step](/content/gh/entireio/git-sync/session/019d6d29-8cf7-7fe3-adc9-8c3e4d9d5603#timeline-671334abfa33/index.html)

Changes

2

326 unmodified lines

327
328
329
330
331
332
333

326 unmodified lines

- Successful under-limit probe packs are now cached and reused during execution, which avoids a second fetch for selected checkpoints.
- Checkpoint selection now also keeps searching within known fit/too-large probe bounds so it can choose the largest fitting checkpoint once an upper bound is known, instead of stopping at the first fitting sample and leaving obvious headroom unused.
- The benchmark harness now also reports planned and actual batch counts across repeated runs, so future batching changes can be compared against a stable baseline instead of relying only on micro-tests.
- The git-http-backend suite now also includes a batch-limit sensitivity test so `#14` experiments can be run against a real smart-HTTP backend and validated by checkpoint-count behavior, not just by unit-level synthetic planners.
- The planner still relies on network probes for sizing, so this remains partial rather than fully solved.

### 15. Materialized fallback path does not scale to large repos

Mdocs/rewrite-issue-list.md+1

569 unmodified lines

570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665

569 unmodified lines

assertGitRefAbsent(t, targetBare, tempRef)
}

func TestBootstrap_GitHTTPBackendBatchedPlanningTracksBatchLimit(t *testing.T) {
    if os.Getenv(gitHTTPBackendEnv) == "" {
        t.Skip("set GITSYNC_E2E_GIT_HTTP_BACKEND=1 to run git-http-backend integration test")
    }

if _, err := exec.LookPath("git"); err != nil {
        t.Skipf("git not available: %v", err)
    }

root := t.TempDir()
    sourceBare := filepath.Join(root, "source.git")
    targetBare := filepath.Join(root, "target.git")
    worktree := filepath.Join(root, "work")

runGit(t, root, "init", "--bare", sourceBare)
    runGit(t, sourceBare, "config", "uploadpack.allowFilter", "true")
    runGit(t, sourceBare, "config", "uploadpack.allowReachableSHA1InWant", "true")
    runGit(t, root, "init", "--bare", targetBare)
    runGit(t, targetBare, "config", "http.receivepack", "true")
    runGit(t, root, "init", "-b", testBranch, worktree)
    runGit(t, worktree, "config", "user.name", "git-sync test")
    runGit(t, worktree, "config", "user.email", "git-sync@example.com")
    runGit(t, worktree, "remote", "add", "origin", sourceBare)

for i := range 8 {
        writePseudoRandomFile(t, filepath.Join(worktree, fmt.Sprintf("limit-%d.bin", i)), int64(180_000+i*29))
        runGit(t, worktree, "add", ".")
        runGit(t, worktree, "commit", "-m", fmt.Sprintf("limit-%d", i))
    }
    runGit(t, worktree, "push", "origin", "HEAD:refs/heads/"+testBranch)

server := newGitHTTPBackendServer(t, root)
    defer server.Close()

sourceURL := server.RepoURL("source.git")
    cfg := Config{
        Source:       Endpoint{URL: sourceURL},
        Target:       Endpoint{URL: server.RepoURL("target.git")},
        ProtocolMode: protocolModeAuto,
    }

stats := newStats(false)
    sourceConn, err := newConn(cfg.Source, "source", stats)
    if err != nil {
        t.Fatalf("create source transport: %v", err)
    }
    sourceRefs, sourceService, err := gitproto.ListSourceRefs(context.Background(), sourceConn, cfg.ProtocolMode, planner.RefPrefixes(cfg.Mappings, cfg.IncludeTags))
    if err != nil {
        t.Fatalf("list source refs: %v", err)
    }
    desired, _, err := planner.BuildDesiredRefs(gitproto.RefHashMap(sourceRefs), planner.PlanConfig{
        Branches: cfg.Branches, Mappings: cfg.Mappings, IncludeTags: cfg.IncludeTags,
        Force: cfg.Force, Prune: cfg.Prune,
    })
    if err != nil {
        t.Fatalf("build desired refs: %v", err)
    }
    ref := desired[plumbing.NewBranchReferenceName(testBranch)]

plan := func(limit int64) []plumbing.Hash {
        t.Helper()
        checkpoints, err := bstrap.PlanCheckpoints(context.Background(), bstrap.Params{
            SourceConn:   sourceConn,
            SourceService: sourceService,
            BatchMaxPack: limit,
            Verbose:      cfg.Verbose,
        }, ref)
        if err != nil {
            t.Fatalf("plan checkpoints with limit %d: %v", limit, err)
        }
        return checkpoints
    }

smaller := plan(250_000)
    larger := plan(450_000)

if len(smaller) < 2 {
        t.Fatalf("expected smaller limit to produce multiple checkpoints, got %d (%v)", len(smaller), smaller)
    }
    if len(larger) == 0 {
        t.Fatal("expected larger limit to produce at least one checkpoint")
    }
    if len(smaller) < len(larger) {
        t.Fatalf("expected smaller batch limit to require at least as many checkpoints as larger limit, got smaller=%d larger=%d", len(smaller), len(larger))
    }
    if smaller[len(smaller)-1] != ref.SourceHash || larger[len(larger)-1] != ref.SourceHash {
        t.Fatalf("expected final checkpoint to be branch tip, got smaller=%s larger=%s tip=%s", smaller[len(smaller)-1], larger[len(larger)-1], ref.SourceHash)
    }
}

func TestBootstrap_GitHTTPBackendBatchedBranchWithTags(t *testing.T) {
    if os.Getenv(gitHTTPBackendEnv) == "" {
        t.Skip("set GITSYNC_E2E_GIT_HTTP_BACKEND=1 to run git-http-backend integration test")
    }