Add Phase A bootstrap batching · Entire

Add Phase A bootstrap batching

2ed0a3f→main·

Soph·3mo ago·4 files·+500 added/-3 removed

Sessions

aebdecbeb417View transcript

Changes

4

140 unmodified lines

fs.BoolVar(&cfg.MeasureMemory, "measure-memory", false, "sample elapsed time and Go heap usage")
fs.BoolVar(&jsonOutput, "json", false, "print JSON output")
fs.Int64Var(&cfg.MaxPackBytes, "max-pack-bytes", 0, "abort bootstrap if the streamed source pack exceeds this many bytes")
fs.Int64Var(&cfg.BatchMaxPackBytes, "batch-max-pack-bytes", 0, "split branch bootstrap into relay batches capped at this many bytes per batch")
fs.StringVar(&cfg.ProtocolMode, "protocol", envOr("GITSYNC_PROTOCOL", "auto"), "protocol mode: auto, v1, or v2")
fs.BoolVar(&cfg.Verbose, "v", false, "verbose logging")

}

func usageError(message string) error {
    usage := "usage:\n  git-sync sync [flags] <source-url> <target-url>\n  git-sync plan [flags] <source-url> <target-url>\n  git-sync bootstrap [flags] <source-url> <target-url>\n  git-sync probe [flags] <source-url> [target-url]\n  git-sync fetch [flags] <source-url>\n\nsync/plan flags:\n  --branch main,dev\n  --map main:stable\n  --tags\n  --force\n  --prune\n  --stats\n  --measure-memory\n  --json\n  --protocol auto|v1|v2\n  --source-token ...\n  --target-token ...\n  --source-username git\n  --target-username git\n  --source-bearer-token ...\n  --target-bearer-token ...\n  -v\n\nbootstrap flags:\n  --branch main,dev\n  --map main:stable\n  --tags\n  --max-pack-bytes 104857600\n  --stats\n  --measure-memory\n  --json\n  --protocol auto|v1|v2\n  --source-token ...\n  --target-token ...\n  --source-username git\n  --target-username git\n  --source-bearer-token ...\n  --target-bearer-token ...\n  -v\n\nprobe flags:\n  --tags\n  --stats\n  --measure-memory\n  --json\n  --protocol auto|v1|v2\n  --source-token ...\n  --source-username git\n  --source-bearer-token ...\n  --target-token ...\n  --target-username git\n  --target-bearer-token ...\n\nfetch flags:\n  --branch main,dev\n  --tags\n  --stats\n  --measure-memory\n  --json\n  --protocol auto|v1|v2\n  --have-ref main\n  --have <hash>\n"
    if message == "" {
        return errors.New(strings.TrimSpace(usage))
    }
}

func TestBootstrap_GitHTTPBackendBatchedBranch(t *testing.T) {
    if os.Getenv(gitHTTPBackendEnv) == "" {
        t.Skip("set GITSYNC_E2E_GIT_HTTP_BACKEND=1 to run git-http-backend integration test")
    }

if _, err := exec.LookPath("git"); err != nil {
        t.Skipf("git not available: %v", err)
    }

root := t.TempDir()
    sourceBare := filepath.Join(root, "source.git")
    targetBare := filepath.Join(root, "target.git")
    worktree := filepath.Join(root, "work")

runGit(t, root, "init", "--bare", sourceBare)
    runGit(t, sourceBare, "config", "uploadpack.allowFilter", "true")
    runGit(t, sourceBare, "config", "uploadpack.allowReachableSHA1InWant", "true")
    runGit(t, root, "init", "--bare", targetBare)
    runGit(t, targetBare, "config", "http.receivepack", "true")
    runGit(t, root, "init", "-b", testBranch, worktree)
    runGit(t, worktree, "config", "user.name", "git-sync test")
    runGit(t, worktree, "config", "user.email", "git-sync@example.com")
    runGit(t, worktree, "remote", "add", "origin", sourceBare)

for i := range 6 {
        writePseudoRandomFile(t, filepath.Join(worktree, fmt.Sprintf("blob-%d.bin", i)), int64(200_000+i*17))
        runGit(t, worktree, "add", ".")
        runGit(t, worktree, "commit", "-m", fmt.Sprintf("commit-%d", i))
    }
    runGit(t, worktree, "push", "origin", "HEAD:refs/heads/"+testBranch)

server := newGitHTTPBackendServer(t, root)
    defer server.Close()

sourceURL := server.RepoURL("source.git")
    targetURL := server.RepoURL("target.git")

result, err := Bootstrap(context.Background(), Config{
        Source:            Endpoint{URL: sourceURL},
        Target:            Endpoint{URL: targetURL},
        BatchMaxPackBytes: 350_000,
    })
    if err != nil {
        t.Fatalf("batched bootstrap failed: %v\nbackend-stderr:\n%s", err, server.Stderr())
    }
    if result.Pushed != 1 || result.Blocked != 0 {
        t.Fatalf("unexpected batched bootstrap result: %+v", result)
    }
    if !result.Relay || result.RelayMode != "bootstrap-batch" || !result.Batching {
        t.Fatalf("expected batched bootstrap relay result, got %+v", result)
    }
    if result.BatchCount < 2 {
        t.Fatalf("expected multiple bootstrap batches, got %+v", result)
    }

assertGitRefEqual(t, sourceBare, targetBare, plumbing.NewBranchReferenceName(testBranch))
}

func assertGitRefAbsent(t *testing.T, repoPath string, ref plumbing.ReferenceName) {
    t.Helper()
    cmd := exec.Command("git", "show-ref", "--verify", "--quiet", ref.String())
    cmd.Dir = repoPath
    cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
    if err := cmd.Run(); err == nil {
        t.Fatalf("expected ref %s to be absent", ref)
    }
}

func writePseudoRandomFile(t *testing.T, path string, size int64) {
    t.Helper()
    rng := rand.New(rand.NewSource(size))
    buf := make([]byte, size)
    for i := range buf {
        buf[i] = byte(rng.Intn(256))
    }
    if err := os.WriteFile(path, buf, 0o644); err != nil {
        t.Fatalf("write %s: %v", path, err)
    }
}

func fetchSourceCommitGraphV2(
    ctx context.Context,
    repo *git.Repository,
    conn *transportConn,
    adv *v2CapabilityAdvertisement,
    ref desiredRef,
) error {
    if !fetchCapabilitySupports(adv, "filter") {
        return fmt.Errorf("source does not advertise fetch filter support")
    }

commandArgs := []string{
        "ofs-delta",
        "no-progress",
        "filter tree:0",
        "want " + ref.SourceHash.String(),
        "done",
    }
    conn.stats.addWantsHaves("source upload-pack", 1, 0)

body, err := encodeV2CommandRequest("fetch", v2RequestCapabilities(adv), commandArgs)
    if err != nil {
        return err
    }

reader, err := postRPCStreamWithPhase(ctx, conn, transport.UploadPackServiceName, body, true, "upload-pack fetch")
    if err != nil {
        return err
    }
    defer ioutil.CheckClose(reader, &err)

if err := storeV2FetchPack(repo, reader); err != nil {
        return err
    }
    return storeFetchedSourceRefs(repo, singleDesiredRef(ref.SourceRef, ref.TargetRef, ref.SourceHash))
}

// Further implementation omitted for conciseness.