Rename --batch-max-pack-bytes to --target-max-pack-bytes · Entire
Rename --batch-max-pack-bytes to --target-max-pack-bytes
e43f5be→main· Soph·3mo ago·16 files·+66 added/-66 removed
With estimate-based planning, PACK header precheck, and auto-subdivide, there's no longer a reason for the user to think in terms of "batch sizes." The single meaningful knob is the target's receive-pack body limit. The system estimates checkpoint placement from it, prechecks PACK headers against it, and auto-subdivides when batches exceed it.
Rename across CLI flags, internal types, unstable API, docs, and tests: --batch-max-pack-bytes → --target-max-pack-bytes BatchMaxPackBytes → TargetMaxPackBytes BatchMaxPack → TargetMaxPack defaultAutoBatchMaxPackBytes → defaultTargetMaxPackBytes autoBatchMaxPackBytes → autoTargetMaxPackBytes
Pure rename, no behavioral changes. All tests pass.
Co-Authored-By: Claude Opus 4.6 (1M context) noreply@anthropic.com
Sessions
fb6ddecdebbfView transcript
?\ what test coverage do we have now for replicate?Claude Code·Opus 4.6[1m]·1 step
Changes
16
MCHANGELOG.md+2/-2
MREADME.md+6/-6
cmd
git-sync-bench
Mmain.go+2/-2
git-sync
Mmain.go+3/-3
docs
Mbenchmarking.md+1/-1
Mbootstrap-batching.md+7/-7
Mbootstrap.md+1/-1
Mtesting.md+1/-1
internal
strategy/bootstrap
Mbootstrap.go+18/-18
Mbootstrap_test.go+3/-3
syncer
Mentire_local_smoke_test.go+2/-2
Mgit_http_backend_test.go+5/-5
Mintegration_test.go+9/-9
Mlive_bootstrap_test.go+1/-1
Msyncer.go+2/-2
pkg/gitsync/unstable
Mclient.go+3/-3
16 unmodified lines
17
18
19
20
20
21
22
23
24
25
26
26
27
28
29
16 unmodified lines
- `gitsync.Client.Replicate` on the stable embedding surface.
- `gitsync.OperationMode`, `gitsync.ModeSync`, `gitsync.ModeReplicate`, and
`SyncPolicy.Mode` for selecting the mode from library callers.
- `--max-pack-bytes` and `--batch-max-pack-bytes` flags on `sync`,
- `--max-pack-bytes` and `--target-max-pack-bytes` flags on `sync`,
`replicate`, and `plan`. Previously only `bootstrap` exposed them, but
replicate's bootstrap-fallback path internally honors both — without
these flags, users couldn't split a huge initial replicate push into
tractable receive-pack POSTs for size-limited targets. The unstable
library `buildSyncConfig` now forwards `MaxPackBytes` and
`BatchMaxPackBytes` from `AdvancedOptions` to `syncer.Config`.
`TargetMaxPackBytes` from `AdvancedOptions` to `syncer.Config`.
### Changed (stable API, breaking)
MCHANGELOG.md+2/-2
171 unmodified lines
172
173
174
175
175
176
177
178
179
179
180
181
182
10 unmodified lines
193
194
195
196
197
196
197
198
199
200
201
202
203
203
204
205
206
21 unmodified lines
228
229
230
231
231
232
233
234
171 unmodified lines
<target-url>
Add `--batch-max-pack-bytes` to split large branch bootstraps into multiple relay batches with temporary refs:
Add `--target-max-pack-bytes` to split large branch bootstraps into multiple relay batches with temporary refs:
```bash
go run ./cmd/git-sync bootstrap \
--batch-max-pack-bytes 1073741824 \
--target-max-pack-bytes 1073741824 \
<source-url> \
<target-url>
```
A practical starting point is:
- `--batch-max-pack-bytes 536870912` for a conservative `512 MiB` target-side batch size
- `--batch-max-pack-bytes 1073741824` when you want fewer, larger batches and the target has more headroom
- `--target-max-pack-bytes 536870912` for a conservative `512 MiB` target-side batch size
- `--target-max-pack-bytes 1073741824` when you want fewer, larger batches and the target has more headroom
For example:
```bash
go run ./cmd/git-sync bootstrap \
--batch-max-pack-bytes 536870912 \
--target-max-pack-bytes 536870912 \
--protocol v2 \
-v \
<source-url> \
--scenario bootstrap \
--source-url /tmp/git-sync-bench/kubernetes.git \
--repeat 3 \
--batch-max-pack-bytes 104857600 \
--target-max-pack-bytes 104857600 \
--stats \
--json
```
MREADME.md+6/-6
110 unmodified lines
111
112
113
114
114
115
116
117
324 unmodified lines
442
443
444
445
445
446
447
448
110 unmodified lines
fs.BoolVar(&cfg.Options.CollectStats, "stats", false, "collect transfer statistics")
fs.BoolVar(&cfg.Options.MeasureMemory, "measure-memory", true, "sample elapsed time and Go heap usage")
fs.Int64Var(&cfg.Options.MaxPackBytes, "max-pack-bytes", 0, "abort bootstrap if the streamed source pack exceeds this many bytes")
fs.Int64Var(&cfg.Options.BatchMaxPackBytes, "batch-max-pack-bytes", 0, "split branch bootstrap into relay batches capped at this many bytes per batch")
fs.Int64Var(&cfg.Options.TargetMaxPackBytes, "target-max-pack-bytes", 0, "target receive-pack body size limit; batches are planned and auto-subdivided to fit")
benchProtocol := benchProtocolModeFlag(benchProtocolMode(validation.ProtocolAuto))
fs.Var(&benchProtocol, "protocol", "protocol mode: auto, v1, or v2")
fs.BoolVar(&cfg.Options.Verbose, "v", false, "verbose logging")
324 unmodified lines
}
func usageError(message string) error {
usage := "usage:\n git-sync-bench --source-url <repo> [flags]\n\nflags:\n --scenario bootstrap|sync\n --repeat 3\n --work-dir /tmp/git-sync-bench\n --keep-targets\n --json\n --branch main,release\n --map main:stable\n --tags\n --force\n --prune\n --stats\n --measure-memory\n --max-pack-bytes 104857600\n --batch-max-pack-bytes 104857600\n --protocol auto|v1|v2\n -v\n"
usage := "usage:\n git-sync-bench --source-url <repo> [flags]\n\nflags:\n --scenario bootstrap|sync\n --repeat 3\n --work-dir /tmp/git-sync-bench\n --keep-targets\n --json\n --branch main,release\n --map main:stable\n --tags\n --force\n --prune\n --stats\n --measure-memory\n --max-pack-bytes 104857600\n --target-max-pack-bytes 104857600\n --protocol auto|v1|v2\n -v\n"
if message == "" {
return errors.New(strings.TrimSpace(usage))
}
}
Mcmd/git-sync-bench/main.go+2/-2
83 unmodified lines
84 85 86 87 87 88 89 90 90 unmodified lines
181 182 183 184 184 185 186 187 221 unmodified lines
409 410 411 412 412 413 414 415
83 unmodified lines
fs.BoolVar(&jsonOutput, "json", false, "print JSON output") fs.IntVar(&req.Options.MaterializedMaxObjects, "materialized-max-objects", unstable.DefaultMaterializedMaxObjects, "abort non-relay materialized syncs above this many objects") fs.Int64Var(&req.Options.MaxPackBytes, "max-pack-bytes", 0, "abort bootstrap-relay push if the streamed source pack exceeds this many bytes") fs.Int64Var(&req.Options.BatchMaxPackBytes, "batch-max-pack-bytes", 0, "split bootstrap-relay push into batches capped at this many bytes per batch") fs.Int64Var(&req.Options.TargetMaxPackBytes, "target-max-pack-bytes", 0, "target receive-pack body size limit; batches are planned and auto-subdivided to fit") protocolValue := protocolModeFlag(protocolMode(envOr("GITSYNC_PROTOCOL", validation.ProtocolAuto))) fs.Var(&protocolValue, "protocol", "protocol mode: auto, v1, or v2") fs.BoolVar(&req.Options.Verbose, "v", false, "verbose logging") 90 unmodified lines
fs.BoolVar(&req.Options.MeasureMemory, "measure-memory", false, "sample elapsed time and Go heap usage") fs.BoolVar(&jsonOutput, "json", false, "print JSON output") fs.Int64Var(&req.Options.MaxPackBytes, "max-pack-bytes", 0, "abort bootstrap if the streamed source pack exceeds this many bytes") fs.Int64Var(&req.Options.BatchMaxPackBytes, "batch-max-pack-bytes", 0, "split branch bootstrap into relay batches capped at this many bytes per batch") fs.Int64Var(&req.Options.TargetMaxPackBytes, "target-max-pack-bytes", 0, "target receive-pack body size limit; batches are planned and auto-subdivided to fit") bootstrapProtocol := protocolModeFlag(protocolMode(envOr("GITSYNC_PROTOCOL", validation.ProtocolAuto))) fs.Var(&bootstrapProtocol, "protocol", "protocol mode: auto, v1, or v2") fs.BoolVar(&req.Options.Verbose, "v", false, "verbose logging") 221 unmodified lines
}
func usageError(message string) error {
usage := fmt.Sprintf("usage:\n git-sync sync [flags]
Mcmd/git-sync/main.go+3/-3
23 unmodified lines
24
25
26
27
27
28
29
30
23 unmodified lines
--scenario bootstrap \
--source-url /tmp/git-sync-bench/kubernetes.git \
--repeat 3 \
--batch-max-pack-bytes 104857600 \
--target-max-pack-bytes 104857600 \
--stats \
--json
Mdocs/benchmarking.md+1/-1
59 unmodified lines
60
61
62
63
63
64
65
66
67
68
69
70
70
71
72
73
74
74
75
76
77
28 unmodified lines
106
107
108
109
109
110
111
112
2 unmodified lines
115
116
117
118
118
119
120
121
138 unmodified lines
260
261
262
263
263
264
265
266
3 unmodified lines
270
271
272
273
273
274
275
276
59 unmodified lines
```bash
git-sync bootstrap \
--batch-max-pack-bytes 1073741824 \
--target-max-pack-bytes 1073741824 \
<source-url> \
<target-url>
```
Possible related flags:
- `--batch-max-pack-bytes`
- `--target-max-pack-bytes`
- `--batch-ref-prefix refs/gitsync/bootstrap/`
- `--keep-temp-refs-on-failure`
The first version should only need `--batch-max-pack-bytes`.
The first version should only need `--target-max-pack-bytes`.
## Temporary Ref Strategy
28 unmodified lines
1. Fetch the commit graph (tree:0 filter, one round-trip — commits only, no blobs/trees).
2. Walk first-parent ancestry backward to get the chain length.
3. Estimate total pack size: `chainLen × 8 KiB/commit`.
4. Compute number of batches: `ceil(estimated / --batch-max-pack-bytes)`.
4. Compute number of batches: `ceil(estimated / --target-max-pack-bytes)`.
5. Place checkpoints evenly along the first-parent chain.
This is a heuristic — real bytes-per-commit varies widely (2–100+ KiB depending on blob churn). The estimate intentionally errs toward more batches.
2 unmodified lines
If the estimate is too optimistic (fewer batches than needed), two safeguards catch it:
1. **PACK header pre-check**: after starting a fetch, peek at the first 12 bytes of the pack to read the object count. Multiply by ~750 bytes/object. If the estimate exceeds `--batch-max-pack-bytes`, abort the fetch (12 bytes wasted, not gigabytes), insert a midpoint checkpoint, and retry. This avoids a full transfer for obviously-oversized batches.
1. **PACK header pre-check**: after starting a fetch, peek at the first 12 bytes of the pack to read the object count. Multiply by ~750 bytes/object. If the estimate exceeds `--target-max-pack-bytes`, abort the fetch (12 bytes wasted, not gigabytes), insert a midpoint checkpoint, and retry. This avoids a full transfer for obviously-oversized batches.
2. **Target rejection retry**: if the target's receive-pack rejects a push for exceeding its body-size limit, detect the error, insert a midpoint checkpoint from the stored chain, and retry. This catches cases where the PACK header estimate was close but the real pack was slightly over.
138 unmodified lines
Progress:
- implemented via `git-sync bootstrap --batch-max-pack-bytes`
- implemented via `git-sync bootstrap --target-max-pack-bytes`
- currently requires source-side protocol v2 with fetch filter support
- resumes from an existing temp ref when that temp ref matches a planned checkpoint
- exercised by `TestBootstrap_GitHTTPBackendBatchedBranch`
3 unmodified lines
- prefer plain `bootstrap` first
- use batching when a single large bootstrap push is too risky, too large, or fails on the target side
- start with `--batch-max-pack-bytes 536870912` and adjust upward only if the target has enough headroom
- start with `--target-max-pack-bytes 536870912` and adjust upward only if the target has enough headroom
Phase B:
Mdocs/bootstrap-batching.md+7/-7
151 unmodified lines
152
153
154
155
155
156
157
158
151 unmodified lines
- explicit mapped refs are supported
- `--max-pack-bytes` provides a first safety threshold for the streamed source pack during bootstrap
- `--batch-max-pack-bytes` now enables a Phase A batched branch-only bootstrap mode for large initial syncs
- `--target-max-pack-bytes` now enables a Phase A batched branch-only bootstrap mode for large initial syncs
Phase 3:
```
Mdocs/bootstrap.md+1/-1
29 unmodified lines
30 31 32 33 33 34 35 36
29 unmodified lines
env GOCACHE=/tmp/go-build GITSYNC_E2E_GIT_HTTP_BACKEND=1 go test ./internal/syncer -run TestBootstrap_GitHTTPBackendBatchedPlanningTracksBatchLimit -v
That test uses a real `git-http-backend` source/target pair and checks that a smaller `--batch-max-pack-bytes` planning limit produces at least as many planned checkpoints as a larger one, while still planning to the branch tip.
That test uses a real `git-http-backend` source/target pair and checks that a smaller `--target-max-pack-bytes` planning limit produces at least as many planned checkpoints as a larger one, while still planning to the branch tip.
## Live Linux Smokes
Mdocs/testing.md+1/-1
27 unmodified lines
28
29
30
31
31
32
33
34
17 unmodified lines
52
53
54
55
55
56
57
58
24 unmodified lines
83
84
85
86
86
87
88
88
89
90
91
92
92
93
94
95
5 unmodified lines
101
102
103
104
104
105
106
107
15 unmodified lines
123
124
125
126
126
127
128
129
130
131
132
131
132
133
134
135
64 unmodified lines
200
201
202
203
203
204
205
206
53 unmodified lines
260
261
262
263
263
264
265
265
266
267
268
142 unmodified lines
411
412
413
414
414
415
416
417
136 unmodified lines
554
555
556
557
557
558
559
560
3 unmodified lines
564
565
566
567
567
568
569
570
54 unmodified lines
625
626
627
628
629
628
629
630
631
632
633
634
635
635
636
637
638
27 unmodified lines
)
const (
defaultAutoBatchMaxPackBytes = 512 * 1024 * 1024
defaultTargetMaxPackBytes = 512 * 1024 * 1024
githubLargeRepoThresholdKB = 1536 * 1024
)
17 unmodified lines
DesiredRefs map[plumbing.ReferenceName]planner.DesiredRef
TargetRefs map[plumbing.ReferenceName]plumbing.Hash
MaxPackBytes int64
BatchMaxPack int64
TargetMaxPack int64
Verbose bool
Logger *slog.Logger
}
24 unmodified lines
// GitHub large-repo preflight
if batchLimit, ok := githubBatchLimit(ctx, p); ok {
p.BatchMaxPack = batchLimit
p.TargetMaxPack = batchLimit
p.log("bootstrap github preflight selected batched mode",
"batch_max_pack_bytes", p.BatchMaxPack)
"target_max_pack_bytes", p.TargetMaxPack)
}
planTargetRefs := p.TargetRefs
if p.BatchMaxPack > 0 {
if p.TargetMaxPack > 0 {
planTargetRefs = adjustedBootstrapTargetRefs(p.DesiredRefs, p.TargetRefs)
}
plans, err := planner.BuildBootstrapPlans(p.DesiredRefs, planTargetRefs)
5 unmodified lines
Plans: plans, Relay: true, RelayMode: "bootstrap", RelayReason: relayReason,
}
if p.BatchMaxPack > 0 {
if p.TargetMaxPack > 0 {
return executeBatched(ctx, p, plans, result)
}
15 unmodified lines
pushErr := p.TargetPusher.PushPack(ctx, cmds, packReader)
_ = packReader.Close()
if pushErr != nil {
autoBatch, ok := autoBatchMaxPackBytes(p, pushErr)
autoBatch, ok := autoTargetMaxPackBytes(p, pushErr)
if !ok {
return result, fmt.Errorf("push target refs: %w", pushErr)
}
p.log("bootstrap retrying with batched mode after target rejection",
"batch_max_pack_bytes", autoBatch)
"target_max_pack_bytes", autoBatch)
p.TargetMaxPack = autoBatch
return executeBatched(ctx, p, plans, result)
}
64 unmodified lines
}
// MaxPackBytes is the hard abort threshold for any single source fetch.
// BatchMaxPack controls checkpoint *placement* (how many batches) but
// TargetMaxPack controls checkpoint *placement* (how many batches) but
// should not cap individual fetches — the estimate may undercount, and
// the actual pack for a batch can legitimately exceed the planning
// heuristic. If the resulting pack is too large for the target's
53 unmodified lines
// If the estimated pack size exceeds the batch limit, subdivide
// immediately instead of pushing a pack the target will reject.
// This avoids wasting a multi-GiB transfer on a doomed push.
if p.BatchMaxPack > 0 && len(batch.chain) > 0 {
if p.TargetMaxPack > 0 && len(batch.chain) > 0 {
subdivided := false
packReader, err = checkPackSizeAndSubdivide(packReader, p.BatchMaxPack, func() bool {
packReader, err = checkPackSizeAndSubdivide(packReader, p.TargetMaxPack, func() bool {
expanded := subdivideCheckpoints(batch.chain, current, batch.Checkpoints[idx:])
if len(expanded) > len(batch.Checkpoints[idx:]) {
p.log("bootstrap batch subdividing before push (pack header estimate)",
142 unmodified lines
return nil, nil, fmt.Errorf("empty first-parent chain for %s", ref.TargetRef)
}
numBatches := estimateBatchCount(int64(len(chain)), p.BatchMaxPack)
numBatches := estimateBatchCount(int64(len(chain)), p.TargetMaxPack)
checkpoints := evenCheckpoints(chain, numBatches)
p.log("bootstrap batch planned checkpoints",
136 unmodified lines
// --- GitHub preflight ---
func githubBatchLimit(ctx context.Context, p Params) (int64, bool) {
if p.BatchMaxPack > 0 || p.SourceConn == nil || p.SourceConn.Endpoint == nil {
if p.TargetMaxPack > 0 || p.SourceConn == nil || p.SourceConn.Endpoint == nil {
return 0, false
}
if p.SourceService == nil || !p.SourceService.SupportsBootstrapBatch() {
return 0, false
}
limit := int64(defaultAutoBatchMaxPackBytes)
limit := int64(defaultTargetMaxPackBytes)
if p.MaxPackBytes > 0 && p.MaxPackBytes < limit {
limit = p.MaxPackBytes
}
54 unmodified lines
return parts[0], parts[1], true
}
func autoBatchMaxPackBytes(p Params, err error) (int64, bool) {
if p.BatchMaxPack > 0 || !isTargetBodyLimitError(err) {
func autoTargetMaxPackBytes(p Params, err error) (int64, bool) {
if p.TargetMaxPack > 0 || !isTargetBodyLimitError(err) {
return 0, false
}
if p.SourceService == nil || !p.SourceService.SupportsBootstrapBatch() {
return 0, false
}
limit := int64(defaultAutoBatchMaxPackBytes)
limit := int64(defaultTargetMaxPackBytes)
if targetLimit := targetBodyLimit(err); targetLimit > 0 {
derived := targetLimit / 2
if derived <= 0 {