# Auto-subdivide batches on target body-size rejection

`85b386e`→[main](/content/gh/entireio/git-sync/commits/main/index.html)·

Soph·3mo ago·2 files·+125 added/-7 removed

When a batched bootstrap push is rejected by the target for exceeding
its body-size limit, instead of failing immediately, subdivide the
remaining checkpoints by inserting midpoints from the stored commit
chain and retry. Each failing batch gets split in half, halving the
commit range (and roughly halving the pack size) per retry. This
converges to batches small enough for the target without requiring the
user to manually lower --batch-max-pack-bytes and re-run.

The commit chain is already in memory from the planning phase (~1.5 MiB
for linux's 75k commits). subdivideCheckpoints walks the remaining
checkpoint list, computes chain-index gaps, and inserts a midpoint for
each gap > 1 commit. Adjacent commits (gap=1) cannot be split further;
if even a single-commit batch is too large, the push fails as before.

Example for the linux kernel at 200 MiB batch estimate:
- Estimate plans 3 batches (25k commits each)
- Batch 3 contains 6.5M objects (~2.3 GiB), target limit is 2 GiB
- Push rejected → subdivide inserts midpoint → 4 checkpoints
- Retry batch 3a (~12.5k commits, ~1.2 GiB) → succeeds
- Continue batch 3b (~12.5k commits, ~1.1 GiB) → succeeds
- No user intervention needed

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

## Sessions

516fe4e51170View transcript

[?\
what test coverage do we have now for replicate?Claude Code·Opus 4.6[1m]·1 step](/content/gh/entireio/git-sync/session/7b2777b1-8075-41f4-a62a-cbfc1b76c01b#timeline-516fe4e51170/index.html)

## Changes

2

- internal/strategy/bootstrap

- Mbootstrap.go+58/-7

- Mbootstrap_test.go+67

```go
// type plannedBatch struct {
// 	planner.BootstrapBatch
// 	chain []plumbing.Hash // full first-parent chain (root→tip) for subdividing on push failure
// }

// Execute runs the bootstrap strategy (one-shot or batched).

cmds := convert.PlansToPushCommands(stagePlans)
if err := p.TargetPusher.PushPack(ctx, cmds, packReader); err != nil {
	_ = packReader.Close()
	if isTargetBodyLimitError(err) && len(batch.chain) > 0 {
		expanded := subdivideCheckpoints(batch.chain, current, batch.Checkpoints[idx:])
		if len(expanded) > len(batch.Checkpoints[idx:]) {
			p.log("bootstrap batch subdividing after target size rejection",
				"branch", batch.Plan.TargetRef.String(),
				"old_remaining", len(batch.Checkpoints[idx:]),
				"new_remaining", len(expanded),
				"error", err.Error())
			batch.Checkpoints = append(batch.Checkpoints[:idx], expanded...)
			continue // retry with finer checkpoints, same idx
		}
	}
	return result, fmt.Errorf("push bootstrap batch for %s: %w", batch.Plan.TargetRef, err)
}
_ = packReader.Close()
```

// PlanCheckpoints plans the checkpoint hashes for a single branch during batched bootstrap.
func PlanCheckpoints(ctx context.Context, p Params, ref planner.DesiredRef) ([]plumbing.Hash, error) {
return planCheckpointsFromChain(ctx, p, ref)
checkpoints, _, err := planCheckpointsFromChain(ctx, p, ref)
return checkpoints, err
}

const estimatedBytesPerCommit = 8192

func planCheckpointsFromChain(ctx context.Context, p Params, ref planner.DesiredRef) ([]plumbing.Hash, error) {
p.log("bootstrap batch fetching commit graph", "branch", ref.TargetRef.String())
graphStore := memory.NewStorage()
gpRef := gitproto.DesiredRef{SourceRef: ref.SourceRef, TargetRef: ref.TargetRef, SourceHash: ref.SourceHash}
if err := p.SourceService.FetchCommitGraph(ctx, graphStore, p.SourceConn, gpRef); err != nil {
	return nil, fmt.Errorf("fetch bootstrap planning graph for %s: %w", ref.TargetRef, err)
}
	return nil, nil, fmt.Errorf("fetch bootstrap planning graph for %s: %w", ref.TargetRef, err)
}
chain, err := planner.FirstParentChain(graphStore, ref.SourceHash)
if err != nil {
	return nil, fmt.Errorf("walk first-parent chain for %s: %w", ref.TargetRef, err)
}
	return nil, nil, fmt.Errorf("walk first-parent chain for %s: %w", ref.TargetRef, err)
}
if len(chain) == 0 {
	return nil, fmt.Errorf("empty first-parent chain for %s", ref.TargetRef)
}

numBatches := estimateBatchCount(int64(len(chain)), p.BatchMaxPack)

return checkpoints, nil
}

// subdivideCheckpoints splits each remaining checkpoint range in half using
// the full commit chain. Called when a batch push is rejected for exceeding
// the target's body-size limit. Returns the expanded checkpoint list; if no
// split is possible (ranges are already 1 commit), returns the input unchanged.
func subdivideCheckpoints(chain []plumbing.Hash, current plumbing.Hash, remaining []plumbing.Hash) []plumbing.Hash {
chainIdx := make(map[plumbing.Hash]int, len(chain))
for i, h := range chain {
	chainIdx[h] = i
}
curIdx, ok := chainIdx[current]
if !ok && !current.IsZero() {
	return remaining
}
if current.IsZero() {
	curIdx = -1
}

expanded := make([]plumbing.Hash, 0, len(remaining)*2)
prev := curIdx
for _, cp := range remaining {
	cpIdx, ok := chainIdx[cp]
	if !ok {
		expanded = append(expanded, cp)
		continue
	}
gap := cpIdx - prev
if gap > 1 {
	midIdx := prev + gap/2
	expanded = append(expanded, chain[midIdx])
}
	expanded = append(expanded, cp)
prev = cpIdx
}
return expanded
}

func evenCheckpoints(chain []plumbing.Hash, numBatches int) []plumbing.Hash {
if numBatches <= 1 || len(chain) <= 1 {
	return []plumbing.Hash{chain[len(chain)-1]}
}
```