Auto-subdivide batches on target body-size rejection

85b386e→main·

Soph·3mo ago·2 files·+125 added/-7 removed

When a batched bootstrap push is rejected by the target for exceeding its body-size limit, instead of failing immediately, subdivide the remaining checkpoints by inserting midpoints from the stored commit chain and retry. Each failing batch gets split in half, halving the commit range (and roughly halving the pack size) per retry. This converges to batches small enough for the target without requiring the user to manually lower --batch-max-pack-bytes and re-run.

The commit chain is already in memory from the planning phase (~1.5 MiB for linux's 75k commits). subdivideCheckpoints walks the remaining checkpoint list, computes chain-index gaps, and inserts a midpoint for each gap > 1 commit. Adjacent commits (gap=1) cannot be split further; if even a single-commit batch is too large, the push fails as before.

Example for the linux kernel at 200 MiB batch estimate:

• Estimate plans 3 batches (25k commits each)
• Batch 3 contains 6.5M objects (~2.3 GiB), target limit is 2 GiB
• Push rejected → subdivide inserts midpoint → 4 checkpoints
• Retry batch 3a (~12.5k commits, ~1.2 GiB) → succeeds
• Continue batch 3b (~12.5k commits, ~1.1 GiB) → succeeds
• No user intervention needed

Co-Authored-By: Claude Opus 4.6 (1M context) noreply@anthropic.com

Sessions

516fe4e51170View transcript

[?
what test coverage do we have now for replicate?Claude Code·Opus 4.6[1m]·1 step](/content/gh/entireio/git-sync/session/7b2777b1-8075-41f4-a62a-cbfc1b76c01b#timeline-516fe4e51170/index.html)

Changes

2

• internal/strategy/bootstrap

• Mbootstrap.go+58/-7

• Mbootstrap_test.go+67

// type plannedBatch struct {
// 	planner.BootstrapBatch
// 	chain []plumbing.Hash // full first-parent chain (root→tip) for subdividing on push failure
// }

// Execute runs the bootstrap strategy (one-shot or batched).

cmds := convert.PlansToPushCommands(stagePlans)
if err := p.TargetPusher.PushPack(ctx, cmds, packReader); err != nil {
    _ = packReader.Close()
    if isTargetBodyLimitError(err) && len(batch.chain) > 0 {
        expanded := subdivideCheckpoints(batch.chain, current, batch.Checkpoints[idx:])
        if len(expanded) > len(batch.Checkpoints[idx:]) {
            p.log("bootstrap batch subdividing after target size rejection",
                "branch", batch.Plan.TargetRef.String(),
                "old_remaining", len(batch.Checkpoints[idx:]),
                "new_remaining", len(expanded),
                "error", err.Error())
            batch.Checkpoints = append(batch.Checkpoints[:idx], expanded...)
            continue // retry with finer checkpoints, same idx
        }
    }
    return result, fmt.Errorf("push bootstrap batch for %s: %w", batch.Plan.TargetRef, err)
}
_ = packReader.Close()

// PlanCheckpoints plans the checkpoint hashes for a single branch during batched bootstrap. func PlanCheckpoints(ctx context.Context, p Params, ref planner.DesiredRef) ([]plumbing.Hash, error) { return planCheckpointsFromChain(ctx, p, ref) checkpoints, _, err := planCheckpointsFromChain(ctx, p, ref) return checkpoints, err }

const estimatedBytesPerCommit = 8192

func planCheckpointsFromChain(ctx context.Context, p Params, ref planner.DesiredRef) ([]plumbing.Hash, error) { p.log("bootstrap batch fetching commit graph", "branch", ref.TargetRef.String()) graphStore := memory.NewStorage() gpRef := gitproto.DesiredRef{SourceRef: ref.SourceRef, TargetRef: ref.TargetRef, SourceHash: ref.SourceHash} if err := p.SourceService.FetchCommitGraph(ctx, graphStore, p.SourceConn, gpRef); err != nil { return nil, fmt.Errorf("fetch bootstrap planning graph for %s: %w", ref.TargetRef, err) } return nil, nil, fmt.Errorf("fetch bootstrap planning graph for %s: %w", ref.TargetRef, err) } chain, err := planner.FirstParentChain(graphStore, ref.SourceHash) if err != nil { return nil, fmt.Errorf("walk first-parent chain for %s: %w", ref.TargetRef, err) } return nil, nil, fmt.Errorf("walk first-parent chain for %s: %w", ref.TargetRef, err) } if len(chain) == 0 { return nil, fmt.Errorf("empty first-parent chain for %s", ref.TargetRef) }

numBatches := estimateBatchCount(int64(len(chain)), p.BatchMaxPack)

return checkpoints, nil }

// subdivideCheckpoints splits each remaining checkpoint range in half using // the full commit chain. Called when a batch push is rejected for exceeding // the target's body-size limit. Returns the expanded checkpoint list; if no // split is possible (ranges are already 1 commit), returns the input unchanged. func subdivideCheckpoints(chain []plumbing.Hash, current plumbing.Hash, remaining []plumbing.Hash) []plumbing.Hash { chainIdx := make(map[plumbing.Hash]int, len(chain)) for i, h := range chain { chainIdx[h] = i } curIdx, ok := chainIdx[current] if !ok && !current.IsZero() { return remaining } if current.IsZero() { curIdx = -1 }

expanded := make([]plumbing.Hash, 0, len(remaining)*2) prev := curIdx for _, cp := range remaining { cpIdx, ok := chainIdx[cp] if !ok { expanded = append(expanded, cp) continue } gap := cpIdx - prev if gap > 1 { midIdx := prev + gap/2 expanded = append(expanded, chain[midIdx]) } expanded = append(expanded, cp) prev = cpIdx } return expanded }

func evenCheckpoints(chain []plumbing.Hash, numBatches int) []plumbing.Hash { if numBatches <= 1 || len(chain) <= 1 { return []plumbing.Hash{chain[len(chain)-1]} }