Reuse successful bootstrap probe packs · Entire
Reuse successful bootstrap probe packs
7407954→main·
Soph·3mo ago·2 files·+81 added/-34 removed
Sessions
4ea409f02833View transcript
Changes
2
docs
- Mrewrite-issue-list.md+3/-1
internal/strategy/bootstrap
- Mbootstrap.go+78/-33
305 unmodified lines
306
307
308
309
309
310
311
312
313
314
305 unmodified lines
- Rewrite branch should significantly reduce HTTP round-trips during batch planning.
Current rewrite note:
- The rewrite adds an initial commit-count heuristic and now memoizes repeated equivalent probe results within a planning run, but it still performs fetch-and-discard sizing probes in the bootstrap planner.
- Successful under-limit probe packs are now cached and reused during execution, which avoids a second fetch for selected checkpoints.
- The planner still relies on network probes for sizing, so this remains partial rather than fully solved.
### 15. Materialized fallback path does not scale to large repos
Mdocs/rewrite-issue-list.md+3/-1
2 unmodified lines
3
4
5
6
7
8
9
53 unmodified lines
63
64
65
66
67
68
69
70
71
72
73
81 unmodified lines
155
156
157
152
158
159
160
161
48 unmodified lines
210
211
212
207
208
209
213
214
215
216
213
217
218
219
57 unmodified lines
277
278
279
277
278
280
281
282
280
283
284
285
286
284
285
286
287
288
287
288
289
290
291
292
293
294
295
296
297
290
291
292
298
299
300
301
1 unmodified line
303
304
305
306
307
308
309
310
311
312
313
314
304
315
316
317
318
308
319
320
321
311
322
323
324
325
15 unmodified lines
341
342
343
333
344
345
346
347
348
337
338
349
350
351
340
352
353
354
355
344
356
357
358
359
348
360
361
362
351
363
364
365
366
367
368
369
5 unmodified lines
375
376
377
363
378
379
380
366
381
382
383
384
385
386
387
388
389
390
371
391
392
393
374
394
395
376
396
397
398
379
399
400
381
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
2 unmodified lines
package bootstrap
import (
"bytes"
"context"
"encoding/json"
"errors"
53 unmodified lines
TempRefs []string
}
type plannedBatch struct {
planner.BootstrapBatch
prefetchedPacks map[plumbing.Hash][]byte
}
// Execute runs the bootstrap strategy (one-shot or batched).
func Execute(ctx context.Context, p Params, relayReason string) (Result, error) {
plans, err := planner.BuildBootstrapPlans(p.DesiredRefs, p.TargetRefs)
81 unmodified lines
planRefs = append(planRefs, p.DesiredRefs[plan.TargetRef])
var batches []planner.BootstrapBatch
var batches []plannedBatch
if len(planRefs) > 0 {
p.log("bootstrap batch planning checkpoints", "branch_ref_count", len(planRefs))
var err error
48 unmodified lines
})
}
desired := singleGP(batch.Plan.SourceRef, batch.TempRef, checkpoint)
haves := planner.SingleHaveMap(current)
packReader, err := p.SourceService.FetchPack(ctx, p.SourceConn, desired, haves)
packReader, err := packReaderForCheckpoint(ctx, p, batch, checkpoint, current, batchLimit)
if err != nil {
return result, fmt.Errorf("fetch source batch pack for %s: %w", batch.Plan.TargetRef, err)
}
packReader = gitproto.LimitPackReader(packReader, batchLimit)
cmds := gitproto.ToPushCommands(convert.PlansToPushPlans(stagePlans))
if err := gitproto.PushPack(ctx, p.TargetConn, p.TargetAdv, cmds, packReader, p.Verbose); err != nil {
return result, fmt.Errorf("push bootstrap batch for %s: %w", batch.Plan.TargetRef, err)
}
57 unmodified lines
// --- Checkpoint planning ---
func planBatches(ctx context.Context, p Params, desired []planner.DesiredRef) ([]planner.BootstrapBatch, error) {
out := make([]planner.BootstrapBatch, 0, len(desired))
func planBatches(ctx context.Context, p Params, desired []planner.DesiredRef) ([]plannedBatch, error) {
out := make([]plannedBatch, 0, len(desired))
for _, ref := range desired {
checkpoints, err := PlanCheckpoints(ctx, p, ref)
checkpoints, prefetched, err := planCheckpointsWithCache(ctx, p, ref)
if err != nil {
return nil, err
}
out = append(out, planner.BootstrapBatch{
Plan: planner.BranchPlan{
Branch: ref.Label, SourceRef: ref.SourceRef,
TargetRef: ref.TargetRef, SourceHash: ref.SourceHash,
Kind: ref.Kind, Action: planner.ActionCreate,
out = append(out, plannedBatch{
BootstrapBatch: planner.BootstrapBatch{
Plan: planner.BranchPlan{
Branch: ref.Label, SourceRef: ref.SourceRef,
TargetRef: ref.TargetRef, SourceHash: ref.SourceHash,
Kind: ref.Kind, Action: planner.ActionCreate,
},
TempRef: planner.BootstrapTempRef(ref.TargetRef),
ResumeHash: p.TargetRefs[planner.BootstrapTempRef(ref.TargetRef)],
Checkpoints: checkpoints,
},
TempRef: planner.BootstrapTempRef(ref.TargetRef),
ResumeHash: p.TargetRefs[planner.BootstrapTempRef(ref.TargetRef)],
Checkpoints: checkpoints,
prefetchedPacks: prefetched,
})
}
return out, nil
1 unmodified line
// PlanCheckpoints plans the checkpoint hashes for a single branch during batched bootstrap.
func PlanCheckpoints(ctx context.Context, p Params, ref planner.DesiredRef) ([]plumbing.Hash, error) {
checkpoints, _, err := planCheckpointsWithCache(ctx, p, ref)
return checkpoints, err
}
func planCheckpointsWithCache(ctx context.Context, p Params, ref planner.DesiredRef) ([]plumbing.Hash, map[plumbing.Hash][]byte, error) {
p.log("bootstrap batch fetching commit graph", "branch", ref.TargetRef.String())
graphStore := memory.NewStorage()
gpRef := gitproto.DesiredRef{SourceRef: ref.SourceRef, TargetRef: ref.TargetRef, SourceHash: ref.SourceHash}
if err := p.SourceService.FetchCommitGraph(ctx, graphStore, p.SourceConn, gpRef); err != nil {
return nil, fmt.Errorf("fetch bootstrap planning graph for %s: %w", ref.TargetRef, err)
return nil, nil, fmt.Errorf("fetch bootstrap planning graph for %s: %w", ref.TargetRef, err)
}
chain, err := planner.FirstParentChain(graphStore, ref.SourceHash)
if err != nil {
return nil, fmt.Errorf("walk first-parent chain for %s: %w", ref.TargetRef, err)
return nil, nil, fmt.Errorf("walk first-parent chain for %s: %w", ref.TargetRef, err)
}
if len(chain) == 0 {
return nil, fmt.Errorf("empty first-parent chain for %s", ref.TargetRef)
return nil, nil, fmt.Errorf("empty first-parent chain for %s", ref.TargetRef)
}
// Issue #14: Use a commit-count heuristic for the initial span estimate
15 unmodified lines
prevIdx := -1
prevHash := plumbing.ZeroHash
prevSpan := initialSpan
probeCache := make(map[string]bool)
probeCache := make(map[string]probeResult)
prefetched := make(map[plumbing.Hash][]byte)
for prevIdx < len(chain)-1 {
bestIdx, err := planner.SampledCheckpointUnderLimit(chain, prevIdx, prevSpan, func(idx int) (bool, error) {
cacheKey := prevHash.String() + ":" + strconv.Itoa(idx)
if tooLarge, ok := probeCache[cacheKey]; ok {
return tooLarge, nil
if result, ok := probeCache[cacheKey]; ok && !result.tooLarge && len(result.data) > 0 {
return result.tooLarge, nil
}
tooLarge, err := packExceedsLimit(ctx, p, ref, chain[idx], prevHash, p.BatchMaxPack)
data, tooLarge, err := fetchPackForProbe(ctx, p, ref, chain[idx], prevHash, p.BatchMaxPack)
if err != nil {
return false, fmt.Errorf("measure bootstrap batch for %s at %s: %w", ref.TargetRef, planner.ShortHash(chain[idx]), err)
}
probeCache[cacheKey] = tooLarge
probeCache[cacheKey] = probeResult{tooLarge: tooLarge, data: data}
return tooLarge, nil
})
if err != nil {
return nil, err
return nil, nil, err
}
if bestIdx <= prevIdx {
return nil, fmt.Errorf("could not find bootstrap checkpoint for %s under batch-max-pack-bytes=%d", ref.TargetRef, p.BatchMaxPack)
return nil, nil, fmt.Errorf("could not find bootstrap checkpoint for %s under batch-max-pack-bytes=%d", ref.TargetRef, p.BatchMaxPack)
}
if result, ok := probeCache[prevHash.String()+":"+strconv.Itoa(bestIdx)]; ok && !result.tooLarge && len(result.data) > 0 {
prefetched[chain[bestIdx]] = result.data
}
prevSpan = bestIdx - prevIdx
prevIdx = bestIdx
5 unmodified lines
"selected", len(checkpoints),
"chain_len", len(chain))
}
return checkpoints, nil
return checkpoints, prefetched, nil
}
func packExceedsLimit(ctx context.Context, p Params, ref planner.DesiredRef, want, have plumbing.Hash, limit int64) (bool, error) {
type probeResult struct {
tooLarge bool
data []byte
}
func fetchPackForProbe(ctx context.Context, p Params, ref planner.DesiredRef, want, have plumbing.Hash, limit int64) ([]byte, bool, error) {
desired := singleGP(ref.SourceRef, ref.TargetRef, want)
haves := planner.SingleHaveMap(have)
packReader, err := p.SourceService.FetchPack(ctx, p.SourceConn, desired, haves)
if err != nil {
return false, err
return nil, false, err
}
defer packReader.Close()
_, err = io.Copy(io.Discard, gitproto.LimitPackReader(packReader, limit))
data, err := io.ReadAll(gitproto.LimitPackReader(packReader, limit))
if err == nil {
return false, nil
return data, false, nil
}
if strings.Contains(err.Error(), "source pack exceeded max-pack-bytes limit") {
return true, nil
return nil, true, nil
}
return false, err
return nil, false, err
}
func packReaderForCheckpoint(
ctx context.Context,
p Params,
batch plannedBatch,
checkpoint plumbing.Hash,
current plumbing.Hash,
batchLimit int64,
) (io.ReadCloser, error) {
if data, ok := batch.prefetchedPacks[checkpoint]; ok && len(data) > 0 {
p.log("bootstrap batch reusing prefetched probe pack",
"branch", batch.Plan.TargetRef.String(),
"checkpoint", planner.ShortHash(checkpoint),
"bytes", len(data))
return io.NopCloser(bytes.NewReader(data)), nil
}
desired := singleGP(batch.Plan.SourceRef, batch.TempRef, checkpoint)
haves := planner.SingleHaveMap(current)
packReader, err := p.SourceService.FetchPack(ctx, p.SourceConn, desired, haves)
if err != nil {
return nil, err
}
return gitproto.LimitPackReader(packReader, batchLimit), nil
}
// --- GitHub preflight ---