# Reuse successful bootstrap probe packs `7407954`→[main](/content/gh/entireio/git-sync/commits/main/index.html)· Soph·3mo ago·2 files·+81 added/-34 removed ## Sessions 4ea409f02833View transcript ## Changes 2 - docs - Mrewrite-issue-list.md+3/-1 - internal/strategy/bootstrap - Mbootstrap.go+78/-33 ``` 305 unmodified lines 306 307 308 309 309 310 311 312 313 314 305 unmodified lines - Rewrite branch should significantly reduce HTTP round-trips during batch planning. Current rewrite note: - The rewrite adds an initial commit-count heuristic and now memoizes repeated equivalent probe results within a planning run, but it still performs fetch-and-discard sizing probes in the bootstrap planner. - Successful under-limit probe packs are now cached and reused during execution, which avoids a second fetch for selected checkpoints. - The planner still relies on network probes for sizing, so this remains partial rather than fully solved. ### 15. Materialized fallback path does not scale to large repos ``` Mdocs/rewrite-issue-list.md+3/-1 ``` 2 unmodified lines 3 4 5 6 7 8 9 53 unmodified lines 63 64 65 66 67 68 69 70 71 72 73 81 unmodified lines 155 156 157 152 158 159 160 161 48 unmodified lines 210 211 212 207 208 209 213 214 215 216 213 217 218 219 57 unmodified lines 277 278 279 277 278 280 281 282 280 283 284 285 286 284 285 286 287 288 287 288 289 290 291 292 293 294 295 296 297 290 291 292 298 299 300 301 1 unmodified line 303 304 305 306 307 308 309 310 311 312 313 314 304 315 316 317 318 308 319 320 321 311 322 323 324 325 15 unmodified lines 341 342 343 333 344 345 346 347 348 337 338 349 350 351 340 352 353 354 355 344 356 357 358 359 348 360 361 362 351 363 364 365 366 367 368 369 5 unmodified lines 375 376 377 363 378 379 380 366 381 382 383 384 385 386 387 388 389 390 371 391 392 393 374 394 395 376 396 397 398 379 399 400 381 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 2 unmodified lines package bootstrap import ( "bytes" "context" "encoding/json" "errors" 53 unmodified lines TempRefs []string } type plannedBatch struct { planner.BootstrapBatch prefetchedPacks map[plumbing.Hash][]byte } // Execute runs the bootstrap strategy (one-shot or batched). func Execute(ctx context.Context, p Params, relayReason string) (Result, error) { plans, err := planner.BuildBootstrapPlans(p.DesiredRefs, p.TargetRefs) 81 unmodified lines planRefs = append(planRefs, p.DesiredRefs[plan.TargetRef]) var batches []planner.BootstrapBatch var batches []plannedBatch if len(planRefs) > 0 { p.log("bootstrap batch planning checkpoints", "branch_ref_count", len(planRefs)) var err error 48 unmodified lines }) } desired := singleGP(batch.Plan.SourceRef, batch.TempRef, checkpoint) haves := planner.SingleHaveMap(current) packReader, err := p.SourceService.FetchPack(ctx, p.SourceConn, desired, haves) packReader, err := packReaderForCheckpoint(ctx, p, batch, checkpoint, current, batchLimit) if err != nil { return result, fmt.Errorf("fetch source batch pack for %s: %w", batch.Plan.TargetRef, err) } packReader = gitproto.LimitPackReader(packReader, batchLimit) cmds := gitproto.ToPushCommands(convert.PlansToPushPlans(stagePlans)) if err := gitproto.PushPack(ctx, p.TargetConn, p.TargetAdv, cmds, packReader, p.Verbose); err != nil { return result, fmt.Errorf("push bootstrap batch for %s: %w", batch.Plan.TargetRef, err) } 57 unmodified lines // --- Checkpoint planning --- func planBatches(ctx context.Context, p Params, desired []planner.DesiredRef) ([]planner.BootstrapBatch, error) { out := make([]planner.BootstrapBatch, 0, len(desired)) func planBatches(ctx context.Context, p Params, desired []planner.DesiredRef) ([]plannedBatch, error) { out := make([]plannedBatch, 0, len(desired)) for _, ref := range desired { checkpoints, err := PlanCheckpoints(ctx, p, ref) checkpoints, prefetched, err := planCheckpointsWithCache(ctx, p, ref) if err != nil { return nil, err } out = append(out, planner.BootstrapBatch{ Plan: planner.BranchPlan{ Branch: ref.Label, SourceRef: ref.SourceRef, TargetRef: ref.TargetRef, SourceHash: ref.SourceHash, Kind: ref.Kind, Action: planner.ActionCreate, out = append(out, plannedBatch{ BootstrapBatch: planner.BootstrapBatch{ Plan: planner.BranchPlan{ Branch: ref.Label, SourceRef: ref.SourceRef, TargetRef: ref.TargetRef, SourceHash: ref.SourceHash, Kind: ref.Kind, Action: planner.ActionCreate, }, TempRef: planner.BootstrapTempRef(ref.TargetRef), ResumeHash: p.TargetRefs[planner.BootstrapTempRef(ref.TargetRef)], Checkpoints: checkpoints, }, TempRef: planner.BootstrapTempRef(ref.TargetRef), ResumeHash: p.TargetRefs[planner.BootstrapTempRef(ref.TargetRef)], Checkpoints: checkpoints, prefetchedPacks: prefetched, }) } return out, nil 1 unmodified line // PlanCheckpoints plans the checkpoint hashes for a single branch during batched bootstrap. func PlanCheckpoints(ctx context.Context, p Params, ref planner.DesiredRef) ([]plumbing.Hash, error) { checkpoints, _, err := planCheckpointsWithCache(ctx, p, ref) return checkpoints, err } func planCheckpointsWithCache(ctx context.Context, p Params, ref planner.DesiredRef) ([]plumbing.Hash, map[plumbing.Hash][]byte, error) { p.log("bootstrap batch fetching commit graph", "branch", ref.TargetRef.String()) graphStore := memory.NewStorage() gpRef := gitproto.DesiredRef{SourceRef: ref.SourceRef, TargetRef: ref.TargetRef, SourceHash: ref.SourceHash} if err := p.SourceService.FetchCommitGraph(ctx, graphStore, p.SourceConn, gpRef); err != nil { return nil, fmt.Errorf("fetch bootstrap planning graph for %s: %w", ref.TargetRef, err) return nil, nil, fmt.Errorf("fetch bootstrap planning graph for %s: %w", ref.TargetRef, err) } chain, err := planner.FirstParentChain(graphStore, ref.SourceHash) if err != nil { return nil, fmt.Errorf("walk first-parent chain for %s: %w", ref.TargetRef, err) return nil, nil, fmt.Errorf("walk first-parent chain for %s: %w", ref.TargetRef, err) } if len(chain) == 0 { return nil, fmt.Errorf("empty first-parent chain for %s", ref.TargetRef) return nil, nil, fmt.Errorf("empty first-parent chain for %s", ref.TargetRef) } // Issue #14: Use a commit-count heuristic for the initial span estimate 15 unmodified lines prevIdx := -1 prevHash := plumbing.ZeroHash prevSpan := initialSpan probeCache := make(map[string]bool) probeCache := make(map[string]probeResult) prefetched := make(map[plumbing.Hash][]byte) for prevIdx < len(chain)-1 { bestIdx, err := planner.SampledCheckpointUnderLimit(chain, prevIdx, prevSpan, func(idx int) (bool, error) { cacheKey := prevHash.String() + ":" + strconv.Itoa(idx) if tooLarge, ok := probeCache[cacheKey]; ok { return tooLarge, nil if result, ok := probeCache[cacheKey]; ok && !result.tooLarge && len(result.data) > 0 { return result.tooLarge, nil } tooLarge, err := packExceedsLimit(ctx, p, ref, chain[idx], prevHash, p.BatchMaxPack) data, tooLarge, err := fetchPackForProbe(ctx, p, ref, chain[idx], prevHash, p.BatchMaxPack) if err != nil { return false, fmt.Errorf("measure bootstrap batch for %s at %s: %w", ref.TargetRef, planner.ShortHash(chain[idx]), err) } probeCache[cacheKey] = tooLarge probeCache[cacheKey] = probeResult{tooLarge: tooLarge, data: data} return tooLarge, nil }) if err != nil { return nil, err return nil, nil, err } if bestIdx <= prevIdx { return nil, fmt.Errorf("could not find bootstrap checkpoint for %s under batch-max-pack-bytes=%d", ref.TargetRef, p.BatchMaxPack) return nil, nil, fmt.Errorf("could not find bootstrap checkpoint for %s under batch-max-pack-bytes=%d", ref.TargetRef, p.BatchMaxPack) } if result, ok := probeCache[prevHash.String()+":"+strconv.Itoa(bestIdx)]; ok && !result.tooLarge && len(result.data) > 0 { prefetched[chain[bestIdx]] = result.data } prevSpan = bestIdx - prevIdx prevIdx = bestIdx 5 unmodified lines "selected", len(checkpoints), "chain_len", len(chain)) } return checkpoints, nil return checkpoints, prefetched, nil } func packExceedsLimit(ctx context.Context, p Params, ref planner.DesiredRef, want, have plumbing.Hash, limit int64) (bool, error) { type probeResult struct { tooLarge bool data []byte } func fetchPackForProbe(ctx context.Context, p Params, ref planner.DesiredRef, want, have plumbing.Hash, limit int64) ([]byte, bool, error) { desired := singleGP(ref.SourceRef, ref.TargetRef, want) haves := planner.SingleHaveMap(have) packReader, err := p.SourceService.FetchPack(ctx, p.SourceConn, desired, haves) if err != nil { return false, err return nil, false, err } defer packReader.Close() _, err = io.Copy(io.Discard, gitproto.LimitPackReader(packReader, limit)) data, err := io.ReadAll(gitproto.LimitPackReader(packReader, limit)) if err == nil { return false, nil return data, false, nil } if strings.Contains(err.Error(), "source pack exceeded max-pack-bytes limit") { return true, nil return nil, true, nil } return false, err return nil, false, err } func packReaderForCheckpoint( ctx context.Context, p Params, batch plannedBatch, checkpoint plumbing.Hash, current plumbing.Hash, batchLimit int64, ) (io.ReadCloser, error) { if data, ok := batch.prefetchedPacks[checkpoint]; ok && len(data) > 0 { p.log("bootstrap batch reusing prefetched probe pack", "branch", batch.Plan.TargetRef.String(), "checkpoint", planner.ShortHash(checkpoint), "bytes", len(data)) return io.NopCloser(bytes.NewReader(data)), nil } desired := singleGP(batch.Plan.SourceRef, batch.TempRef, checkpoint) haves := planner.SingleHaveMap(current) packReader, err := p.SourceService.FetchPack(ctx, p.SourceConn, desired, haves) if err != nil { return nil, err } return gitproto.LimitPackReader(packReader, batchLimit), nil } // --- GitHub preflight ---