Expose materialized sync safety limit · Entire
Expose materialized sync safety limit
b1842f4→main·
Soph·3mo ago·8 files·+111 added/-58 removed
Sessions
cb50436aaefcView transcript
Changes
8
MREADME.md+3
cmd/git-sync
Mmain.go+1
docs
Marchitecture.md+1
- Mrewrite-issue-list.md+2/-1
internal
strategy/materialized
Mmaterialized.go+13/-4
Mmaterialized_test.go+16/-7
syncer
Mintegration_test.go+57/-31
Msyncer.go+18/-15
74 unmodified lines
75
76
77
78
79
80
81
116 unmodified lines
198
199
200
201
202
203
204
205
74 unmodified lines
- Ref mapping is explicit, not wildcard-based.
- Only smart HTTP remotes are supported.
- Objects are kept in memory for the duration of the run.
- Non-relay materialized syncs are bounded by `--materialized-max-objects` and fail clearly when they exceed that limit.
## Quick Start
116 unmodified lines
`sync` also uses a narrow incremental relay path for fast-forward branch updates and tag creation when there is no prune/delete, no force, and the target does not advertise `no-thin`. This now includes multi-branch batches, branch-to-branch mappings, and create-only tags. Tag retargeting and other more complex updates still fall back to the normal local decode-and-repack path.
If `sync` falls back to the materialized path, `--materialized-max-objects` sets an explicit safety bound for the in-memory object set. The default is conservative; raise it only when you intend to trade memory headroom for broader non-relay coverage.
Sync specific branches:
```bash
MREADME.md+3
72 unmodified lines
73
74
75
76
77
78
79
72 unmodified lines
fs.BoolVar(&cfg.ShowStats, "stats", false, "print transfer statistics")
fs.BoolVar(&cfg.MeasureMemory, "measure-memory", false, "sample elapsed time and Go heap usage")
fs.BoolVar(&jsonOutput, "json", false, "print JSON output")
fs.IntVar(&cfg.MaterializedMaxObjects, "materialized-max-objects", syncer.DefaultMaterializedMaxObjects, "abort non-relay materialized syncs above this many objects")
fs.StringVar(&cfg.ProtocolMode, "protocol", envOr("GITSYNC_PROTOCOL", validation.ProtocolAuto), "protocol mode: auto, v1, or v2")
fs.BoolVar(&cfg.Verbose, "v", false, "verbose logging")
Mcmd/git-sync/main.go+1
47 unmodified lines
48
49
50
51
52
53
54
47 unmodified lines
- incremental relay
- narrow fast path for safe updates
- materialized fallback
The fallback remains intentionally bounded: non-relay object materialization is kept in memory and guarded by an explicit object-count limit rather than being treated as unbounded.
- decode/repack path when relay is not safe
- batched bootstrap
- large initial migration fallback
Mdocs/architecture.md+1
328 unmodified lines
329
330
331
332
332
333
334
335
336
328 unmodified lines
- If no, fail early and clearly outside safe operating bounds.
Current rewrite note:
- The rewrite introduces a materialized strategy package and safety limits, but it still relies on in-memory object storage rather than a fundamentally new scaling model.
- The rewrite introduces a materialized strategy package, exposes an explicit `--materialized-max-objects` operating limit, and fails clearly when that bound is exceeded.
- It still relies on in-memory object storage rather than a fundamentally new scaling model, so this remains partial.
### 16. Fast-forward checks can degenerate into full graph walks
Mdocs/rewrite-issue-list.md+2/-1
28 unmodified lines
29
30
31
32
33
34
34
35
36
37
37
38
39
40
41
19 unmodified lines
61
62
63
63
64
65
66
67
66
68
69
70
71
29 unmodified lines
101
102
103
104
105
106
107
108
109
110
28 unmodified lines
DesiredRefs map[plumbing.ReferenceName]planner.DesiredRef
TargetRefs map[plumbing.ReferenceName]plumbing.Hash
PushPlans []planner.BranchPlan
MaxObjects int
}
// MaxMaterializedObjects is the safety limit for the materialized fallback path.
// DefaultMaxMaterializedObjects is the default safety limit for the materialized fallback path.
// Beyond this count, the in-memory object store would consume excessive memory.
// Fail early rather than OOM (issue #15).
const MaxMaterializedObjects = 500_000
const DefaultMaxMaterializedObjects = 500_000
// Execute runs the materialized fallback: ensures tag objects are local,
// computes the object closure, and pushes to the target.
19 unmodified lines
}
// Issue #15: guard against unbounded memory usage on large non-relay syncs.
if len(hashes) > MaxMaterializedObjects {
maxObjects := effectiveMaxObjects(p.MaxObjects)
if len(hashes) > maxObjects {
return fmt.Errorf(
"materialized push requires %d objects (limit %d); use bootstrap for large initial syncs",
len(hashes), MaxMaterializedObjects,
len(hashes), maxObjects,
)
}
29 unmodified lines
}
return nil
}
func effectiveMaxObjects(limit int) int {
if limit > 0 {
return limit
}
return DefaultMaxMaterializedObjects
}
Minternal/strategy/materialized/materialized.go+13/-4
107 unmodified lines
108
109
110
111
111
112
113
114
113
114
115
116
117
118
119
118
119
120
121
122
121
122
123
124
125
126
127
128
129
130
131
132
133
107 unmodified lines
}
}
func TestMaxMaterializedObjectsExported(t *testing.T) {
func TestDefaultMaxMaterializedObjectsExported(t *testing.T) {
// Verify the constant is exported and has a reasonable positive value.
if MaxMaterializedObjects <= 0 {
t.Fatalf("MaxMaterializedObjects should be positive, got %d", MaxMaterializedObjects)
}
if DefaultMaxMaterializedObjects <= 0 {
t.Fatalf("DefaultMaxMaterializedObjects should be positive, got %d", DefaultMaxMaterializedObjects)
}
// Sanity: it should be at least 1000 to be useful for real repos,
// but not so large that it defeats its purpose as a safety limit.
if MaxMaterializedObjects < 1_000 {
t.Fatalf("MaxMaterializedObjects too small: %d", MaxMaterializedObjects)
}
if DefaultMaxMaterializedObjects < 1_000 {
t.Fatalf("DefaultMaxMaterializedObjects too small: %d", DefaultMaxMaterializedObjects)
}
if MaxMaterializedObjects > 10_000_000 {
t.Fatalf("MaxMaterializedObjects unreasonably large: %d", MaxMaterializedObjects)
}
if DefaultMaxMaterializedObjects > 10_000_000 {
t.Fatalf("DefaultMaxMaterializedObjects unreasonably large: %d", DefaultMaxMaterializedObjects)
}
}
func TestEffectiveMaxObjects(t *testing.T) {
if got := effectiveMaxObjects(123); got != 123 {
t.Fatalf("effectiveMaxObjects(123) = %d, want 123", got)
}
if got := effectiveMaxObjects(0); got != DefaultMaxMaterializedObjects {
t.Fatalf("effectiveMaxObjects(0) = %d, want %d", got, DefaultMaxMaterializedObjects)
}
}
Minternal/strategy/materialized/materialized_test.go+16/-7
123 unmodified lines
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
407 unmodified lines
577
578
579
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
580
581
563
582
583
584
585
586
568
569
587
588
589
590
591
592
593
594
595
596
597
598
599
13 unmodified lines
613
614
615
591
592
616
617
618
619
620
6 unmodified lines
627
628
629
605
606
607
630
631
632
633
612
613
634
635
636
637
638
639
640
641
642
123 unmodified lines
}
}
func TestRun_IntegrationMaterializedLimitFailsClearly(t *testing.T) {
sourceRepo, sourceFS := newSourceRepo(t)
makeCommits(t, sourceRepo, sourceFS, 3)
targetRepo, err := git.Init(memory.NewStorage())
if err != nil {
t.Fatalf("init target repo: %v", err)
}
if err := copyRefsAndObjects(sourceRepo.Storer, targetRepo.Storer, []plumbing.ReferenceName{plumbing.NewBranchReferenceName(testBranch)}); err != nil {
t.Fatalf("copy target baseline: %v", err)
}
makeCommits(t, sourceRepo, sourceFS, 1)
sourceHead, err := sourceRepo.Reference(plumbing.NewBranchReferenceName(testBranch), true)
if err != nil {
t.Fatalf("resolve source head: %v", err)
}
if err := sourceRepo.Storer.SetReference(plumbing.NewHashReference(plumbing.NewBranchReferenceName("release"), sourceHead.Hash())); err != nil {
t.Fatalf("set source release branch: %v", err)
}
sourceServer := newSmartHTTPRepoServerV2(t, sourceRepo)
targetServer := newSmartHTTPRepoServer(t, targetRepo)
defer sourceServer.Close()
defer targetServer.Close()
_, err = Run(context.Background(), Config{
Source: Endpoint{URL: sourceServer.RepoURL()},
Target: Endpoint{URL: targetServer.RepoURL()},
ProtocolMode: protocolModeAuto,
MaterializedMaxObjects: 1,
})
if err == nil {
t.Fatal("expected materialized limit failure")
}
if !strings.Contains(err.Error(), "materialized push requires") {
t.Fatalf("expected materialized limit error, got %v", err)
}
}
func TestRun_IntegrationPlanSuggestsBootstrapOnEmptyTarget(t *testing.T) {
sourceRepo, sourceFS := newSourceRepo(t)
makeCommits(t, sourceRepo, sourceFS, 2)
407 unmodified lines
BatchMaxPackBytes: 350_000,
}
s, err := newSession(context.Background(), cfg, false)
if err != nil {
t.Fatalf("new session: %v", err)
}
desired, _, err := planner.BuildDesiredRefs(s.sourceRefMap, planConfig(cfg))
if err != nil {
t.Fatalf("build desired refs: %v", err)
}
ref := desired[plumbing.NewBranchReferenceName(testBranch)]
checkpoints, err := bstrap.PlanCheckpoints(context.Background(), bstrap.Params{
SourceConn: s.sourceConn,
SourceService: s.sourceService,
BatchMaxPack: cfg.BatchMaxPackBytes,
}, ref)
if err != nil {
t.Fatalf("plan checkpoints: %v", err)
}
if len(checkpoints) < 2 {
t.Fatalf("expected multiple checkpoints, got %d", len(checkpoints))
}
targetRef := plumbing.NewBranchReferenceName(testBranch)
tempRef := planner.BootstrapTempRef(targetRef)
packPushes := 0
failedAfterProgress := false
targetServer.receivePackHook = func(req *packp.UpdateRequests, hasPack bool) *packp.ReportStatus {
if !hasPack || len(req.Commands) == 0 || req.Commands[0].Name != tempRef {
return nil
}
packPushes++
if packPushes != 2 {
if failedAfterProgress {
return nil
}
if _, err := targetRepo.Reference(tempRef, true); err != nil {
return nil
}
if _, err := targetRepo.Reference(targetRef, true); err == nil {
return nil
}
failedAfterProgress = true
report := packp.NewReportStatus()
report.UnpackStatus = "ok"
for _, cmd := range req.Commands {
13 unmodified lines
if err != nil {
t.Fatalf("resolve temp ref after failed checkpoint push: %v", err)
}
if targetTemp.Hash() != checkpoints[0] {
t.Fatalf("expected temp ref at first checkpoint %s after failure, got %s", checkpoints[0], targetTemp.Hash())
}
if targetTemp.Hash().IsZero() {
t.Fatalf("expected non-zero temp ref after failed checkpoint push")
}
if _, err := targetRepo.Reference(targetRef, true); err == nil {
t.Fatalf("expected target ref %s to remain absent after failed checkpoint push", targetRef)
}
}
if !result.Relay || result.RelayMode != "bootstrap-batch" {
t.Fatalf("expected batched bootstrap result, got %+v", result)
}
if result.BatchCount >= result.PlannedBatchCount {
t.Fatalf("expected resumed retry to execute fewer batches than planned, got %+v", result)
}
targetHead, err := targetRepo.Reference(targetRef, true)
if err != nil {
t.Fatalf("resolve target ref after retry: %v", err)
}
if targetHead.Hash() != ref.SourceHash {
t.Fatalf("expected target head %s after retry, got %s", ref.SourceHash, targetHead.Hash())
}
sourceHead, err := sourceRepo.Reference(targetRef, true)
if err != nil {
t.Fatalf("resolve source head after retry: %v", err)
}
if targetHead.Hash() != sourceHead.Hash() {
t.Fatalf("expected target head %s after retry, got %s", sourceHead.Hash(), targetHead.Hash())
}
if _, err := targetRepo.Reference(tempRef, true); err == nil {
t.Fatalf("expected temp ref %s to be deleted after retry", tempRef)
}
}
Minternal/syncer/integration_test.go+57/-31
35 unmodified lines
36
37
38
39
40
41
42
43
8 unmodified lines
52
53
54
53
54
55
56
57
58
59
60
61
62
63
64
65
66
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
412 unmodified lines
485
486
487
485
488
489
490
491