Add --target-max-ref-updates flag; quiet ref-only batch progress · Entire
Add --target-max-ref-updates flag; quiet ref-only batch progress
6f12383→main·
Soph·4w ago·6 files·+168 added/-65 removed
Plumb a per-target ref-update batch size through the Pusher (MaxRefUpdates), the syncer config, the unstable Options, and the replicate/sync/plan/bootstrap commands as --target-max-ref-updates. Zero keeps the env-or-default limit (GITSYNC_MAX_REF_UPDATES_PER_PUSH or 5000); a positive value overrides it — raise it for entire-server targets (up to 25k), lower it for stricter providers. No global mutable state: the value rides on the Pusher.
Also stop spewing a bare "target:" sideband line per ref-only follow-up batch: those carry no useful progress, so push them with progress suppressed and, when verbose, emit one concise "pushed ref-update batch N/M (K refs)" line per batch instead.
Co-Authored-By: Claude Opus 4.8 (1M context) noreply@anthropic.com
Sessions
Changes
6
cmd/git-sync
Mbootstrap.go+1
Msyncplan.go+1
internal
gitproto
Mpush.go+72/-32
Mpush_test.go+86/-30
syncer
Msyncer.go+2
unstable
Mclient.go+6/-3
78 unmodified lines
79
80
81
82
83
84
85
78 unmodified lines
cmd.Flags().BoolVar(&jsonOutput, "json", false, "print JSON output")
cmd.Flags().Int64Var(&req.Options.MaxPackBytes, "max-pack-bytes", 0, "abort bootstrap if the streamed source pack exceeds this many bytes")
cmd.Flags().Int64Var(&req.Options.TargetMaxPackBytes, "target-max-pack-bytes", 0, "target receive-pack body size limit; batches are planned and auto-subdivided to fit")
cmd.Flags().IntVar(&req.Options.TargetMaxRefUpdates, "target-max-ref-updates", 0, "max ref-update commands per receive-pack request; 0 uses the default (env GITSYNC_MAX_REF_UPDATES_PER_PUSH or 5000). Raise for entire-server targets (up to 25000); lower for providers that reject large ref pushes")
cmd.Flags().StringVar(&req.Options.BootstrapStrategy, "bootstrap-strategy", "", "checkpoint chain ordering: \"first-parent\" (default) or \"topo\". Use \"topo\" for merge-heavy repos where individual first-parent steps drag in unboundedly large side branches; requires the target to allow non-fast-forward updates on the refs/gitsync/ namespace")
addProtocolFlag(cmd, &protocolVal)
cmd.Flags().BoolVarP(&req.Options.Verbose, "verbose", "v", false, "verbose logging")
Mcmd/git-sync/bootstrap.go+1
137 unmodified lines
138
139
140
141
142
143
144
137 unmodified lines
cmd.Flags().IntVar(&req.Options.MaterializedMaxObjects, "materialized-max-objects", unstable.DefaultMaterializedMaxObjects, "abort non-relay materialized syncs above this many objects")
cmd.Flags().Int64Var(&req.Options.MaxPackBytes, "max-pack-bytes", 0, "abort bootstrap-relay push if the streamed source pack exceeds this many bytes")
cmd.Flags().Int64Var(&req.Options.TargetMaxPackBytes, "target-max-pack-bytes", 0, "target receive-pack body size limit; batches are planned and auto-subdivided to fit")
cmd.Flags().IntVar(&req.Options.TargetMaxRefUpdates, "target-max-ref-updates", 0, "max ref-update commands per receive-pack request; 0 uses the default (env GITSYNC_MAX_REF_UPDATES_PER_PUSH or 5000). Raise for entire-server targets (up to 25000); lower for providers that reject large ref pushes")
cmd.Flags().StringVar(&req.Options.BootstrapStrategy, "bootstrap-strategy", "", "checkpoint chain ordering for bootstrap: \"first-parent\" (default) or \"topo\". Use \"topo\" for merge-heavy repos where individual first-parent steps drag in unboundedly large side branches; requires the target to allow non-fast-forward updates on the refs/gitsync/ namespace")
addProtocolFlag(cmd, &protocolVal)
cmd.Flags().BoolVarP(&req.Options.Verbose, "verbose", "v", false, "verbose logging")
Mcmd/git-sync/syncplan.go+1
43 unmodified lines
44
45
46
47
48
49
50
51
52
53
54
36 unmodified lines
91
92
93
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
96
97
98
110
111
112
113
114
115
116
117
104
105
106
107
108
118
119
120
121
122
123
124
111
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
116
146
147
148
149
150
151
122
152
153
154
155
156
127
157
158
159
160
288 unmodified lines
449
450
451
422
423
424
425
426
452
453
454
455
456
457
458
459
460
1 unmodified line
462
463
464
465
466
467
468
437
469
470
471
472
473
8 unmodified lines
482
483
484
452
485
486
487
488
489
457
490
491
492
493
494
495
463
496
497
498
499
203 unmodified lines
703
704
705
706
707
708
709
6 unmodified lines
716
717
718
685
686
687
719
720
721
722
723
724
11 unmodified lines
736
737
738
705
739
740
741
742
13 unmodified lines
756
757
758
759
760
761
762
728
729
763
764
765
766
767
768
769
770
771
772
773
774
43 unmodified lines
Adv *packp.AdvRefs
Verbose bool
OnRejection func(refName plumbing.ReferenceName, status string)
// MaxRefUpdates caps ref-update commands per receive-pack request. Zero
// uses the env-or-default limit (see MaxRefUpdatesEnv); a positive value
// overrides it — e.g. from the --target-max-ref-updates flag.
MaxRefUpdates int
}
// NewPusher builds a target-side push executor.
36 unmodified lines
return defaultMaxRefUpdatesPerPush
}
// chunkRefUpdates splits commands into batches no larger than
// maxRefUpdatesPerPush. Input that already fits is returned as a single batch
// (including the empty slice, so callers preserve their one-request behavior).
func chunkRefUpdates(commands []PushCommand) [][]PushCommand {
if len(commands) <= maxRefUpdatesPerPush {
// effectiveMaxRefUpdates resolves a per-push limit: a positive override wins,
// otherwise the env-or-default limit applies.
func effectiveMaxRefUpdates(maxRefUpdates int) int {
if maxRefUpdates > 0 {
return maxRefUpdates
}
return maxRefUpdatesPerPush
}
// chunkRefUpdates splits commands into batches no larger than limit. Input that
// already fits is returned as a single batch (including the empty slice, so
// callers preserve their one-request behavior).
func chunkRefUpdates(commands []PushCommand, limit int) [][]PushCommand {
if len(commands) <= limit {
return [][]PushCommand{commands}
}
batches := make([][]PushCommand, 0, (len(commands)+maxRefUpdatesPerPush-1)/maxRefUpdatesPerPush)
for start := 0; start < len(commands); start += maxRefUpdatesPerPush {
end := min(start+maxRefUpdatesPerPush, len(commands))
batches := make([][]PushCommand, 0, (len(commands)+limit-1)/limit)
for start := 0; start < len(commands); start += limit {
end := min(start+limit, len(commands))
batches = append(batches, commands[start:end])
}
return batches
}
// splitFirstBatch peels off the first batch (up to maxRefUpdatesPerPush) so a
// push can carry the pack with that batch and send the remainder as ref-only
// follow-ups. rest is nil when commands already fit in a single request.
func splitFirstBatch(commands []PushCommand) (first, rest []PushCommand) {
if len(commands) <= maxRefUpdatesPerPush {
// splitFirstBatch peels off the first batch (up to limit) so a push can carry
// the pack with that batch and send the remainder as ref-only follow-ups. rest
// is nil when commands already fit in a single request.
func splitFirstBatch(commands []PushCommand, limit int) (first, rest []PushCommand) {
if len(commands) <= limit {
return commands, nil
}
return commands[:maxRefUpdatesPerPush], commands[maxRefUpdatesPerPush:]
return commands[:limit], commands[limit:]
}
// logRefUpdateBatch reports completion of one ref-update batch to the progress
// writer. Ref-only follow-up batches push with progress suppressed (their
// sideband carries nothing but a bare "target:" line per batch), so this is the
// only per-batch signal; it stays quiet unless verbose and the push actually
// spanned multiple batches.
func logRefUpdateBatch(conn Conn, verbose bool, batchNum, totalBatches, refs int) {
if !verbose || totalBatches <= 1 {
return
}
w := conn.ProgressWriter()
if w == nil {
w = os.Stderr
}
fmt.Fprintf(w, "target: pushed ref-update batch %d/%d (%d refs)\n", batchNum, totalBatches, refs)
}
// PushPack streams a pack to the target.
func (p *Pusher) PushPack(ctx context.Context, commands []PushCommand, pack io.ReadCloser) error {
return PushPack(ctx, p.Conn, p.Adv, commands, pack, p.Verbose, p.OnRejection)
return PushPack(ctx, p.Conn, p.Adv, commands, pack, p.MaxRefUpdates, p.Verbose, p.OnRejection)
}
// PushCommands sends ref-only updates. Creates/updates carry an empty pack;
// delete-only pushes carry no pack. See the package-level PushCommands.
func (p *Pusher) PushCommands(ctx context.Context, commands []PushCommand) error {
return PushCommands(ctx, p.Conn, p.Adv, commands, p.Verbose, p.OnRejection)
return PushCommands(ctx, p.Conn, p.Adv, commands, p.MaxRefUpdates, p.Verbose, p.OnRejection)
}
// PushObjects encodes and pushes locally materialized objects.
func (p *Pusher) PushObjects(ctx context.Context, commands []PushCommand, store storer.Storer, hashes []plumbing.Hash) error {
return PushObjects(ctx, p.Conn, p.Adv, commands, store, hashes, p.Verbose, p.OnRejection)
return PushObjects(ctx, p.Conn, p.Adv, commands, store, hashes, p.MaxRefUpdates, p.Verbose, p.OnRejection)
}
// buildUpdateRequest builds the receive-pack update request.
288 unmodified lines
// PushObjects pushes locally-materialized objects to the target.
// A push within the per-request ref-update cap (maxRefUpdatesPerPush) is a
// single atomic receive-pack request. A larger push is split: the materialized
// pack — which carries every object for the whole push — rides with the first
// batch of object-bearing commands, then the remaining refs (and any deletes)
// move as ref-only updates because the objects are already committed.
// A push within the per-request ref-update limit (see effectiveMaxRefUpdates)
// is a single atomic receive-pack request. A larger push is split: the
// materialized pack — which carries every object for the whole push — rides
// with the first batch of object-bearing commands, then the remaining refs (and
// any deletes) move as ref-only updates because the objects are already
// committed.
func PushObjects(
ctx context.Context,
conn Conn,
1 unmodified line
commands []PushCommand,
store storer.Storer,
hashes []plumbing.Hash,
maxRefUpdates int,
verbose bool,
onRejection func(plumbing.ReferenceName, string),
) error {
if len(commands) <= maxRefUpdatesPerPush {
limit := effectiveMaxRefUpdates(maxRefUpdates)
if len(commands) <= limit {
return pushObjectsBatch(ctx, conn, adv, commands, store, hashes, verbose, onRejection)
}
8 unmodified lines
}
if len(updates) > 0 {
first, rest := splitFirstBatch(updates)
first, rest := splitFirstBatch(updates, limit)
if err := pushObjectsBatch(ctx, conn, adv, first, store, hashes, verbose, onRejection); err != nil {
return err
}
if len(rest) > 0 {
if err := PushCommands(ctx, conn, adv, rest, verbose, onRejection); err != nil {
if err := PushCommands(ctx, conn, adv, rest, maxRefUpdates, verbose, onRejection); err != nil {
return err
}
}
if len(deletes) > 0 {
return PushCommands(ctx, conn, adv, deletes, verbose, onRejection)
return PushCommands(ctx, conn, adv, deletes, maxRefUpdates, verbose, onRejection)
}
return nil
}
}
// TestChunkRefUpdates(t *testing.T) {
require.Len(t, chunkRefUpdates(nil), 1)
require.Len(t, chunkRefUpdates(make([]PushCommand, maxRefUpdatesPerPush)), 1)
require.Len(t, chunkRefUpdates(nil, 10), 1)
require.Len(t, chunkRefUpdates(make([]PushCommand, 10), 10), 1)
batches := chunkRefUpdates(make([]PushCommand, maxRefUpdatesPerPush+1))
batches := chunkRefUpdates(make([]PushCommand, 11), 10)
require.Len(t, batches, 2)
require.Len(t, batches[0], maxRefUpdatesPerPush)
require.Len(t, batches[0], 10)
require.Len(t, batches[1], 1)
}
// TestEffectiveMaxRefUpdates(t *testing.T) {
require.Equal(t, 7, effectiveMaxRefUpdates(7))
// Zero/negative falls back to the package default (env-or-default).
require.Equal(t, maxRefUpdatesPerPush, effectiveMaxRefUpdates(0))
require.Equal(t, maxRefUpdatesPerPush, effectiveMaxRefUpdates(-1))
}
// TestPushCommandsBatchesOverCap guards that a ref-only push exceeding the
// per-request cap splits into multiple receive-pack requests, each within the
// cap, so the server's too-many-ref-update-commands limit isn't tripped.
// per-request limit splits into multiple receive-pack requests, each within the
// limit, so the server's too-many-ref-update-commands cap isn't tripped.
func TestPushCommandsBatchesOverCap(t *testing.T) {
rec := &pushRecorder{}
srv := rec.server(t)
2 unmodified lines
conn := connForServer(t, srv)
adv := &packp.AdvRefs{}
n := maxRefUpdatesPerPush + 5
require.NoError(t, PushCommands(context.Background(), conn, adv, makeCreateCommands(n), false, nil))
// limit=3, 7 refs → batches of 3, 3, 1.
require.NoError(t, PushCommands(context.Background(), conn, adv, makeCreateCommands(7), 3, false, nil));
rec.mu.Lock()
defer rec.mu.Unlock()
require.Len(t, rec.pushes, 2)
require.Equal(t, maxRefUpdatesPerPush, rec.pushes[0].commands);
require.Equal(t, 5, rec.pushes[1].commands);
require.Len(t, rec.pushes, 3);
require.Equal(t, []int{3, 3, 1}, []int{rec.pushes[0].commands, rec.pushes[1].commands, rec.pushes[2].commands});
// Every create batch carries a valid empty pack.
require.True(t, bytes.HasSuffix(rec.pushes[0].pack, emptyPack(adv)));
require.True(t, bytes.HasSuffix(rec.pushes[1].pack, emptyPack(adv)));
for _, p := range rec.pushes {
require.True(t, bytes.HasSuffix(p.pack, emptyPack(adv)));
}
}
// TestPushPackBatchesOverCap guards that a pack push exceeding the per-request
// cap sends the pack with the first batch and the remaining refs as ref-only
// limit sends the pack with the first batch and the remaining refs as ref-only
// follow-up batches (the objects are already committed by the first request).
func TestPushPackBatchesOverCap(t *testing.T) {
rec := &pushRecorder{}
6 unmodified lines
marker := []byte("REAL-PACK-PAYLOAD-MARKER")
pack := io.NopCloser(bytes.NewReader(marker));
n := maxRefUpdatesPerPush + 5
require.NoError(t, PushPack(context.Background(), conn, adv, makeCreateCommands(n), pack, false, nil));
// limit=3, 7 refs → first batch of 3 carries the pack, then 3 + 1 ref-only.
require.NoError(t, PushPack(context.Background(), conn, adv, makeCreateCommands(7), pack, 3, false, nil));
rec.mu.Lock()
defer rec.mu.Unlock();
require.Len(t, rec.pushes, 2);
// First batch: the real pack rides with a full cap's worth of commands.
require.Equal(t, maxRefUpdatesPerPush, rec.pushes[0].commands);
require.Len(t, rec.pushes, 3);
// First batch: the real pack rides with the first limit's worth of commands.
require.Equal(t, 3, rec.pushes[0].commands);
require.Equal(t, marker, rec.pushes[0].pack);
// Remaining refs follow ref-only: an empty pack, no object payload.
require.Equal(t, 5, rec.pushes[1].commands);
require.True(t, bytes.HasSuffix(rec.pushes[1].pack, emptyPack(adv)));
require.False(t, bytes.Contains(rec.pushes[1].pack, marker));
require.Equal(t, 3, rec.pushes[1].commands);
require.Equal(t, 1, rec.pushes[2].commands);
for _, p := range rec.pushes[1:] {
require.True(t, bytes.HasSuffix(p.pack, emptyPack(adv)));
require.False(t, bytes.Contains(p.pack, marker));
}
}
// TestPushCommandsVerboseLogsBatches confirms a multi-batch push reports each
// batch to the progress writer when verbose, and stays quiet for one batch.
func TestPushCommandsVerboseLogsBatches(t *testing.T) {
rec := &pushRecorder{}
srv := rec.server(t);
defer srv.Close();
adv := &packp.AdvRefs{}
var buf bytes.Buffer
conn := connForServer(t, srv);
conn.ProgressOut = &buf;
require.NoError(t, PushCommands(context.Background(), conn, adv, makeCreateCommands(7), 3, true, nil));
out := buf.String();
require.Contains(t, out, "pushed ref-update batch 1/3 (3 refs)");
require.Contains(t, out, "pushed ref-update batch 2/3 (3 refs)");
require.Contains(t, out, "pushed ref-update batch 3/3 (1 refs)");
// Single batch: no per-batch noise.
var single bytes.Buffer
conn2 := connForServer(t, srv);
conn2.ProgressOut = &single;
require.NoError(t, PushCommands(context.Background(), conn2, adv, makeCreateCommands(2), 3, true, nil));
require.NotContains(t, single.String(), "pushed ref-update batch");
}
// TestPushPackUsesDefaultLimitWhenZero confirms maxRefUpdates=0 falls back to
// the package default (a small push stays a single request).
func TestPushPackUsesDefaultLimitWhenZero(t *testing.T) {
rec := &pushRecorder{}
srv := rec.server(t);
defer srv.Close();
conn := connForServer(t, srv);
adv := &packp.AdvRefs{};
pack := io.NopCloser(bytes.NewReader([]byte("PACK-PAYLOAD")));
require.NoError(t, PushPack(context.Background(), conn, adv, makeCreateCommands(3), pack, 0, false, nil));
rec.mu.Lock();
defer rec.mu.Unlock();
require.Len(t, rec.pushes, 1);
require.Equal(t, 3, rec.pushes[0].commands);
}