Add bootstrap pack size limit · Entire
Add bootstrap pack size limit
55f5f45→main·
Soph·3mo ago·5 files·+75 added/-1 removed
Sessions
72f07f367d81View transcript
Changes
5
MREADME.md+9
cmd/git-sync
Mmain.go+2/-1
docs
Mbootstrap.md+6
internal/syncer
Mintegration_test.go+31
- Msyncer.go+27
67 unmodified lines
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
67 unmodified lines
https://github.com/target-org/target-repo.git
```
Add `--max-pack-bytes` to abort bootstrap if the streamed source pack grows past a safety threshold:
```bash
go run ./cmd/git-sync bootstrap \
--max-pack-bytes 104857600 \
<source-url> \
<target-url>
```
Sync specific branches:
```bash
MREADME.md+9
137 unmodified lines
138
139
140
141
142
143
144
201 unmodified lines
346
347
348
348
349
350
351
352
137 unmodified lines
fs.BoolVar(&cfg.IncludeTags, "tags", false, "mirror tags")
fs.BoolVar(&cfg.ShowStats, "stats", false, "print transfer statistics")
fs.BoolVar(&jsonOutput, "json", false, "print JSON output")
fs.Int64Var(&cfg.MaxPackBytes, "max-pack-bytes", 0, "abort bootstrap if the streamed source pack exceeds this many bytes")
fs.StringVar(&cfg.ProtocolMode, "protocol", envOr("GITSYNC_PROTOCOL", "auto"), "protocol mode: auto, v1, or v2")
fs.BoolVar(&cfg.Verbose, "v", false, "verbose logging")
201 unmodified lines
}
func usageError(message string) error {
usage := "usage:\n git-sync sync [flags] <source-url> <target-url>\n git-sync plan [flags] <source-url> <target-url>\n git-sync bootstrap [flags] <source-url> <target-url>\n git-sync probe [flags] <source-url> [target-url]\n git-sync fetch [flags] <source-url>\n\nsync/plan flags:\n --branch main,dev\n --map main:stable\n --tags\n --force\n --prune\n --stats\n --json\n --protocol auto|v1|v2\n --source-token ...\n --target-token ...\n --source-username git\n --target-username git\n --source-bearer-token ...\n --target-bearer-token ...\n -v\n\nbootstrap flags:\n --branch main,dev\n --map main:stable\n --tags\n --stats\n --json\n --protocol auto|v1|v2\n --source-token ...\n --target-token ...\n --source-username git\n --target-username git\n --source-bearer-token ...\n --target-bearer-token ...\n -v\n\nprobe flags:\n --tags\n --stats\n --json\n --protocol auto|v1|v2\n --source-token ...\n --source-username git\n --source-bearer-token ...\n --target-token ...\n --target-username git\n --target-bearer-token ...\n\nfetch flags:\n --branch main,dev\n --tags\n --stats\n --json\n --protocol auto|v1|v2\n --have-ref main\n --have <hash>\n\n"
usage := "usage:\n git-sync sync [flags] <source-url> <target-url>\n git-sync plan [flags] <source-url> <target-url>\n git-sync bootstrap [flags] <source-url> <target-url>\n git-sync probe [flags] <source-url> [target-url]\n git-sync fetch [flags] <source-url>\n\nsync/plan flags:\n --branch main,dev\n --map main:stable\n --tags\n --force\n --prune\n --stats\n --json\n --protocol auto|v1|v2\n --source-token ...\n --target-token ...\n --source-username git\n --target-username git\n --source-bearer-token ...\n --target-bearer-token ...\n -v\n\nbootstrap flags:\n --branch main,dev\n --map main:stable\n --tags\n --stats\n --json\n --protocol auto|v1|v2\n --source-token ...\n --target-token ...\n --source-username git\n --target-username git\n --source-bearer-token ...\n --target-bearer-token ...\n -v\n\nprobe flags:\n --tags\n --stats\n --json\n --protocol auto|v1|v2\n --source-token ...\n --source-username git\n --source-bearer-token ...\n --target-token ...\n --target-username git\n --target-bearer-token ...\n\nfetch flags:\n --branch main,dev\n --tags\n --stats\n --json\n --protocol auto|v1|v2\n --have-ref main\n --have <hash>\n\n"
if message == "" {
return errors.New(strings.TrimSpace(usage))
}
}
Mcmd/git-sync/main.go+2/-1
52 unmodified lines
53 54 55 56 57 58 59 80 unmodified lines
140 141 142 143 144 145 146 147 148 149 150
52 unmodified lines
--branch--map--tags--max-pack-bytes--stats--json--protocol auto|v1|v280 unmodified linesadd better operator output for large initial transfers
add safety thresholds for advertised/fetched bytes
Progress:
- explicit mapped refs are supported
--max-pack-bytesprovides a first safety threshold for the streamed source pack during bootstrap
Phase 3:
- investigate hybrid behavior: relay when the target is empty, otherwise fail fast into normal
sync
Mdocs/bootstrap.md+6
222 unmodified lines
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
222 unmodified lines
}
}
func TestBootstrap_IntegrationPackLimit(t *testing.T) {
sourceRepo, sourceFS := newSourceRepo(t)
makeCommits(t, sourceRepo, sourceFS, 2)
targetRepo, err := git.Init(memory.NewStorage(), nil)
if err != nil {
t.Fatalf("init target repo: %%v", err)
}
sourceServer := newSmartHTTPRepoServerV2(t, sourceRepo)
targetServer := newSmartHTTPRepoServer(t, targetRepo)
defer sourceServer.Close()
defer targetServer.Close()
_, err = Bootstrap(context.Background(), Config{
Source: Endpoint{URL: sourceServer.RepoURL()},
Target: Endpoint{URL: targetServer.RepoURL()},
ProtocolMode: protocolModeAuto,
MaxPackBytes: 32,
})
if err == nil {
t.Fatalf("expected bootstrap failure when pack exceeds threshold")
}
if !strings.Contains(err.Error(), "max-pack-bytes") {
t.Fatalf("expected max-pack-bytes error, got %%v", err)
}
if _, err := targetRepo.Reference(plumbing.NewBranchReferenceName(testBranch), true); err != plumbing.ErrReferenceNotFound {
t.Fatalf("expected target branch to remain absent, got %%v", err)
}
}
func TestRun_IntegrationResyncFetchesLessFromSource(t *testing.T) {
sourceRepo, sourceFS := newSourceRepo(t)
makeCommits(t, sourceRepo, sourceFS, 10)
Minternal/syncer/integration_test.go+31
56 unmodified lines
57
58
59
60
61
62
63
433 unmodified lines
497
498
499
500
501
502
503
798 unmodified lines
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
56 unmodified lines
ShowStats bool
Force bool
Prune bool
MaxPackBytes int64
ProtocolMode string
}
433 unmodified lines
return result, fmt.Errorf("fetch source pack: %%w", err)
}
defer packReader.Close()
packReader = limitPackReadCloser(packReader, cfg.MaxPackBytes)
if err := pushPackToTarget(ctx, targetConn, targetAdv, plans, packReader, cfg.Verbose); err != nil {
return result, fmt.Errorf("push target refs: %%w", err)
}
798 unmodified lines
return r.closeFn()
}
func limitPackReadCloser(r io.ReadCloser, maxBytes int64) io.ReadCloser {
if maxBytes <= 0 {
return r
}
return &packLimitReadCloser{
ReadCloser: r,
maxBytes: maxBytes,
}
}
type packLimitReadCloser struct {
io.ReadCloser
maxBytes int64
read int64
}
func (r *packLimitReadCloser) Read(p []byte) (int, error) {
n, err := r.ReadCloser.Read(p)
r.read += int64(n)
if r.read > r.maxBytes {
return n, fmt.Errorf("source pack exceeded max-pack-bytes limit (%%d)", r.maxBytes)
}
return n, err
}
func branchMapFromRefHashMap(refs map[plumbing.ReferenceName]plumbing.Hash) map[string]plumbing.Hash {
branches := make(map[string]plumbing.Hash)
for name, hash := range refs {