# Add memory measurement output `9435d85`→[main](/content/gh/entireio/git-sync/commits/main/index.html)· Soph·3mo ago·4 files·+142 added/-1 removed ## Sessions 4805c08b89a8View transcript ## Changes 4 - MREADME.md+12 - cmd/git-sync - Mmain.go+5/-1 - Mmain\_test.go+12 - internal/syncer - Msyncer.go+113 ```bash Add `--measure-memory` to `bootstrap`, `sync`, `plan`, `probe`, or `fetch` to sample elapsed time and Go heap usage: go run ./cmd/git-sync bootstrap \ --measure-memory \ --json \ \ ``` That is the intended way to compare the bootstrap relay path against the normal sync path on the same fixture or test repo. Sync specific branches: ```bash ``` ```plaintext MREADME.md+12 ``` ```plaintext 67 unmodified lines ``` ```plaintext 68 69 70 71 72 73 74 63 unmodified lines ``` ```plaintext 138 139 140 141 142 143 144 54 unmodified lines ``` ```plaintext 199 200 201 202 203 204 205 39 unmodified lines ``` ```plaintext 245 246 247 248 249 250 251 98 unmodified lines ``` ```plaintext 350 351 352 349 353 354 355 356 ``` ```plaintext 67 unmodified lines ``` ```go fs.BoolVar(&cfg.Force, "force", false, "allow non-fast-forward branch updates and retarget tags") fs.BoolVar(&cfg.Prune, "prune", false, "delete managed target refs that no longer exist on source") fs.BoolVar(&cfg.ShowStats, "stats", false, "print transfer statistics") fs.BoolVar(&cfg.MeasureMemory, "measure-memory", false, "sample elapsed time and Go heap usage") fs.BoolVar(&jsonOutput, "json", false, "print JSON output") fs.StringVar(&cfg.ProtocolMode, "protocol", envOr("GITSYNC_PROTOCOL", "auto"), "protocol mode: auto, v1, or v2") fs.BoolVar(&cfg.Verbose, "v", false, "verbose logging") ``` ```plaintext 63 unmodified lines ``` ```go fs.Var(&mappings, "map", "ref mapping in src:dst form; short names map branches, full refs map exact refs") fs.BoolVar(&cfg.IncludeTags, "tags", false, "mirror tags") fs.BoolVar(&cfg.ShowStats, "stats", false, "print transfer statistics") fs.BoolVar(&cfg.MeasureMemory, "measure-memory", false, "sample elapsed time and Go heap usage") fs.BoolVar(&jsonOutput, "json", false, "print JSON output") fs.Int64Var(&cfg.MaxPackBytes, "max-pack-bytes", 0, "abort bootstrap if the streamed source pack exceeds this many bytes") fs.StringVar(&cfg.ProtocolMode, "protocol", envOr("GITSYNC_PROTOCOL", "auto"), "protocol mode: auto, v1, or v2") ``` ```plaintext 54 unmodified lines ``` ```go fs.BoolVar(&cfg.IncludeTags, "tags", false, "include tag ref prefixes in probe") fs.StringVar(&cfg.ProtocolMode, "protocol", envOr("GITSYNC_PROTOCOL", "auto"), "protocol mode: auto, v1, or v2") fs.BoolVar(&cfg.ShowStats, "stats", false, "print transfer statistics") fs.BoolVar(&cfg.MeasureMemory, "measure-memory", false, "sample elapsed time and Go heap usage") fs.BoolVar(&jsonOutput, "json", false, "print JSON output") ``` ```go if err := fs.Parse(args); err != nil { 39 unmodified lines ``` ```go fs.BoolVar(&cfg.IncludeTags, "tags", false, "include tags in the fetch request") fs.StringVar(&cfg.ProtocolMode, "protocol", envOr("GITSYNC_PROTOCOL", "auto"), "protocol mode: auto, v1, or v2") fs.BoolVar(&cfg.ShowStats, "stats", false, "print transfer statistics") fs.BoolVar(&cfg.MeasureMemory, "measure-memory", false, "sample elapsed time and Go heap usage") fs.BoolVar(&jsonOutput, "json", false, "print JSON output") fs.Var(&haveRefs, "have-ref", "source ref name to advertise as have; short names map to branches") fs.Var(&haveHashesRaw, "have", "explicit object hash to advertise as have") ``` ```plaintext 98 unmodified lines ``` ```go func usageError(message string) error { usage := "usage:\n git-sync sync [flags] \n git-sync plan [flags] \n git-sync bootstrap [flags] \n git-sync probe [flags] [target-url]\n git-sync fetch [flags] \n\nsync/plan flags:\n --branch main,dev\n --map main:stable\n --tags\n --force\n --prune\n --stats\n --json\n --protocol auto|v1|v2\n --source-token ...\n --target-token ...\n --source-username git\n --target-username git\n --source-bearer-token ...\n --target-bearer-token ...\n -v\n\nbootstrap flags:\n --branch main,dev\n --map main:stable\n --tags\n --max-pack-bytes 104857600\n --stats\n --json\n --protocol auto|v1|v2\n --source-token ...\n --target-token ...\n --source-username git\n --target-username git\n --source-bearer-token ...\n --target-bearer-token ...\n -v\n\nprobe flags:\n --tags\n --stats\n --json\n --protocol auto|v1|v2\n --source-token ...\n --source-username git\n --source-bearer-token ...\n --target-token ...\n --target-username git\n --target-bearer-token ...\n\nfetch flags:\n --branch main,dev\n --tags\n --stats\n --json\n --protocol auto|v1|v2\n --have-ref main\n --have \n --source-token ...\n --source-username git\n --source-bearer-token ...\n" usage := "usage:\n git-sync sync [flags] \n git-sync plan [flags] \n git-sync bootstrap [flags] \n git-sync probe [flags] [target-url]\n git-sync fetch [flags] \n\nsync/plan flags:\n --branch main,dev\n --map main:stable\n --tags\n --force\n --prune\n --stats\n --measure-memory\n --json\n --protocol auto|v1|v2\n --source-token ...\n --target-token ...\n --source-username git\n --target-username git\n --source-bearer-token ...\n --target-bearer-token ...\n -v\n" if message == "" { return errors.New(strings.TrimSpace(usage)) } } ``` ```plaintext Mcmd/git-sync/main.go+5/-1 ``` ```plaintext 38 unmodified lines ``` ```plaintext 39 40 41 42 43 44 45 46 47 48 49 50 51 52 13 unmodified lines ``` ```plaintext 66 67 68 69 70 71 72 73 74 75 ``` ```plaintext 38 unmodified lines ``` ```go Wants: []syncer.RefInfo{{Name: "refs/heads/main", Hash: plumbing.NewHash("1111111111111111111111111111111111111111")}}, Haves: []plumbing.Hash{plumbing.NewHash("2222222222222222222222222222222222222222")}, FetchedObjects: 42, Measurement: syncer.Measurement{ Enabled: true, ElapsedMillis: 12, PeakAllocBytes: 100, PeakHeapInuseBytes: 200, TotalAllocBytes: 300, GCCount: 1, }, ```go if err != nil { t.Fatalf("marshalOutput returned error: %v", err) } ``` ```plaintext 13 unmodified lines ``` ```go if got := decoded["fetched_objects"]; got != float64(42) { t.Fatalf("unexpected fetched_objects: %#v", got) } measurement, ok := decoded["measurement"].(map[string]any) if !ok || measurement["elapsed_millis"] != float64(12) { t.Fatalf("unexpected measurement: %#v", decoded["measurement"]) } wants, ok := decoded["wants"].([]any) if !ok || len(wants) != 1 { t.Fatalf("unexpected wants: %#v", decoded["wants"]) } ``` ```plaintext Mcmd/git-sync/main_test.go+12 ``` ```plaintext 9 unmodified lines ``` ```plaintext 10 11 12 13 14 15 16 17 18 19 20 37 unmodified lines ``` ```plaintext 58 59 60 61 62 63 64 36 unmodified lines ``` ```plaintext 101 102 103 104 105 106 107 7 unmodified lines ``` ```plaintext 115 116 117 118 119 120 121 9 unmodified lines ``` ```plaintext 131 132 133 134 135 136 137 11 unmodified lines ``` ```plaintext 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 37 unmodified lines ``` ```plaintext 201 202 203 204 205 206 207 7 unmodified lines ``` ```plaintext 215 216 217 218 219 220 221 34 unmodified lines ``` ```plaintext 256 257 258 259 260 261 262 263 264 265 266 267 37 unmodified lines ``` ```plaintext 305 306 307 308 309 310 311 312 313 314 315 316 27 unmodified lines ``` ```plaintext 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 55 unmodified lines ``` ```plaintext 416 417 418 419 420 421 422 39 unmodified lines ``` ```plaintext 462 463 464 465 466 467 468 469 470 471 472 473 52 unmodified lines ``` ```plaintext 526 527 528 529 530 531 532 13 unmodified lines ``` ```plaintext 546 547 548 549 550 551 552 553 554 555 556 557 37 unmodified lines ``` ```plaintext 595 596 597 598 599 600 601 8 unmodified lines ``` ```plaintext 610 611 612 613 614 615 616 617 618 619 620 621 622 623 67 unmodified lines ``` ```plaintext 691 692 693 694 695 696 697 676 unmodified lines ``` ```plaintext 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 ``` ```go "net/http" "os" "os/exec" "runtime" "sort" "strings" "sync" "time" git "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" ``` ```plaintext DryRun bool Verbose bool ShowStats bool MeasureMemory bool Force bool Prune bool MaxPackBytes int64 ``` ```go Deleted int `json:"deleted"` DryRun bool `json:"dry_run"` Stats Stats `json:"stats"` Measurement Measurement `json:"measurement"` Protocol string `json:"protocol"` ``` ```go type RefInfo struct { Haves []plumbing.Hash `json:"haves"` FetchedObjects int `json:"fetched_objects"` Stats Stats `json:"stats"` Measurement Measurement `json:"measurement"` } ``` ```go type Stats struct { Commands int `json:"commands"` } ``` ```go type Measurement struct { Enabled bool `json:"enabled"` ElapsedMillis int64 `json:"elapsed_millis"` PeakAllocBytes uint64 `json:"peak_alloc_bytes"` PeakHeapInuseBytes uint64 `json:"peak_heap_inuse_bytes"` TotalAllocBytes uint64 `json:"total_alloc_bytes"` GCCount uint32 `json:"gc_count"` } ``` ```go func (p BranchPlan) MarshalJSON() ([]byte, error) { type branchPlanJSON struct { Branch string `json:"branch"` Haves []string `json:"haves"` FetchedObjects int `json:"fetched_objects"` Stats Stats `json:"stats"` Measurement Measurement `json:"measurement"` } haves := make([]string, 0, len(r.Haves)) for _, hash := range r.Haves { ``` ```plaintext 34 unmodified lines ``` ```go if r.Measurement.Enabled { lines = append(lines, fmt.Sprintf( "measurement: elapsed-ms=%d peak-alloc-bytes=%d peak-heap-inuse-bytes=%d total-alloc-bytes=%d gc-count=%d", r.Measurement.ElapsedMillis, r.Measurement.PeakAllocBytes, r.Measurement.PeakHeapInuseBytes, r.Measurement.TotalAllocBytes, r.Measurement.GCCount, )) } return lines } ``` ```go func Run(ctx context.Context, cfg Config) (Result, error) { measurementDone := startMeasurement(cfg.MeasureMemory) if cfg.ProtocolMode == "" { cfg.ProtocolMode = protocolModeAuto } ``` ```go Plans: plans, DryRun: cfg.DryRun, Stats: stats.snapshot(), Measurement: measurementDone(), Protocol: sourceService.protocol, } ``` ```go result.Stats = stats.snapshot() result.Measurement = measurementDone() return result, nil } ``` ```go func Bootstrap(ctx context.Context, cfg Config) (Result, error) { measurementDone := startMeasurement(cfg.MeasureMemory) if cfg.ProtocolMode == "" { cfg.ProtocolMode = protocolModeAuto } ``` ```go result := Result{ Plans: plans, Stats: stats.snapshot(), Measurement: measurementDone(), Protocol: sourceService.protocol, } ``` ```go result.Pushed = len(plans) result.Stats = stats.snapshot() result.Measurement = measurementDone() return result, nil } ``` ```go func Probe(ctx context.Context, cfg Config) (ProbeResult, error) { measurementDone := startMeasurement(cfg.MeasureMemory) if cfg.ProtocolMode == "" { cfg.ProtocolMode = protocolModeAuto } ``` ```go Capabilities: sourceCapabilities(service), Refs: refInfos, Stats: stats.snapshot(), Measurement: measurementDone(), } ``` ```go if cfg.Target.URL != "" { result.TargetURL = cfg.Target.URL result.TargetCaps = advCapabilities(targetAdv) result.Stats = stats.snapshot() result.Measurement = measurementDone() } ``` ```go return result, nil } ``` ```go func Fetch(ctx context.Context, cfg Config, haveRefs []string, haveHashes []plumbing.Hash) (FetchResult, error) { measurementDone := startMeasurement(cfg.MeasureMemory) if cfg.ProtocolMode == "" { cfg.ProtocolMode = protocolModeAuto } ``` ```go Haves: sortedUniqueHashes(mapsRefValues(targetRefMap)), FetchedObjects: objectCount, Stats: stats.snapshot(), Measurement: measurementDone(), }, nil } ``` ```plaintext 676 unmodified lines ``` ```go func startMeasurement(enabled bool) func() Measurement { if !enabled { return func() Measurement { return Measurement{} } } ``` ```go start := time.Now() var startStats runtime.MemStats runtime.ReadMemStats(&startStats) ``` ```go done := make(chan struct{}) var ( mu sync.Mutex peakAlloc = startStats.Alloc peakHeapInuse = startStats.HeapInuse result Measurement ) ``` ```go go func() { ticker := time.NewTicker(10 * time.Millisecond) defer ticker.Stop() for { select { case <-done: return case <-ticker.C: var current runtime.MemStats runtime.ReadMemStats(¤t) mu.Lock() if current.Alloc > peakAlloc { peakAlloc = current.Alloc } if current.HeapInuse > peakHeapInuse { peakHeapInuse = current.HeapInuse } mu.Unlock() } } } ``` ```go var once sync.Once return func() Measurement { once.Do(func() { close(done) var endStats runtime.MemStats runtime.ReadMemStats(&endStats) mu.Lock() if endStats.Alloc > peakAlloc { peakAlloc = endStats.Alloc } if endStats.HeapInuse > peakHeapInuse { peakHeapInuse = endStats.HeapInuse } result = Measurement{ Enabled: true, ElapsedMillis: time.Since(start).Milliseconds(), PeakAllocBytes: peakAlloc, PeakHeapInuseBytes: peakHeapInuse, TotalAllocBytes: endStats.TotalAlloc - startStats.TotalAlloc, GCCount: endStats.NumGC - startStats.NumGC, } mu.Unlock() }) return result } } ``` ```go func branchMapFromRefHashMap(refs map[plumbing.ReferenceName]plumbing.Hash) map[string]plumbing.Hash { branches := make(map[string]plumbing.Hash) for name, hash := range refs { ```