Add memory measurement output · Entire

Add memory measurement output

9435d85→main·

Soph·3mo ago·4 files·+142 added/-1 removed

Sessions

4805c08b89a8View transcript

Changes

4

Add `--measure-memory` to `bootstrap`, `sync`, `plan`, `probe`, or `fetch` to sample elapsed time and Go heap usage:

go run ./cmd/git-sync bootstrap \
  --measure-memory \
  --json \
  <source-url> \
  <target-url>

That is the intended way to compare the bootstrap relay path against the normal sync path on the same fixture or test repo.

Sync specific branches:



MREADME.md+12

67 unmodified lines

68
69
70
71
72
73
74
63 unmodified lines

138
139
140
141
142
143
144
54 unmodified lines

199
200
201
202
203
204
205
39 unmodified lines

245
246
247
248
249
250
251
98 unmodified lines

350
351
352
349
353
354
355
356

67 unmodified lines

fs.BoolVar(&cfg.Force, "force", false, "allow non-fast-forward branch updates and retarget tags")
fs.BoolVar(&cfg.Prune, "prune", false, "delete managed target refs that no longer exist on source")
fs.BoolVar(&cfg.ShowStats, "stats", false, "print transfer statistics")
fs.BoolVar(&cfg.MeasureMemory, "measure-memory", false, "sample elapsed time and Go heap usage")
fs.BoolVar(&jsonOutput, "json", false, "print JSON output")
fs.StringVar(&cfg.ProtocolMode, "protocol", envOr("GITSYNC_PROTOCOL", "auto"), "protocol mode: auto, v1, or v2")
fs.BoolVar(&cfg.Verbose, "v", false, "verbose logging")

63 unmodified lines

fs.Var(&mappings, "map", "ref mapping in src:dst form; short names map branches, full refs map exact refs")
fs.BoolVar(&cfg.IncludeTags, "tags", false, "mirror tags")
fs.BoolVar(&cfg.ShowStats, "stats", false, "print transfer statistics")
fs.BoolVar(&cfg.MeasureMemory, "measure-memory", false, "sample elapsed time and Go heap usage")
fs.BoolVar(&jsonOutput, "json", false, "print JSON output")
fs.Int64Var(&cfg.MaxPackBytes, "max-pack-bytes", 0, "abort bootstrap if the streamed source pack exceeds this many bytes")
fs.StringVar(&cfg.ProtocolMode, "protocol", envOr("GITSYNC_PROTOCOL", "auto"), "protocol mode: auto, v1, or v2")

54 unmodified lines

fs.BoolVar(&cfg.IncludeTags, "tags", false, "include tag ref prefixes in probe")
fs.StringVar(&cfg.ProtocolMode, "protocol", envOr("GITSYNC_PROTOCOL", "auto"), "protocol mode: auto, v1, or v2")
fs.BoolVar(&cfg.ShowStats, "stats", false, "print transfer statistics")
fs.BoolVar(&cfg.MeasureMemory, "measure-memory", false, "sample elapsed time and Go heap usage")
fs.BoolVar(&jsonOutput, "json", false, "print JSON output")

if err := fs.Parse(args); err != nil {
39 unmodified lines

fs.BoolVar(&cfg.IncludeTags, "tags", false, "include tags in the fetch request")
fs.StringVar(&cfg.ProtocolMode, "protocol", envOr("GITSYNC_PROTOCOL", "auto"), "protocol mode: auto, v1, or v2")
fs.BoolVar(&cfg.ShowStats, "stats", false, "print transfer statistics")
fs.BoolVar(&cfg.MeasureMemory, "measure-memory", false, "sample elapsed time and Go heap usage")
fs.BoolVar(&jsonOutput, "json", false, "print JSON output")
fs.Var(&haveRefs, "have-ref", "source ref name to advertise as have; short names map to branches")
fs.Var(&haveHashesRaw, "have", "explicit object hash to advertise as have")

98 unmodified lines

func usageError(message string) error {
  usage := "usage:\n  git-sync sync [flags] <source-url> <target-url>\n  git-sync plan [flags] <source-url> <target-url>\n  git-sync bootstrap [flags] <source-url> <target-url>\n  git-sync probe [flags] <source-url> [target-url]\n  git-sync fetch [flags] <source-url>\n\nsync/plan flags:\n  --branch main,dev\n  --map main:stable\n  --tags\n  --force\n  --prune\n  --stats\n  --json\n  --protocol auto|v1|v2\n  --source-token ...\n  --target-token ...\n  --source-username git\n  --target-username git\n  --source-bearer-token ...\n  --target-bearer-token ...\n  -v\n\nbootstrap flags:\n  --branch main,dev\n  --map main:stable\n  --tags\n  --max-pack-bytes 104857600\n  --stats\n  --json\n  --protocol auto|v1|v2\n  --source-token ...\n  --target-token ...\n  --source-username git\n  --target-username git\n  --source-bearer-token ...\n  --target-bearer-token ...\n  -v\n\nprobe flags:\n  --tags\n  --stats\n  --json\n  --protocol auto|v1|v2\n  --source-token ...\n  --source-username git\n  --source-bearer-token ...\n  --target-token ...\n  --target-username git\n  --target-bearer-token ...\n\nfetch flags:\n  --branch main,dev\n  --tags\n  --stats\n  --json\n  --protocol auto|v1|v2\n  --have-ref main\n  --have <hash>\n  --source-token ...\n  --source-username git\n  --source-bearer-token ...\n"
  usage := "usage:\n  git-sync sync [flags] <source-url> <target-url>\n  git-sync plan [flags] <source-url> <target-url>\n  git-sync bootstrap [flags] <source-url> <target-url>\n  git-sync probe [flags] <source-url> [target-url]\n  git-sync fetch [flags] <source-url>\n\nsync/plan flags:\n  --branch main,dev\n  --map main:stable\n  --tags\n  --force\n  --prune\n  --stats\n  --measure-memory\n  --json\n  --protocol auto|v1|v2\n  --source-token ...\n  --target-token ...\n  --source-username git\n  --target-username git\n  --source-bearer-token ...\n  --target-bearer-token ...\n  -v\n"  
  if message == "" {
    return errors.New(strings.TrimSpace(usage))
  }
}

Mcmd/git-sync/main.go+5/-1

38 unmodified lines

39
40
41
42
43
44
45
46
47
48
49
50
51
52
13 unmodified lines

66
67
68
69
70
71
72
73
74
75

38 unmodified lines

Wants:          []syncer.RefInfo{{Name: "refs/heads/main", Hash: plumbing.NewHash("1111111111111111111111111111111111111111")}},
Haves:          []plumbing.Hash{plumbing.NewHash("2222222222222222222222222222222222222222")},
FetchedObjects: 42,
Measurement: syncer.Measurement{
    Enabled:            true,
    ElapsedMillis:      12,
    PeakAllocBytes:     100,
    PeakHeapInuseBytes: 200,
    TotalAllocBytes:    300,
    GCCount:            1,
},

```go
if err != nil {
    t.Fatalf("marshalOutput returned error: %v", err)
}

13 unmodified lines

if got := decoded["fetched_objects"]; got != float64(42) {
    t.Fatalf("unexpected fetched_objects: %#v", got)
}
measurement, ok := decoded["measurement"].(map[string]any)
if !ok || measurement["elapsed_millis"] != float64(12) {
    t.Fatalf("unexpected measurement: %#v", decoded["measurement"])
}
wants, ok := decoded["wants"].([]any)
if !ok || len(wants) != 1 {
    t.Fatalf("unexpected wants: %#v", decoded["wants"])
}

Mcmd/git-sync/main_test.go+12

9 unmodified lines

10
11
12
13
14
15
16
17
18
19
20
37 unmodified lines

58
59
60
61
62
63
64
36 unmodified lines

101
102
103
104
105
106
107
7 unmodified lines

115
116
117
118
119
120
121
9 unmodified lines

131
132
133
134
135
136
137
11 unmodified lines

149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
37 unmodified lines

201
202
203
204
205
206
207
7 unmodified lines

215
216
217
218
219
220
221
34 unmodified lines

256
257
258
259
260
261
262
263
264
265
266
267
37 unmodified lines

305
306
307
308
309
310
311
312
313
314
315
316
27 unmodified lines

344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
55 unmodified lines

416
417
418
419
420
421
422
39 unmodified lines

462
463
464
465
466
467
468
469
470
471
472
473
52 unmodified lines

526
527
528
529
530
531
532
13 unmodified lines

546
547
548
549
550
551
552
553
554
555
556
557
37 unmodified lines

595
596
597
598
599
600
601
8 unmodified lines

610
611
612
613
614
615
616
617
618
619
620
621
622
623
67 unmodified lines

691
692
693
694
695
696
697
676 unmodified lines

1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445

"net/http"
"os"
"os/exec"
"runtime"
"sort"
"strings"
"sync"
"time"

git "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"

DryRun       bool
Verbose      bool
ShowStats    bool
MeasureMemory bool
Force        bool
Prune        bool
MaxPackBytes int64

Deleted  int          `json:"deleted"`
DryRun   bool         `json:"dry_run"`
Stats    Stats        `json:"stats"`
Measurement Measurement `json:"measurement"`
Protocol string       `json:"protocol"`

type RefInfo struct {

Haves          []plumbing.Hash `json:"haves"`
FetchedObjects int             `json:"fetched_objects"`
Stats          Stats           `json:"stats"`
Measurement    Measurement     `json:"measurement"`
}

type Stats struct {

Commands      int    `json:"commands"`
}

type Measurement struct {
Enabled            bool   `json:"enabled"`
ElapsedMillis      int64  `json:"elapsed_millis"`
PeakAllocBytes     uint64 `json:"peak_alloc_bytes"`
PeakHeapInuseBytes uint64 `json:"peak_heap_inuse_bytes"`
TotalAllocBytes    uint64 `json:"total_alloc_bytes"`
GCCount            uint32 `json:"gc_count"`
}

func (p BranchPlan) MarshalJSON() ([]byte, error) {
    type branchPlanJSON struct {
        Branch     string  `json:"branch"`

Haves          []string  `json:"haves"`
FetchedObjects int       `json:"fetched_objects"`
Stats          Stats     `json:"stats"`
Measurement    Measurement `json:"measurement"`
}

haves := make([]string, 0, len(r.Haves))
for _, hash := range r.Haves {

34 unmodified lines

if r.Measurement.Enabled {
    lines = append(lines, fmt.Sprintf(
        "measurement: elapsed-ms=%d peak-alloc-bytes=%d peak-heap-inuse-bytes=%d total-alloc-bytes=%d gc-count=%d",
        r.Measurement.ElapsedMillis, r.Measurement.PeakAllocBytes, r.Measurement.PeakHeapInuseBytes, r.Measurement.TotalAllocBytes, r.Measurement.GCCount,
    ))
}

return lines
}

func Run(ctx context.Context, cfg Config) (Result, error) {
measurementDone := startMeasurement(cfg.MeasureMemory)
if cfg.ProtocolMode == "" {
    cfg.ProtocolMode = protocolModeAuto
}

Plans:    plans,
DryRun:   cfg.DryRun,
Stats:    stats.snapshot(),
Measurement: measurementDone(),
Protocol: sourceService.protocol,
}

result.Stats = stats.snapshot()
result.Measurement = measurementDone()
return result, nil
}

func Bootstrap(ctx context.Context, cfg Config) (Result, error) {
measurementDone := startMeasurement(cfg.MeasureMemory)
if cfg.ProtocolMode == "" {
    cfg.ProtocolMode = protocolModeAuto
}

result := Result{
Plans:    plans,
Stats:    stats.snapshot(),
Measurement: measurementDone(),
Protocol: sourceService.protocol,
}

result.Pushed = len(plans)
result.Stats = stats.snapshot()
result.Measurement = measurementDone()
return result, nil
}

func Probe(ctx context.Context, cfg Config) (ProbeResult, error) {
measurementDone := startMeasurement(cfg.MeasureMemory)
if cfg.ProtocolMode == "" {
    cfg.ProtocolMode = protocolModeAuto
}

Capabilities:  sourceCapabilities(service),
Refs:          refInfos,
Stats:         stats.snapshot(),
Measurement:   measurementDone(),
}

if cfg.Target.URL != "" {
result.TargetURL = cfg.Target.URL
result.TargetCaps = advCapabilities(targetAdv)
result.Stats = stats.snapshot()
result.Measurement = measurementDone()
}

return result, nil
}

func Fetch(ctx context.Context, cfg Config, haveRefs []string, haveHashes []plumbing.Hash) (FetchResult, error) {
measurementDone := startMeasurement(cfg.MeasureMemory)
if cfg.ProtocolMode == "" {
    cfg.ProtocolMode = protocolModeAuto
}

Haves:          sortedUniqueHashes(mapsRefValues(targetRefMap)),
FetchedObjects: objectCount,
Stats:          stats.snapshot(),
Measurement:    measurementDone(),
}, nil
}

676 unmodified lines

func startMeasurement(enabled bool) func() Measurement {
if !enabled {
    return func() Measurement { return Measurement{} }
}

start := time.Now()
var startStats runtime.MemStats
runtime.ReadMemStats(&startStats)

done := make(chan struct{})
var (
mu            sync.Mutex
peakAlloc     = startStats.Alloc
peakHeapInuse = startStats.HeapInuse
result        Measurement
)

go func() {
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-done:
return
case <-ticker.C:
var current runtime.MemStats
runtime.ReadMemStats(&current)
mu.Lock()
if current.Alloc > peakAlloc {
peakAlloc = current.Alloc
}
if current.HeapInuse > peakHeapInuse {
peakHeapInuse = current.HeapInuse
}
mu.Unlock()
}
}
}

var once sync.Once
return func() Measurement {
once.Do(func() {
close(done)
var endStats runtime.MemStats
runtime.ReadMemStats(&endStats)
mu.Lock()
if endStats.Alloc > peakAlloc {
peakAlloc = endStats.Alloc
}
if endStats.HeapInuse > peakHeapInuse {
peakHeapInuse = endStats.HeapInuse
}
result = Measurement{
Enabled:            true,
ElapsedMillis:      time.Since(start).Milliseconds(),
PeakAllocBytes:     peakAlloc,
PeakHeapInuseBytes: peakHeapInuse,
TotalAllocBytes:    endStats.TotalAlloc - startStats.TotalAlloc,
GCCount:            endStats.NumGC - startStats.NumGC,
}
mu.Unlock()
})
return result
}
}

func branchMapFromRefHashMap(refs map[plumbing.ReferenceName]plumbing.Hash) map[string]plumbing.Hash {
branches := make(map[string]plumbing.Hash)
for name, hash := range refs {