Add repeatable benchmark command · Entire

Add repeatable benchmark command

package main

import (
    "context"
    "encoding/json"
    "errors"
    "flag"
    "fmt"
    "net/url"
    "os"
    "path/filepath"
    "slices"
    "strings"
    "time"

git "github.com/go-git/go-git/v6"

"github.com/soph/git-sync/internal/syncer"
    "github.com/soph/git-sync/internal/validation"
)

type scenario string

const (
    scenarioBootstrap scenario = "bootstrap"
    scenarioSync      scenario = "sync"
)

type runSummary struct {
    Index      int           `json:"index"`
    TargetPath string        `json:"target_path"`
    TargetURL  string        `json:"target_url"`
    WallMillis int64         `json:"wall_millis"`
    Result     syncer.Result `json:"result"`
    Error      string        `json:"error,omitempty"`
}

type aggregateSummary struct {
    SuccessfulRuns        int      `json:"successful_runs"`
    FailedRuns            int      `json:"failed_runs"`
    MinWallMillis         int64    `json:"min_wall_millis"`
    MaxWallMillis         int64    `json:"max_wall_millis"`
    AvgWallMillis         float64  `json:"avg_wall_millis"`
    MinSyncElapsedMillis  int64    `json:"min_sync_elapsed_millis"`
    MaxSyncElapsedMillis  int64    `json:"max_sync_elapsed_millis"`
    AvgSyncElapsedMillis  float64  `json:"avg_sync_elapsed_millis"`
    MaxPeakAllocBytes     uint64   `json:"max_peak_alloc_bytes"`
    MaxPeakHeapInuseBytes uint64   `json:"max_peak_heap_inuse_bytes"`
    MaxTotalAllocBytes    uint64   `json:"max_total_alloc_bytes"`
    MaxGCCount            uint32   `json:"max_gc_count"`
    RelayModes            []string `json:"relay_modes,omitempty"`
}

type benchmarkReport struct {
    Scenario    scenario         `json:"scenario"`
    SourceURL   string           `json:"source_url"`
    Repeat      int              `json:"repeat"`
    KeepTargets bool             `json:"keep_targets"`
    WorkDir     string           `json:"work_dir"`
    Config      syncer.Config    `json:"config"`
    Aggregate   aggregateSummary `json:"aggregate"`
    Runs        []runSummary     `json:"runs"`
}

func main() {
    if err := run(context.Background(), os.Args[1:]); err != nil {
        fmt.Fprintf(os.Stderr, "error: %v\n", err)
        os.Exit(1)
    }
}

func run(ctx context.Context, args []string) error {
    fs := flag.NewFlagSet("git-sync-bench", flag.ContinueOnError)
    fs.SetOutput(os.Stderr)

var cfg syncer.Config
    var scenarioName string
    var workDir string
    var repeat int
    var keepTargets bool
    var jsonOutput bool
    var mappings multiStringFlag

fs.StringVar(&scenarioName, "scenario", string(scenarioBootstrap), "benchmark scenario: bootstrap or sync")
    fs.StringVar(&cfg.Source.URL, "source-url", "", "source repository URL or local path")
    fs.StringVar(&workDir, "work-dir", "", "directory for temporary target repositories")
    fs.IntVar(&repeat, "repeat", 1, "number of runs to execute")
    fs.BoolVar(&keepTargets, "keep-targets", false, "retain generated target repositories after the run")
    fs.BoolVar(&jsonOutput, "json", true, "print JSON output")

branches := fs.String("branch", "", "comma-separated branch list; default is all source branches")
    fs.Var(&mappings, "map", "ref mapping in src:dst form; short names map branches, full refs map exact refs")
    fs.BoolVar(&cfg.IncludeTags, "tags", false, "mirror tags")
    fs.BoolVar(&cfg.Force, "force", false, "allow non-fast-forward branch updates and retarget tags")
    fs.BoolVar(&cfg.Prune, "prune", false, "delete managed target refs that no longer exist on source")
    fs.BoolVar(&cfg.ShowStats, "stats", false, "collect transfer statistics")
    fs.BoolVar(&cfg.MeasureMemory, "measure-memory", true, "sample elapsed time and Go heap usage")
    fs.Int64Var(&cfg.MaxPackBytes, "max-pack-bytes", 0, "abort bootstrap if the streamed source pack exceeds this many bytes")
    fs.Int64Var(&cfg.BatchMaxPackBytes, "batch-max-pack-bytes", 0, "split branch bootstrap into relay batches capped at this many bytes per batch")
    fs.StringVar(&cfg.ProtocolMode, "protocol", validation.ProtocolAuto, "protocol mode: auto, v1, or v2")
    fs.BoolVar(&cfg.Verbose, "v", false, "verbose logging")

if err := fs.Parse(args); err != nil {
        return err
    }
    if len(fs.Args()) > 0 {
        return usageError("unexpected positional arguments")
    }
    if repeat < 1 {
        return usageError("--repeat must be at least 1")
    }

mode, err := validation.NormalizeProtocolMode(cfg.ProtocolMode)
    if err != nil {
        return err
    }
    cfg.ProtocolMode = mode

if *branches != "" {
        cfg.Branches = splitCSV(*branches)
    }
    for _, raw := range mappings {
        mapping, err := validation.ParseMapping(raw)
        if err != nil {
            return err
        }
        cfg.Mappings = append(cfg.Mappings, mapping)
    }

srcURL, err := normalizeRepoURL(cfg.Source.URL)
    if err != nil {
        return err
    }
    cfg.Source.URL = srcURL

sc, err := parseScenario(scenarioName)
    if err != nil {
        return err
    }
    if sc == scenarioBootstrap {
        if cfg.Force || cfg.Prune {
            return usageError("bootstrap benchmarks do not support --force or --prune")
        }
    }

if workDir == "" {
        workDir, err = os.MkdirTemp("", "git-sync-bench-*")
        if err != nil {
            return fmt.Errorf("create temp work dir: %w", err)
        }
    } else {
        if err := os.MkdirAll(workDir, 0o755); err != nil {
            return fmt.Errorf("create work dir: %w", err)
        }
    }

report := benchmarkReport{
        Scenario:    sc,
        SourceURL:   cfg.Source.URL,
        Repeat:      repeat,
        KeepTargets: keepTargets,
        WorkDir:     workDir,
        Config:      cfg,
        Runs:        make([]runSummary, 0, repeat),
    }

for i := 0; i < repeat; i++ {
        runCfg := cfg
        targetPath := filepath.Join(workDir, fmt.Sprintf("%s-run-%03d.git", sc, i+1))
        if err := os.RemoveAll(targetPath); err != nil {
            return fmt.Errorf("clear target path %s: %w", targetPath, err)
        }
        if _, err := git.PlainInit(targetPath, true); err != nil {
            return fmt.Errorf("init target repo %s: %w", targetPath, err)
        }
        targetURL, err := fileURL(targetPath)
        if err != nil {
            return err
        }
        runCfg.Target.URL = targetURL

start := time.Now()
        runResult, runErr := executeScenario(ctx, sc, runCfg)
        summary := runSummary{
            Index:      i + 1,
            TargetPath: targetPath,
            TargetURL:  targetURL,
            WallMillis: time.Since(start).Milliseconds(),
            Result:     runResult,
        }
        if runErr != nil {
            summary.Error = runErr.Error()
        }
        report.Runs = append(report.Runs, summary)

if !keepTargets {
            if err := os.RemoveAll(targetPath); err != nil {
                return fmt.Errorf("remove target path %s: %w", targetPath, err)
            }
        }
    }

report.Aggregate = summarizeRuns(report.Runs)

if jsonOutput {
        data, err := json.MarshalIndent(report, "", "  ")
        if err != nil {
            return fmt.Errorf("marshal report: %w", err)
        }
        fmt.Println(string(data))
        return nil
    }

printTextReport(report)
    return nil
}

func executeScenario(ctx context.Context, sc scenario, cfg syncer.Config) (syncer.Result, error) {
    switch sc {
    case scenarioBootstrap:
        return syncer.Bootstrap(ctx, cfg)
    case scenarioSync:
        return syncer.Run(ctx, cfg)
    default:
        return syncer.Result{}, fmt.Errorf("unsupported scenario %q", sc)
    }
}

func summarizeRuns(runs []runSummary) aggregateSummary {
    if len(runs) == 0 {
        return aggregateSummary{}
    }

var (
        okRuns           int
        failedRuns       int
        totalWall        int64
        totalSyncElapsed int64
        relayModes       []string
    )
    summary := aggregateSummary{
        MinWallMillis:        -1,
        MinSyncElapsedMillis: -1,
    }

for _, run := range runs {
        if run.Error != "" {
            failedRuns++
            continue
        }
        okRuns++
        totalWall += run.WallMillis
        if summary.MinWallMillis < 0 || run.WallMillis < summary.MinWallMillis {
            summary.MinWallMillis = run.WallMillis
        }
        if run.WallMillis > summary.MaxWallMillis {
            summary.MaxWallMillis = run.WallMillis
        }

m := run.Result.Measurement
        totalSyncElapsed += m.ElapsedMillis
        if summary.MinSyncElapsedMillis < 0 || m.ElapsedMillis < summary.MinSyncElapsedMillis {
            summary.MinSyncElapsedMillis = m.ElapsedMillis
        }
        if m.ElapsedMillis > summary.MaxSyncElapsedMillis {
            summary.MaxSyncElapsedMillis = m.ElapsedMillis
        }
        if m.PeakAllocBytes > summary.MaxPeakAllocBytes {
            summary.MaxPeakAllocBytes = m.PeakAllocBytes
        }
        if m.PeakHeapInuseBytes > summary.MaxPeakHeapInuseBytes {
            summary.MaxPeakHeapInuseBytes = m.PeakHeapInuseBytes
        }
        if m.TotalAllocBytes > summary.MaxTotalAllocBytes {
            summary.MaxTotalAllocBytes = m.TotalAllocBytes
        }
        if m.GCCount > summary.MaxGCCount {
            summary.MaxGCCount = m.GCCount
        }
        if mode := strings.TrimSpace(run.Result.RelayMode); mode != "" {
            relayModes = append(relayModes, mode)
        }
    }

summary.SuccessfulRuns = okRuns
    summary.FailedRuns = failedRuns
    if okRuns > 0 {
        summary.AvgWallMillis = float64(totalWall) / float64(okRuns)
        summary.AvgSyncElapsedMillis = float64(totalSyncElapsed) / float64(okRuns)
    }
    summary.RelayModes = uniqueStrings(relayModes)
    return summary
}

func uniqueStrings(input []string) []string {
    if len(input) == 0 {
        return nil
    }
    slices.Sort(input)
    out := input[:0]
    var prev string
    for i, item := range input {
        if i == 0 || item != prev {
            out = append(out, item)
            prev = item
        }
    }
    return out
}

func normalizeRepoURL(raw string) (string, error) {
    raw = strings.TrimSpace(raw)
    if raw == "" {
        return "", usageError("--source-url is required")
    }
    if u, err := url.Parse(raw); err == nil && u.Scheme != "" {
        return raw, nil
    }
    return fileURL(raw)
}

func fileURL(path string) (string, error) {
    abs, err := filepath.Abs(path)
    if err != nil {
        return "", fmt.Errorf("resolve path %q: %w", path, err)
    }
    return (&url.URL{Scheme: "file", Path: filepath.ToSlash(abs)}).String(), nil
}

func parseScenario(raw string) (scenario, error) {
    switch scenario(strings.TrimSpace(strings.ToLower(raw))) {
    case scenarioBootstrap:
        return scenarioBootstrap, nil
    case scenarioSync:
        return scenarioSync, nil
    default:
        return "", usageError(fmt.Sprintf("unsupported --scenario %q", raw))
    }
}

func printTextReport(report benchmarkReport) {
    fmt.Printf("scenario: %s\n", report.Scenario)
    fmt.Printf("source: %s\n", report.SourceURL)
    fmt.Printf("runs: %d success=%d failed=%d\n", report.Repeat, report.Aggregate.SuccessfulRuns, report.Aggregate.FailedRuns)
    fmt.Printf("wall-ms: avg=%.1f min=%d max=%d\n", report.Aggregate.AvgWallMillis, report.Aggregate.MinWallMillis, report.Aggregate.MaxWallMillis)
    fmt.Printf("sync-elapsed-ms: avg=%.1f min=%d max=%d\n", report.Aggregate.AvgSyncElapsedMillis, report.Aggregate.MinSyncElapsedMillis, report.Aggregate.MaxSyncElapsedMillis)
    fmt.Printf("peak-alloc-bytes: %d\n", report.Aggregate.MaxPeakAllocBytes)
    fmt.Printf("peak-heap-inuse-bytes: %d\n", report.Aggregate.MaxPeakHeapInuseBytes)
    if len(report.Aggregate.RelayModes) > 0 {
        fmt.Printf("relay-modes: %s\n", strings.Join(report.Aggregate.RelayModes, ","))
    }
    for _, run := range report.Runs {
        status := "ok"
        if run.Error != "" {
            status = "error=" + run.Error
        }
        fmt.Printf("run[%d]: wall-ms=%d relay-mode=%s batch-count=%d target=%s %s\n",
            run.Index, run.WallMillis, run.Result.RelayMode, run.Result.BatchCount, run.TargetPath, status)
    }
}

type multiStringFlag []string

func (m *multiStringFlag) String() string {
    return strings.Join(*m, ",")
}

func (m *multiStringFlag) Set(value string) error {
    *m = append(*m, value)
    return nil
}

func splitCSV(value string) []string {
    parts := strings.Split(value, ",")
    out := make([]string, 0, len(parts))
    for _, part := range parts {
        part = strings.TrimSpace(part)
        if part != "" {
            out = append(out, part)
        }
    }
    return out
}

func usageError(message string) error {
    usage := "usage:\n  git-sync-bench --source-url <repo> [flags]\n\nflags:\n  --scenario bootstrap|sync\n  --repeat 3\n  --work-dir /tmp/git-sync-bench\n  --keep-targets\n  --json\n  --branch main,release\n  --map main:stable\n  --tags\n  --force\n  --prune\n  --stats\n  --measure-memory\n  --max-pack-bytes 104857600\n  --batch-max-pack-bytes 104857600\n  --protocol auto|v1|v2\n  -v\n"
    if message == "" {
        return errors.New(strings.TrimSpace(usage))
    }
    return fmt.Errorf("%s\n\n%s", message, usage)
}

Benchmarking

git-sync-bench runs repeatable empty-target benchmarks against a source repository.

It currently supports:

The tool creates a fresh bare target repository for each run and reports both wall-clock time and the internal syncer measurement data.

Build

go build -o /tmp/git-sync-bench ./cmd/git-sync-bench

Example

Against a local mirror:

/tmp/git-sync-bench \
  --scenario bootstrap \
  --source-url /tmp/git-sync-bench/kubernetes.git \
  --repeat 3 \
  --batch-max-pack-bytes 104857600 \
  --stats \
  --json

If --source-url is a filesystem path, the tool converts it to file://... automatically.

Output

The JSON report includes:

Notes