fix: dedupe merged code-search stats across mirror cells · Entire

fix: dedupe merged code-search stats across mirror cells

b72a6e6→main· Soph·1w ago·2 files·+120 added/-31 removed

Fanning out code search across mirror placements makes a mirrored repo return the same matches from its home and mirror cells. mergeSearchResults already deduped the result list but summed the stats, so a repo with 3 matches searched in two cells reported "6 matches across 4 files in 2 repos" for 3 unique results, and falsely claimed truncation ("Showing 3 of 6 matches").

Dedupe RepoStats by repo name (keep the representative max, not the sum) and subtract the duplicated portion from TotalMatches / TotalFiles / ReposSearched. The base stays peregrine's own per-cell totals, so per-cell truncation and zero-match repos are preserved; only the provable mirror duplicate is removed.

Addresses trail #794 finding 019f40a4-d27.

Co-Authored-By: Claude Opus 4.8 (1M context) noreply@anthropic.com

Sessions

01KX0F53C6MZJYAD1ZEBF43G7SView transcript

[?
Fix Fan-Out Code Search Across Mirror PlacementsClaude Code·Opus 4.8[1m]·4 steps](/content/gh/entireio/cli/session/40cc261d-529d-4379-82a8-b12624975a98#timeline-01KX0F53C6MZJYAD1ZEBF43G7S/index.html)

Changes

2

706 unmodified lines

707
708
709
710
711
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
714
715
716
717
718
719
720
726
727
728
729
730
731
732
733
734
735
736
737
723
724
725
726
727
738
739
740
741
742
743
744
745
746
747
731
732
733
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763

706 unmodified lines

}
merged.Results = deduped

// Deduplicate RepoStats by repo name, summing match/file counts.
repoStatsMap := make(map[string]*codesearch.RepoStats, len(merged.RepoStats))
// Deduplicate RepoStats by repo name. A repo that appears in more than one
// cell is a mirror placement returning the SAME content (this PR fans out
// across placements, so e.g. a US-homed repo with an EU mirror is now
// searched in both cells) — not additional matches. Keep one representative
// entry per repo (the max of each count; mirror copies are identical, max
// only guards against minor per-cell skew) instead of summing, and record
// the duplicated portion so the aggregate stats can drop the double-count.
type repoStatAcc struct {
    idx                    int
    sumMatches, maxMatches int
    sumFiles, maxFiles     int
    cellCount              int
}
accByRepo := make(map[string]*repoStatAcc, len(merged.RepoStats))
var dedupedStats []codesearch.RepoStats
for _, rs := range merged.RepoStats {
    if existing, ok := repoStatsMap[rs.Repo]; ok {
        existing.MatchCount += rs.MatchCount
        existing.FileCount += rs.FileCount
    } else {
        entry := rs // copy
        repoStatsMap[rs.Repo] = &entry
        dedupedStats = append(dedupedStats, entry)
    }
    acc, ok := accByRepo[rs.Repo]
    if !ok {
        acc = &repoStatAcc{idx: len(dedupedStats)}
        accByRepo[rs.Repo] = acc
        dedupedStats = append(dedupedStats, codesearch.RepoStats{Repo: rs.Repo})
    }
    acc.cellCount++
    acc.sumMatches += rs.MatchCount
    acc.sumFiles += rs.FileCount
    acc.maxMatches = max(acc.maxMatches, rs.MatchCount)
    acc.maxFiles = max(acc.maxFiles, rs.FileCount)
}
// Write back merged values.
for i := range dedupedStats {
    if m, ok := repoStatsMap[dedupedStats[i].Repo]; ok {
        dedupedStats[i] = *m
    }
}
var overcountMatches, overcountFiles, overcountRepos int
for _, acc := range accByRepo {
    dedupedStats[acc.idx].MatchCount = acc.maxMatches
    dedupedStats[acc.idx].FileCount = acc.maxFiles
    overcountMatches += acc.sumMatches - acc.maxMatches
    overcountFiles += acc.sumFiles - acc.maxFiles
    overcountRepos += acc.cellCount - 1
}
merged.RepoStats = dedupedStats

// Stats are preserved as the sum of per-cell peregrine stats — they
// reflect the true totals (including zero-match repos and per-cell
// truncation), not just the deduped result slice.
// The per-cell Stats were summed above, so a mirrored repo's matches were
// counted once per cell. Subtract the duplicated copies identified via
// RepoStats so the totals reflect distinct content, not the same content
// seen from every mirror cell. This preserves per-cell truncation (the
// base is peregrine's own totals; we only remove the provable duplicate
// portion) and zero-match repos (they contribute 0 to the subtraction).
// A repo with matches but no RepoStats row, or a zero-match mirror repo,
// can't be de-duplicated from the response and keeps its summed
// contribution — a mild over-count, far less misleading than reporting
// every mirrored match twice. Clamp at zero against inconsistent input.
merged.Stats.TotalMatches = max(0, merged.Stats.TotalMatches-overcountMatches)
merged.Stats.TotalFiles = max(0, merged.Stats.TotalFiles-overcountFiles)
merged.Stats.ReposSearched = max(0, merged.Stats.ReposSearched-overcountRepos)

// Cap to the caller's requested limit.
if limit > 0 && len(merged.Results) > limit {

Mcmd/entire/cli/search_cmd.go+44/-17


381 unmodified lines

382
383
384
385
386
387
388
389
390
391
392
386
387
388
389
390
391
392
393
394
395
396
397
398
399
393
394
395
396
397
3 unmodified lines

401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473

381 unmodified lines

t.Parallel()

dup := codesearch.Result{Repo: "acme/web", Path: "main.go", Line: 10, Column: 5, Score: 0.9}
cellVal := func() *codesearch.SearchResponse {
    return &codesearch.SearchResponse{
        Results:   []codesearch.Result{dup},
        Stats:     codesearch.Stats{TotalMatches: 1, TotalFiles: 1, ReposSearched: 1},
        RepoStats: []codesearch.RepoStats{{Repo: "acme/web", MatchCount: 1, FileCount: 1}},
    }
}
results := []cellCallResult[*codesearch.SearchResponse]{
    {
        group: cellGroup{cell: "", jurisdiction: ""},
        value: &codesearch.SearchResponse{
            Results: []codesearch.Result{dup},
            Stats:   codesearch.Stats{TotalMatches: 1, TotalFiles: 1, ReposSearched: 1},
        },
    },
    {
        group: cellGroup{cell: "aws-us-east-2", jurisdiction: "us"},
        value: &codesearch.SearchResponse{
            Results: []codesearch.Result{dup},
            Stats:   codesearch.Stats{TotalMatches: 1, TotalFiles: 1, ReposSearched: 1},
        },
    },
    {group: cellGroup{cell: "", jurisdiction: ""}, value: cellVal()},
    {group: cellGroup{cell: "aws-us-east-2", jurisdiction: "us"}, value: cellVal()},
}

merged, err := mergeSearchResults(context.Background(), 0, results)
3 unmodified lines

if len(merged.Results) != 1 {
    t.Fatalf("len(Results) = %d, want 1 (duplicate removed)", len(merged.Results))
}
// Stats must not double-count the overlapping match either.
if merged.Stats.TotalMatches != 1 {
    t.Errorf("TotalMatches = %d, want 1 (overlapping cells must not double-count)", merged.Stats.TotalMatches)
}
if merged.Stats.ReposSearched != 1 {
    t.Errorf("ReposSearched = %d, want 1 (one logical repo)", merged.Stats.ReposSearched)
}
if len(merged.RepoStats) != 1 || merged.RepoStats[0].MatchCount != 1 {
    t.Errorf("RepoStats = %+v, want one entry with MatchCount 1", merged.RepoStats)
}
}

func TestMergeSearchResults_MirrorPlacementsDoNotDoubleCount(t *testing.T) {
    t.Parallel()

// A US-homed repo with an EU mirror indexes the same content, so the
    // fan-out queries both cells and each returns the SAME matches. Merged
    // results dedupe by repo+path+line; the stats must dedupe too, or the
    // summary reports "6 matches across 4 files in 2 repos" for 3 unique
    // results (and falsely claims truncation). Regression guard for the
    // mirror fan-out this trail introduced.
    matches := []codesearch.Result{
        {Repo: "acme/web", Path: "main.go", Line: 1, Column: 0, Score: 0.9},
        {Repo: "acme/web", Path: "main.go", Line: 2, Column: 0, Score: 0.8},
        {Repo: "acme/web", Path: "util.go", Line: 5, Column: 0, Score: 0.7},
    }
    cell := func(name, jur string) cellCallResult[*codesearch.SearchResponse] {
        return cellCallResult[*codesearch.SearchResponse]{{
            group: cellGroup{cell: name, jurisdiction: jur},
            value: &codesearch.SearchResponse{
                Query:     "handleRequest",
                Stats:     codesearch.Stats{TotalMatches: 3, TotalFiles: 2, ReposSearched: 1, DurationMs: 10},
                RepoStats: []codesearch.RepoStats{{Repo: "acme/web", MatchCount: 3, FileCount: 2}},
                Results:   matches,
            },
        }}
    }
    results := []cellCallResult[*codesearch.SearchResponse]{
        cell("aws-us-east-2", "us"),
        cell(testCellEU, "eu"),
    }

merged, err := mergeSearchResults(context.Background(), 0, results)
    if err != nil {
        t.Fatalf("unexpected error: %v", err)
    }
    if len(merged.Results) != 3 {
        t.Fatalf("len(Results) = %d, want 3 (mirror duplicates removed)", len(merged.Results))
    }
    if merged.Stats.TotalMatches != 3 {
        t.Errorf("TotalMatches = %d, want 3 (mirror must not double-count)", merged.Stats.TotalMatches)
    }
    if merged.Stats.TotalFiles != 2 {
        t.Errorf("TotalFiles = %d, want 2 (mirror must not double-count)", merged.Stats.TotalFiles)
    }
    if merged.Stats.ReposSearched != 1 {
        t.Errorf("ReposSearched = %d, want 1 (one logical repo across two cells)", merged.Stats.ReposSearched)
    }
    if merged.Stats.DurationMs != 10 {
        t.Errorf("DurationMs = %v, want 10 (slowest cell preserved)", merged.Stats.DurationMs)
    }
    if len(merged.RepoStats) != 1 {
        t.Fatalf("len(RepoStats) = %d, want 1 (deduped by repo)", len(merged.RepoStats))
    }
    if merged.RepoStats[0].MatchCount != 3 || merged.RepoStats[0].FileCount != 2 {
        t.Errorf("RepoStats[0] = %+v, want representative {3,2} not summed {6,4}", merged.RepoStats[0])
    }
}

func TestResolveRepoFilters_GhPrefix(t *testing.T) {