Sample bootstrap batch checkpoints · Entire

Sample bootstrap batch checkpoints

1235981→main·

Soph·3mo ago·2 files·+85 added/-87 removed

Sessions

121669700ffcView transcript

Changes

2

1211 unmodified lines

1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1241
1236
1237
1238
1239
5 unmodified lines

1245
1246
1247
1253
1248
1249
1255
1256
1250
1251
1252
1253
1254
1255
1262
1263
1264
1256
1257
1258
1259
1260
1270
1271
1272
1273
1261
1262
1263
1264
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1265
1266
1267
1268
1269
1289
1290
1291
1270
1271
1272
1273
1293
1294
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1296
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1299
1295
1296
1297
1298
1299
1304
1305
1306
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1310
1311
1312
1313
1314
1315
1316
1317
1313
1314
1315
1316
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331

1211 unmodified lines

prevHash plumbing.Hash,
    prevSpan int,
) (int, error) {
    return largestCheckpointUnderLimitByProbe(
        chain,
        prevIdx,
        prevSpan,
        func(idx int) (bool, error) {
            progressf(
                cfg.Verbose,
                "bootstrap-batch: branch=%s probe checkpoint=%s base=%s",
                ref.TargetRef,
                shortHash(chain[idx]),
                shortHash(prevHash),
            )
            tooLarge, err := sourcePackExceedsLimit(ctx, sourceConn, sourceService, ref, chain[idx], prevHash, cfg.BatchMaxPackBytes)
            if err != nil {
                return false, fmt.Errorf("measure bootstrap batch for %s at %s: %w", ref.TargetRef, shortHash(chain[idx]), err)
            }
            if tooLarge {
                progressf(cfg.Verbose, "bootstrap-batch: checkpoint=%s exceeds limit=%d", shortHash(chain[idx]), cfg.BatchMaxPackBytes)
            } else {
                progressf(cfg.Verbose, "bootstrap-batch: checkpoint=%s fits limit=%d", shortHash(chain[idx]), cfg.BatchMaxPackBytes)
            }
            return tooLarge, nil
        },
    )
    return sampledCheckpointUnderLimitByProbe(chain, prevIdx, prevSpan, func(idx int) (bool, error) {
        progressf(
            cfg.Verbose,
            "bootstrap-batch: branch=%s probe checkpoint=%s base=%s",
            ref.TargetRef,
            shortHash(chain[idx]),
            shortHash(prevHash),
        )
        tooLarge, err := sourcePackExceedsLimit(ctx, sourceConn, sourceService, ref, chain[idx], prevHash, cfg.BatchMaxPackBytes)
        if err != nil {
            return false, fmt.Errorf("measure bootstrap batch for %s at %s: %w", ref.TargetRef, shortHash(chain[idx]), err)
        }
        if tooLarge {
            progressf(cfg.Verbose, "bootstrap-batch: checkpoint=%s exceeds limit=%d", shortHash(chain[idx]), cfg.BatchMaxPackBytes)
        } else {
            progressf(cfg.Verbose, "bootstrap-batch: checkpoint=%s fits limit=%d", shortHash(chain[idx]), cfg.BatchMaxPackBytes)
        }
        return tooLarge, nil
    })
}

func largestCheckpointUnderLimitByProbe(
func sampledCheckpointUnderLimitByProbe(
    chain []plumbing.Hash,
    prevIdx int,
    prevSpan int,
5 unmodified lines

return -1, nil
    }

coarse := coarseCheckpointCandidates(lo, hi, prevSpan)
    samples := sampledCheckpointCandidates(lo, hi, prevSpan)
    best := -1
    firstTooLarge := -1
    for _, idx := range coarse {
    for _, idx := range samples {
        tooLarge, err := probe(idx)
        if err != nil {
            return -1, err
        }
        if tooLarge {
            if firstTooLarge == -1 || idx < firstTooLarge {
                firstTooLarge = idx
            }
            continue
        }
        best = idx
        break
    }
    if best == -1 {
        return -1, nil
    }

if best == hi {
    if best != -1 {
        return best, nil
    }

searchLo := best + 1
    searchHi := hi
    if firstTooLarge != -1 {
        searchHi = firstTooLarge - 1
    }

for searchLo <= searchHi {
        mid := searchLo + (searchHi-searchLo)/2
        tooLarge, err := probe(mid)
        if err != nil {
            return -1, err
    if prevSpan > 1 {
        shrunk := prevSpan / 2
        if shrunk < 1 {
            shrunk = 1
        }
        if tooLarge {
            searchHi = mid - 1
            continue
        }
        idx := lo + shrunk - 1
        if idx > hi {
            idx = hi
        }
        best = mid
        searchLo = mid + 1
        if idx >= lo {
            tooLarge, err := probe(idx)
            if err != nil {
                return -1, err
            }
            if !tooLarge {
                return idx, nil
            }
        }
    }
    return best, nil

tooLarge, err := probe(lo)
    if err != nil {
        return -1, err
    }
    if tooLarge {
        return -1, nil
    }
    return lo, nil
}

func coarseCheckpointCandidates(lo, hi int, prevSpan int) []int {
func sampledCheckpointCandidates(lo, hi int, prevSpan int) []int {
    if lo > hi {
        return nil
    }

set := map[int]struct{}{
        hi: {},
        lo: {},
    set := map[int]struct{}{}
    add := func(idx int) {
        if idx < lo {
            idx = lo
        }
        if idx > hi {
            idx = hi
        }
        set[idx] = struct{}{}
    }

projected := hi;
    if prevSpan > 0 {
        projected := lo + prevSpan - 1
        if projected < lo {
            projected = lo
        }
        if projected > hi {
            projected = hi
        }
        set[projected] = struct{}{}
        projected = lo + prevSpan - 1
    }
    add(projected)

const coarseBuckets = 6
    width := hi - lo;
    if width > 0 {
        for i := 1; i < coarseBuckets; i++ {
            idx := lo + (width*i)/coarseBuckets
            if idx < lo {
                idx = lo
            }
            if idx > hi {
                idx = hi;
            }
            set[idx] = struct{}{}
    const sampleCount = 4
    current := projected
    for i := 0; i < sampleCount-1; i++ {
        if current <= lo {
            add(lo)
            continue
        }
        distance := current - lo;
        current = lo + distance/2;
        add(current)
    }
    add(lo);

candidates := make([]int, 0, len(set))
    for idx := range set {

Minternal/syncer/syncer.go+71/-76

111 unmodified lines

112
113
114
115
116
115
116
117
118
118
119
120
120
121
122
123
124
125
126
127
128
129
130
128
131
132
133
134
135
136
137
135
138
139
140
141
142
140
143
144
142
143
145
146
147
145
146
148
149
150
151

111 unmodified lines

return hash
}

func TestCoarseCheckpointCandidates(t *testing.T) {
    candidates := coarseCheckpointCandidates(10, 100, 20)
func TestSampledCheckpointCandidates(t *testing.T) {
    candidates := sampledCheckpointCandidates(10, 100, 20)
    if len(candidates) == 0 {
        t.Fatalf("expected coarse candidates")
        t.Fatalf("expected sampled candidates")
    }
    if candidates[0] != 100 {
    if candidates[0] != 29 {
        t.Fatalf("expected highest candidate first, got %v", candidates)
    }
    if !slices.Contains(candidates, 29) {
        t.Fatalf("expected projected candidate near previous span, got %v", candidates)
    }
    if !slices.Contains(candidates, 10) {
        t.Fatalf("expected lower bound candidate, got %v", candidates)
    }
}

func TestLargestCheckpointUnderLimitByProbe(t *testing.T) {
func TestSampledCheckpointUnderLimitByProbe(t *testing.T) {
    chain := make([]plumbing.Hash, 40)
    for i := range chain {
        chain[i] = plumbing.NewHash(fmt.Sprintf("%040x", i+1))
    }

var probes []int
    best, err := largestCheckpointUnderLimitByProbe(chain, 4, 8, func(idx int) (bool, error) {
best, err := sampledCheckpointUnderLimitByProbe(chain, 4, 8, func(idx int) (bool, error) {
    probes = append(probes, idx)
    return idx > 19, nil
})
    if err != nil {
        t.Fatalf("largestCheckpointUnderLimitByProbe: %v", err)
    }
    t.Fatalf("sampledCheckpointUnderLimitByProbe: %v", err)
    if best != 19 {
        t.Fatalf("expected best checkpoint index 19, got %d", best)
    }
    if best < 12 || best > 19 {
        t.Fatalf("expected a reasonable sampled checkpoint, got %d", best)
    }
    if len(probes) > 10 {
        t.Fatalf("expected narrowed probe count, got %d probes: %v", len(probes), probes)
    }
    if len(probes) > 6 {
        t.Fatalf("expected fixed small probe count, got %d probes: %v", len(probes), probes)
    }
}