Merge pull request #38 from entireio/soph/smart-subdivision · Entire

Merge pull request #38 from entireio/soph/smart-subdivision

1782678→main· Soph·2mo ago·2 files·+431 added/-21 removed

Use observed pack size to size post-rejection subdivisions

Changes

223 unmodified lines

...
273 unmodified lines

// master and nocache-cleanup share ~99% of history).
completedRefs := planner.CopyRefHashMap(p.TargetRefs)

// calibratedBytesPerObject tracks the per-object byte estimate
// updated from observed rejected pushes. Starts at the static
// default (which under-counts blob-heavy repos) and ratchets up as
// 413s reveal that the real bytes/object are much higher than 750.
// Used in checkPackSizeAndSubdivide so subsequent sub-pack fetches
// are pre-emptively split when the calibrated estimate exceeds
// p.TargetMaxPack — saving an entire ~limit-sized wasted upload.
calibratedBytesPerObject := int64(estimatedBytesPerObject)

for _, batch := range batches {
    if batch.subsumed {
        cmds := []gitproto.PushCommand{{
            p.TargetMaxPack = expanded
        }
    }
}

// checkPackSizeAndSubdivide reads the 12-byte PACK header to get the object
// count, estimates total pack size, and if it exceeds batchLimit, closes the
// reader and calls subdivide(). Returns (nil, nil) when subdivided (caller
// should continue to retry), or (prepended reader, nil) to proceed with push.
// 
// bytesPerObject lets the caller use a per-run calibrated value
// instead of the static estimatedBytesPerObject default. Calibrating
// after each rejection (using the bytes that flowed through
// packReadCounter) catches blob-heavy repos where the static 750-byte
// average is 10–20× too low.
func checkPackSizeAndSubdivide(
    r io.ReadCloser,
    batchLimit int64,
    bytesPerObject int64,
    subdivide func() bool,
) (io.ReadCloser, error) { 
    //...
}

func calibrateBytesPerObject(sentBytes, objectCount, current int64) int64 {
    if sentBytes <= 0 || objectCount <= 0 {
        return 0
    }
    const safetyMultiplier = 2
    calibrated := safetyMultiplier * sentBytes / objectCount
    if calibrated <= current {
        return 0
    }
    return calibrated
}

type wrappedMultiRC struct {
    io.ReadCloser
}

Test Cases

  1. TestCalibrateBytesPerObject
  2. TestSubdivideCheckpoints
  3. TestObservedSubdivisionFactor
  4. TestSubdivideToFactorReachesTarget
func TestObservedSubdivisionFactor(t *testing.T) {
    t.Parallel()
    cases := []struct {
        name      string
        sentBytes int64
        limit     int64
        want      int
    }{
        {
            name: "no signal falls back to halving",
            want: 2,
        },
        {
            name: "sent comfortably under the limit uses conservative 2x multiplier",
            sentBytes: 100, limit: 1000, want: 2,
        },
        // Other test cases...
    }
}

Function Descriptions