Batch ref-update commands to stay under receive-pack cap · Entire

Batch ref-update commands to stay under receive-pack cap

752e16c→main·

A sync of a repo with many refs sent every ref-update command in a single receive-pack request, which entire-server rejects past 25,000 commands ("too many ref-update commands: 55006 (limit 25000)"). The relay/materialize strategies (replicate, incremental, materialized) had no command-count batching.

Batch inside the gitproto push primitives so every strategy benefits:

Works against both entire-server and canonical git/GitHub (GitHub's per-push branch/tag limit is opt-in, default unlimited).

Sessions

5d4b8f06380eView transcript

Changes

2

    49 unmodified lines
    ```

---

// maxRefUpdatesPerPush bounds how many ref-update commands ride in a single
// receive-pack request. entire-server rejects a push carrying more than 25_000
// commands (server/githttp.maxRefUpdateCommands), and other servers may impose
// their own caps; staying well under that lets a sync of a many-ref repo split
// across several pushes instead of failing outright.
//
// Splitting is safe because the pack accompanying the first batch carries every
// object for the whole push: receive-pack commits the entire received pack into
// the object store (entire-server via CommitQuarantinedFanout, canonical git via
// tmp_objdir_migrate — neither prunes objects unreachable from the pushed tips),
// so the remaining batches only need to move ref pointers and carry no pack.
const maxRefUpdatesPerPush = 20_000

// chunkRefUpdates splits commands into batches no larger than
// maxRefUpdatesPerPush. Input that already fits is returned as a single batch
// (including the empty slice, so callers preserve their one-request behavior).
func chunkRefUpdates(commands []PushCommand) [][]PushCommand {
    if len(commands) <= maxRefUpdatesPerPush {
        return [][]PushCommand{commands}
    }
    batches := make([][]PushCommand, 0, (len(commands)+maxRefUpdatesPerPush-1)/maxRefUpdatesPerPush)
    for start := 0; start < len(commands); start += maxRefUpdatesPerPush {
        end := min(start+maxRefUpdatesPerPush, len(commands))
        batches = append(batches, commands[start:end])
    }
    return batches
}

// PushPack streams a pack to the target.
func (p *Pusher) PushPack(ctx context.Context, commands []PushCommand, pack io.ReadCloser) error {
    return PushPack(ctx, p.Conn, p.Adv, commands, pack, p.Verbose, p.OnRejection)
}

// PushObjects pushes locally-materialized objects to the target.
func PushObjects(
    ctx context.Context,
    conn Conn,
    adv *packp.AdvRefs,
    commands []PushCommand,
    store storer.Storer,
    hashes []plumbing.Hash,
    verbose bool,
    onRejection func(plumbing.ReferenceName, string),
) error {
    if len(commands) <= maxRefUpdatesPerPush {
        return pushObjectsBatch(ctx, conn, adv, commands, store, hashes, verbose, onRejection)
    }

updates := make([]PushCommand, 0, len(commands))
    var deletes []PushCommand
    for _, c := range commands {
        if c.Delete {
            deletes = append(deletes, c)
        } else {
            updates = append(updates, c)
        }
    }

if len(updates) > 0 {
        first, rest := splitFirstBatch(updates)
        if err := pushObjectsBatch(ctx, conn, adv, first, store, hashes, verbose, onRejection); err != nil {
            return err
        }
        if len(rest) > 0 {
            if err := PushCommands(ctx, conn, adv, rest, verbose, onRejection); err != nil {
                return err
            }
        }
    }
    if len(deletes) > 0 {
        return PushCommands(ctx, conn, adv, deletes, verbose, onRejection)
    }
    return nil
}

// TestChunkRefUpdates tests chunking of ref updates.
func TestChunkRefUpdates(t *testing.T) {
    require.Len(t, chunkRefUpdates(nil), 1)
    require.Len(t, chunkRefUpdates(make([]PushCommand, maxRefUpdatesPerPush)), 1)

batches := chunkRefUpdates(make([]PushCommand, maxRefUpdatesPerPush+1))
    require.Len(t, batches, 2)
    require.Len(t, batches[0], maxRefUpdatesPerPush)
    require.Len(t, batches[1], 1)
}

// TestPushCommandsBatchesOverCap guards that a ref-only push exceeding the
// per-request cap splits into multiple receive-pack requests, each within the
// cap, so the server's too-many-ref-update-commands limit isn't tripped.
func TestPushCommandsBatchesOverCap(t *testing.T) {
    // Test implementation
}

// TestPushPackBatchesOverCap guards that a pack push exceeding the per-request
// cap sends the pack with the first batch and the remaining refs as ref-only
// follow-up batches (the objects are already committed by the first request).
func TestPushPackBatchesOverCap(t *testing.T) {
    // Test implementation
}