# Batch ref-update commands to stay under receive-pack cap

`752e16c`→[main](/content/gh/entireio/git-sync/commits/main/index.html)·

A sync of a repo with many refs sent every ref-update command in a single
receive-pack request, which entire-server rejects past 25,000 commands
("too many ref-update commands: 55006 (limit 25000)"). The relay/materialize
strategies (replicate, incremental, materialized) had no command-count
batching.

Batch inside the gitproto push primitives so every strategy benefits:

- PushPack / PushObjects send the pack with the first batch and the remaining refs as ref-only follow-ups. The pack carries every object for the whole push, and receive-pack commits the entire received pack (entire-server via CommitQuarantinedFanout, canonical git via tmp_objdir_migrate — neither prunes objects unreachable from the pushed tips), so later batches only move ref pointers.
- PushCommands chunks all commands under maxRefUpdatesPerPush (20_000, with headroom under the server's 25_000 cap).

Works against both entire-server and canonical git/GitHub (GitHub's per-push branch/tag limit is opt-in, default unlimited).

## Sessions

5d4b8f06380eView transcript

## Changes

2

- internal/gitproto

- Mpush.go+117/-2
  - Mpush_test.go+117

```
    49 unmodified lines
    ```

---

// maxRefUpdatesPerPush bounds how many ref-update commands ride in a single
// receive-pack request. entire-server rejects a push carrying more than 25_000
// commands (server/githttp.maxRefUpdateCommands), and other servers may impose
// their own caps; staying well under that lets a sync of a many-ref repo split
// across several pushes instead of failing outright.
//
// Splitting is safe because the pack accompanying the first batch carries every
// object for the whole push: receive-pack commits the entire received pack into
// the object store (entire-server via CommitQuarantinedFanout, canonical git via
// tmp_objdir_migrate — neither prunes objects unreachable from the pushed tips),
// so the remaining batches only need to move ref pointers and carry no pack.
const maxRefUpdatesPerPush = 20_000

// chunkRefUpdates splits commands into batches no larger than
// maxRefUpdatesPerPush. Input that already fits is returned as a single batch
// (including the empty slice, so callers preserve their one-request behavior).
func chunkRefUpdates(commands []PushCommand) [][]PushCommand {
    if len(commands) <= maxRefUpdatesPerPush {
        return [][]PushCommand{commands}
    }
    batches := make([][]PushCommand, 0, (len(commands)+maxRefUpdatesPerPush-1)/maxRefUpdatesPerPush)
    for start := 0; start < len(commands); start += maxRefUpdatesPerPush {
        end := min(start+maxRefUpdatesPerPush, len(commands))
        batches = append(batches, commands[start:end])
    }
    return batches
}

// PushPack streams a pack to the target.
func (p *Pusher) PushPack(ctx context.Context, commands []PushCommand, pack io.ReadCloser) error {
    return PushPack(ctx, p.Conn, p.Adv, commands, pack, p.Verbose, p.OnRejection)
}

// PushObjects pushes locally-materialized objects to the target.
func PushObjects(
    ctx context.Context,
    conn Conn,
    adv *packp.AdvRefs,
    commands []PushCommand,
    store storer.Storer,
    hashes []plumbing.Hash,
    verbose bool,
    onRejection func(plumbing.ReferenceName, string),
) error {
    if len(commands) <= maxRefUpdatesPerPush {
        return pushObjectsBatch(ctx, conn, adv, commands, store, hashes, verbose, onRejection)
    }

updates := make([]PushCommand, 0, len(commands))
    var deletes []PushCommand
    for _, c := range commands {
        if c.Delete {
            deletes = append(deletes, c)
        } else {
            updates = append(updates, c)
        }
    }

if len(updates) > 0 {
        first, rest := splitFirstBatch(updates)
        if err := pushObjectsBatch(ctx, conn, adv, first, store, hashes, verbose, onRejection); err != nil {
            return err
        }
        if len(rest) > 0 {
            if err := PushCommands(ctx, conn, adv, rest, verbose, onRejection); err != nil {
                return err
            }
        }
    }
    if len(deletes) > 0 {
        return PushCommands(ctx, conn, adv, deletes, verbose, onRejection)
    }
    return nil
}

// TestChunkRefUpdates tests chunking of ref updates.
func TestChunkRefUpdates(t *testing.T) {
    require.Len(t, chunkRefUpdates(nil), 1)
    require.Len(t, chunkRefUpdates(make([]PushCommand, maxRefUpdatesPerPush)), 1)

batches := chunkRefUpdates(make([]PushCommand, maxRefUpdatesPerPush+1))
    require.Len(t, batches, 2)
    require.Len(t, batches[0], maxRefUpdatesPerPush)
    require.Len(t, batches[1], 1)
}

// TestPushCommandsBatchesOverCap guards that a ref-only push exceeding the
// per-request cap splits into multiple receive-pack requests, each within the
// cap, so the server's too-many-ref-update-commands limit isn't tripped.
func TestPushCommandsBatchesOverCap(t *testing.T) {
    // Test implementation
}

// TestPushPackBatchesOverCap guards that a pack push exceeding the per-request
// cap sends the pack with the first batch and the remaining refs as ref-only
// follow-up batches (the objects are already committed by the first request).
func TestPushPackBatchesOverCap(t *testing.T) {
    // Test implementation
}

```