Tolerate transient poll errors during clone wait · Entire
Tolerate transient poll errors during clone wait
e5692a0→main·
toothbrush·3w ago·2 files·+61 added/-22 removed
awaitMirrorReady previously aborted the whole wait on any GetMirror error, so a brief network/API glitch during a long initial clone failed mirror create even though the clone was still progressing. Retry on the poll interval, giving up only after maxConsecutivePollErrors consecutive failures (resets on success) or when the context ends — a persistent error (deleted mirror, revoked auth) still surfaces instead of spinning to the deadline. (Cursor)
Co-Authored-By: Claude Opus 4.8 (1M context) noreply@anthropic.com
Sessions
5d285b69feafView transcript
Changes
2
cmd/entire/cli
Mrepo_mirror_probe.go+34/-17
Mrepo_mirror_test.go+27/-5
66 unmodified lines
67
68
69
70
71
72
73
74
75
76
77
78
79
16 unmodified lines
96
97
98
92
99
100
101
102
103
12 unmodified lines
116
117
118
119
120
121
113
122
123
124
125
126
117
118
119
120
121
122
127
128
129
130
131
132
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
66 unmodified lines
// for the initial clone. A package var (not const) so tests can shorten it.
var mirrorPollInterval = 2 * time.Second
// maxConsecutivePollErrors bounds how many back-to-back GetMirror failures the
// clone wait tolerates before giving up. A brief network/API glitch during a
// long initial clone shouldn't fail the create, but a persistent error
// (deleted mirror, revoked auth) should surface rather than spin to the
// deadline. The counter resets on any successful poll.
const maxConsecutivePollErrors = 5
var (
// errMirrorCloneFailed reports the mirror's initial clone reached the
// terminal "failed" status — the server gave up cloning the upstream.
16 unmodified lines
// - nil when ready (the repo is clonable)
// - errMirrorCloneFailed when the initial clone failed
// - errMirrorSuspended when the placement is suspended
// - a timeout/transport err when the wait deadline passed or a poll errored
// - a timeout/transport err when the wait deadline passed, or polls kept
// erroring past maxConsecutivePollErrors (transient glitches are retried)
//
// "processing" keeps the loop running. This replaces the old smart-HTTP
// info/refs probe: the control plane now reports clone readiness directly via
12 unmodified lines
defer ticker.Stop()
var last coreapi.MirrorStatus
var consecutiveErrs int
for {
m, err := c.GetMirror(ctx, coreapi.GetMirrorParams{MirrorId: mirrorID})
if err != nil {
switch {
case err != nil:
if ctx.Err() != nil {
return last, classifyWaitContextErr(ctx.Err())
}
return last, fmt.Errorf("poll mirror status: %w", err)
}
if s, ok := m.Status.Get(); ok {
last = s
if onStatus != nil {
onStatus(s)
}
// Tolerate transient glitches: the clone may still be progressing,
// so retry on the next tick. Only give up once errors persist.
consecutiveErrs++
if consecutiveErrs >= maxConsecutivePollErrors {
return last, fmt.Errorf("poll mirror status: %w", err)
}
switch s {
case coreapi.MirrorStatusReady:
return s, nil
case coreapi.MirrorStatusFailed:
return s, errMirrorCloneFailed
case coreapi.MirrorStatusSuspended:
return s, errMirrorSuspended
case coreapi.MirrorStatusProcessing:
// keep waiting
}
}
}
select {
Mcmd/entire/cli/repo_mirror_probe.go+34/-17
28 unmodified lines
29
30
31
32
33
34
33
34
35
35
36
37
38
39
40
41
42
43
44
45
46
42
47
48
49
50
51
52
53
46
54
55
56
46 unmodified lines
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
28 unmodified lines
// fakeMirrorGetter feeds awaitMirrorReady a scripted sequence of statuses (the
// last entry repeats) or a fixed error, standing in for *coreapi.Client.GetMirror.
// errsBefore makes the first N calls return a transient error before the status
// sequence begins, to exercise the poll's retry tolerance.
type fakeMirrorGetter struct {
statuses []coreapi.MirrorStatus
err error
calls int
statuses []coreapi.MirrorStatus
err error
errBefore int
calls int
}
func (f *fakeMirrorGetter) GetMirror(_ context.Context, _ coreapi.GetMirrorParams) (*coreapi.Mirror, error) {
n := f.calls
f.calls++
if f.err != nil {
return nil, f.err
}
i := f.calls
if n < f.errsBefore {
return nil, errors.New("transient: connection reset")
}
i := n - f.errsBefore
if i >= len(f.statuses) {
i = len(f.statuses) - 1
}
f.calls++
m := &coreapi.Mirror{}
m.Status = coreapi.NewOptMirrorStatus(f.statuses[i])
return m, nil
}
_, err := awaitMirrorReady(ctx, f, "m", 20*time.Millisecond, nil)
require.ErrorIs(t, err, context.DeadlineExceeded)
t.Run("transient errors are tolerated, then ready", func(t *testing.T) {
// Fewer consecutive errors than the cap, so the poll rides them out.
f := &fakeMirrorGetter{errsBefore: maxConsecutivePollErrors - 1, statuses: []coreapi.MirrorStatus{coreapi.MirrorStatusReady}}
status, err := awaitMirrorReady(ctx, f, "m", time.Second, nil)
require.NoError(t, err)
require.Equal(t, coreapi.MirrorStatusReady, status)
})
t.Run("persistent errors give up after the cap", func(t *testing.T) {
f := &fakeMirrorGetter{err: errors.New("boom")}
_, err := awaitMirrorReady(ctx, f, "m", time.Second, nil)
require.ErrorContains(t, err, "poll mirror status")
require.Equal(t, maxConsecutivePollErrors, f.calls, "should stop at the cap, not spin to the deadline")
})
}
// TestReportOneShotMirror exercises the one-shot create's presentation across