generalize checkpoint ref reconciliation · Entire
generalize checkpoint ref reconciliation
461f9fd→main·
pfleidi·1mo ago·12 files·+204 added/-110 removed
Reconcile disconnected metadata against the ref being synced so custom checkpoint refs preserve local data before rebasing.
Rename push sync helpers around ref terminology and refresh stale explain/doc references.
Sessions
656b0c34301eView transcript
[?
Fix Review Attach Metadata and Checkpoint ReconciliationCodex·GPT-5.5·1 step](/content/gh/entireio/cli/session/019e953d-3ce8-7881-a9eb-31211a16b642#timeline-656b0c34301e/index.html)
Changes
12
cmd/entire/cli
checkpoint/remote
Mgit_test.go+1/-1
Mdoctor.go+1/-1
Mexplain.go+4/-4
integration_test
Mhttp_remote_test.go+3/-3
Mremote_operations_test.go+6/-6
strategy
Mcheckpoint_remote.go+1/-1
Mmetadata_reconcile.go+23/-16
Mmetadata_reconcile_test.go+17/-13
Mpush_common.go+15/-15
Mpush_common_budget_unix_test.go+3/-3
docs/architecture
Msessions-and-checkpoints.md+3/-2
144 unmodified lines
145
146
147
148
148
149
150
151
144 unmodified lines
{
// Without checkpoint_remote configured the push should use the
// remote name so git updates refs/remotes/origin/<branch> and
// subsequent hasUnpushedSessionsCommon checks can short-circuit.
// subsequent hasUnpushedBranchRef checks can short-circuit.
name: "no checkpoint remote keeps remote name",
originURL: "git@github.com:acme/app.git",
settingsJSON: `{\"enabled\":true}`,
Mcmd/entire/cli/checkpoint/remote/git_test.go+1/-1
378 unmodified lines
379
380
381
382
382
383
384
385
378 unmodified lines
}
}
if fixErr := strategy.ReconcileDisconnectedMetadataBranch(ctx, repo, remoteRefName, cmd.ErrOrStderr()); fixErr != nil {
if fixErr := strategy.ReconcileDisconnectedMetadataRef(ctx, repo, refs.Primary, remoteRefName, cmd.ErrOrStderr()); fixErr != nil {
return fmt.Errorf("failed to reconcile metadata branches: %w", fixErr)
}
Mcmd/entire/cli/doctor.go+1/-1
765 unmodified lines
766
767
768
769
769
770
771
772
773
772
773
774
775
776
3 unmodified lines
780
781
782
783
783
784
785
786
765 unmodified lines
loadPrimaryRoot := func(repo *git.Repository) (*object.Tree, error) {
return loadPrimaryMetadataRootTree(ctx, repo, refs)
}
v1FT := buildCheckpointFetchingTree(ctx, repo, cpID, "primary", loadPrimaryRoot)
primaryFT := buildCheckpointFetchingTree(ctx, repo, cpID, "primary", loadPrimaryRoot)
missingCount := 0
if v1FT != nil {
missingCount += len(v1FT.CollectMissingBlobs())
if primaryFT != nil {
missingCount += len(primaryFT.CollectMissingBlobs())
}
if missingCount == 0 {
return
}
Mcmd/entire/cli/explain.go+4/-4
197 unmodified lines
198
199
200
201
201
202
203
204
47 unmodified lines
252
253
254
255
256
255
256
257
258
259
197 unmodified lines
// checkpoint branch to an HTTPS remote when ENTIRE_CHECKPOINT_TOKEN is set.
// This exercises:
// - remote.newCommand HTTPS protocol detection and token injection
// - tryPushSessionsCommon over HTTPS with Authorization header
// - tryPushRefCommon over HTTPS with Authorization header
// - go-git backend.requireReceivePackAuth validates the header
func TestHTTPS_PushCheckpointBranchToRemote(t *testing.T) {
t.Parallel()
47 unmodified lines
//
// Code paths exercised:
// - resolvePushSettings -> PushURL -> deriveCheckpointURLFromInfo (push routing)
// - fetchAndRebaseSessionsCommon with checkpoint URL target (fetch routing)
// - tryPushSessionsCommon retry after rebase (push retry)
// - fetchAndRebaseRefCommon with checkpoint URL target (fetch routing)
// - tryPushRefCommon retry after rebase (push retry)
func TestHTTPS_CheckpointRemoteRoutesToSeparateRepo(t *testing.T) {
t.Parallel()
Mcmd/entire/cli/integration_test/http_remote_test.go+3/-3
171 unmodified lines
172
173
174
175
175
176
177
178
2 unmodified lines
181
182
183
184
185
184
185
186
187
188
340 unmodified lines
529
530
531
532
532
533
534
535
534
535
536
537
538
171 unmodified lines
//
// This is a data routing verification test. It validates that when the production
// code's pushBranchIfNeeded is called with different targets for checkpoints,
// code's pushRefIfNeeded is called with different targets for checkpoints,
// the branches land on the correct remotes with correct data.
//
// Why not test through PrePush directly: resolvePushSettings derives the checkpoint
2 unmodified lines
// origin. The URL derivation logic is unit-tested in checkpoint_remote_test.go
// (TestDeriveCheckpointURL, TestResolvePushSettings_WithCheckpointRemote_*).
//
// The pushBranchIfNeeded function (which PrePush calls with the resolved target)
// is exercised in push_common_test.go:TestPushBranchIfNeeded_LocalBareRepo_PushesSuccessfully,
// The pushRefIfNeeded function (which PrePush calls with the resolved target)
// is exercised in push_common_test.go:TestPushRefIfNeeded_LocalBareRepo_PushesSuccessfully,
// verifying it works with local bare repo paths.
func TestPrePush_CheckpointRemoteRoutesToSeparateRemote(t *testing.T) {
t.Parallel()
340 unmodified lines
// origin. Trails are always pushed to origin regardless of checkpoint_remote config.
// This test exercises the PrePush -> resolvePushSettings -> fallback code path.
// The actual doPushBranch graceful degradation (push to unreachable URL returns nil,
// The actual doPushRef graceful degradation (push to unreachable URL returns nil,
// not an error) is tested in push_common_test.go:
// - TestDoPushBranch_UnreachableTarget_ReturnsNil
// - TestPushBranchIfNeeded_UnreachableTarget_ReturnsNil
// - TestDoPushRef_UnreachableTarget_ReturnsNil
// - TestPushRefIfNeeded_UnreachableTarget_ReturnsNil
func TestGracefulDegradation_UnreachableCheckpointRemotePushContinues(t *testing.T) {
t.Parallel()
env := NewFeatureBranchEnv(t)
Mcmd/entire/cli/integration_test/remote_operations_test.go+6/-6
49 unmodified lines
50
51
52
53
53
54
55
56
49 unmodified lines
// - Skips if the push remote owner differs from the checkpoint repo owner (fork detection)
// - If a checkpoint branch doesn't exist locally, attempts to fetch it from the URL
//
// The push itself handles failures gracefully (doPushBranch warns and continues),
// The push itself handles failures gracefully (doPushRef warns and continues),
// so no reachability check is needed here.
func resolvePushSettings(ctx context.Context, pushRemoteName string) pushSettings {
s, err := settings.Load(ctx)
Mcmd/entire/cli/strategy/checkpoint_remote.go+1/-1
92 unmodified lines
93
94
95
96
97
96
97
98
99
99
100
101
102
2 unmodified lines
105
106
107
108
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
116
117
123
124
125
119
126
127
128
122
129
130
131
125
132
133
134
128
135
136
137
131
138
139
140
141
12 unmodified lines
154
155
156
150
157
158
159
160
28 unmodified lines
189
190
191
185
186
192
193
194
195
196
6 unmodified lines
203
204
205
199
200
206
207
208
209
210
92 unmodified lines
})
}
// ReconcileDisconnectedMetadataBranch detects and repairs disconnected local/remote
// entire/checkpoints/v1 branches. Disconnected means no common ancestor, which
// ReconcileDisconnectedMetadataRef detects and repairs disconnected local/remote
// metadata refs. Disconnected means no common ancestor, which
// only happens due to the empty-orphan bug. Diverged (shared ancestor) is normal
// and handled by the push path's tree merge.
// and handled by the push path.
//
// Repair strategy: cherry-pick local commits onto remote tip, preserving all data.
// Checkpoint shards use unique paths (<id[:2]>/<id[2:]>/), so cherry-picks always
2 unmodified lines
// Progress messages are written to w (typically os.Stderr for hooks or
// cmd.ErrOrStderr() for commands).
// The remote ref can be either a remote-tracking ref or a temporary fetched ref.
func ReconcileDisconnectedMetadataBranch(
func ReconcileDisconnectedMetadataRef(
ctx context.Context,
repo *git.Repository,
localRefName plumbing.ReferenceName,
remoteRefName plumbing.ReferenceName,
w io.Writer,
) error {
refs := checkpoint.ResolveCommittedRefs(ctx)
advance := func(hash plumbing.Hash) error {
if localRefName == refs.Primary {
return AdvanceCommittedPrimary(ctx, repo, refs, hash)
}
return repo.Storer.SetReference(plumbing.NewHashReference(localRefName, hash))
}
// Check local primary ref
localRef, err := repo.Reference(refs.Primary, true)
// Check local ref
localRef, err := repo.Reference(localRefName, true)
if errors.Is(err, plumbing.ErrReferenceNotFound) {
return nil // No local branch — nothing to reconcile
return nil // No local ref — nothing to reconcile
}
if err != nil {
return fmt.Errorf("failed to check local metadata branch: %w", err)
return fmt.Errorf("failed to check local metadata ref: %w", err)
}
// Check remote-tracking branch
// Check remote-tracking or fetched ref
remoteRef, err := repo.Reference(remoteRefName, true)
if errors.Is(err, plumbing.ErrReferenceNotFound) {
return nil // No remote branch — nothing to reconcile
return nil // No remote ref — nothing to reconcile
}
if err != nil {
return fmt.Errorf("failed to check remote metadata branch: %w", err)
return fmt.Errorf("failed to check remote metadata ref: %w", err)
}
localHash := localRef.Hash()
12 unmodified lines
disconnected, err := isDisconnected(ctx, repoPath, localHash.String(), remoteHash.String())
if err != nil {
return fmt.Errorf("failed to check metadata branch ancestry: %w", err)
return fmt.Errorf("failed to check metadata ref ancestry: %w", err)
}
if !disconnected {
// Shared ancestry (diverged or ancestor) — not our problem
28 unmodified lines
if len(dataCommits) == 0 {
// Local only had empty orphan — just point to remote
if err := AdvanceCommittedPrimary(ctx, repo, refs, remoteHash); err != nil {
return fmt.Errorf("failed to reset metadata branch to remote: %w", err)
if err := advance(remoteHash); err != nil {
return fmt.Errorf("failed to reset metadata ref to remote: %w", err)
}
fmt.Fprintln(w, "[entire] Done — local had no checkpoint data, reset to remote")
return nil
}
6 unmodified lines
return fmt.Errorf("failed to cherry-pick local commits onto remote: %w", err)
}
if err := AdvanceCommittedPrimary(ctx, repo, refs, newTip); err != nil {
return fmt.Errorf("failed to update metadata branch: %w", err)
if err := advance(newTip); err != nil {
return fmt.Errorf("failed to update metadata ref: %w", err)
}
fmt.Fprintln(w, "[entire] Done — all local and remote checkpoints preserved")
Mcmd/entire/cli/strategy/metadata_reconcile.go+23/-16
23 unmodified lines
24
25
26
27
28
29
30
31
32
33
32 unmodified lines
66
67
68
65
69
70
71
72
11 unmodified lines
84
85
86
83
87
88
89
90
15 unmodified lines
106
107
108
105
109
110
111
112
34 unmodified lines
147
148
149
146
150
151
152
153
40 unmodified lines
194
195
196
193
194
197
198
199
200
201
94 unmodified lines
296
297
298
295
299
300
301
302
65 unmodified lines
368
369
370
367
368
371
372
373
374
375
288 unmodified lines
664
665
666
663
664
667
668
669
670
671
183 unmodified lines
855
856
857
854
858
859
860
861
43 unmodified lines
905
906
907
904
908
909
910
911
23 unmodified lines
return plumbing.NewRemoteReferenceName("origin", paths.MetadataBranchName)
}
func metadataLocalRef() plumbing.ReferenceName {
return plumbing.NewBranchReferenceName(paths.MetadataBranchName)
}
func TestReconcileDisconnected_NoRemote(t *testing.T) {
t.Parallel()
32 unmodified lines
}
// Should be a no-op (no remote)
if err := ReconcileDisconnectedMetadataBranch(context.Background(), repo, metadataOriginRemoteRef(), io.Discard); err != nil {
if err := ReconcileDisconnectedMetadataRef(context.Background(), repo, metadataLocalRef(), metadataOriginRemoteRef(), io.Discard); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
11 unmodified lines
}
// No local branch → no-op
if err := ReconcileDisconnectedMetadataBranch(context.Background(), repo, metadataOriginRemoteRef(), io.Discard); err != nil {
if err := ReconcileDisconnectedMetadataRef(context.Background(), repo, metadataLocalRef(), metadataOriginRemoteRef(), io.Discard); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
15 unmodified lines
}
// Same hash → no-op
if err := ReconcileDisconnectedMetadataBranch(context.Background(), repo, metadataOriginRemoteRef(), io.Discard); err != nil {
if err := ReconcileDisconnectedMetadataRef(context.Background(), repo, metadataLocalRef(), metadataOriginRemoteRef(), io.Discard); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
34 unmodified lines
}
// Shared ancestry → no-op
if err := ReconcileDisconnectedMetadataBranch(context.Background(), repo, metadataOriginRemoteRef(), io.Discard); err != nil {
if err := ReconcileDisconnectedMetadataRef(context.Background(), repo, metadataLocalRef(), metadataOriginRemoteRef(), io.Discard); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
40 unmodified lines
}
// Run reconciliation
if err := ReconcileDisconnectedMetadataBranch(context.Background(), repo, metadataOriginRemoteRef(), io.Discard); err != nil {
t.Fatalf("ReconcileDisconnectedMetadataBranch() failed: %v", err)
if err := ReconcileDisconnectedMetadataRef(context.Background(), repo, metadataLocalRef(), metadataOriginRemoteRef(), io.Discard); err != nil {
t.Fatalf("ReconcileDisconnectedMetadataRef() failed: %v", err)
}
// Verify result
94 unmodified lines
_, ok := v1CustomRefHash(t, repo)
require.False(t, ok, "custom ref should not exist before reconciliation")
require.NoError(t, ReconcileDisconnectedMetadataBranch(t.Context(), repo, metadataOriginRemoteRef(), io.Discard))
require.NoError(t, ReconcileDisconnectedMetadataRef(t.Context(), repo, metadataLocalRef(), metadataOriginRemoteRef(), io.Discard))
localRef, err := repo.Reference(plumbing.NewBranchReferenceName(paths.MetadataBranchName), true)
require.NoError(t, err)
65 unmodified lines
}
// Verify the MODIFIED metadata.json has session_count:2, not the original 1
183 unmodified lines
remoteRef, err := repo.Reference(remoteRefName, true)
require.NoError(t, err)
err = ReconcileDisconnectedMetadataBranch(context.Background(), repo, metadataOriginRemoteRef(), io.Discard)
err = ReconcileDisconnectedMetadataRef(context.Background(), repo, metadataLocalRef(), metadataOriginRemoteRef(), io.Discard)
require.NoError(t, err)
// Local branch should now point to the remote tip (reset, not cherry-picked)
43 unmodified lines
repo, err := git.PlainOpen(cloneDir)
require.NoError(t, err)
// Verify merged tree: should have remote data + first local checkpoint,
Mcmd/entire/cli/strategy/metadata_reconcile_test.go+17/-13
44 unmodified lines
45
46
47
48
48
49
50
51
52
53
54
55
55
56
57
57
58
59
60
19 unmodified lines
80
81
82
83
83
84
85
86
6 unmodified lines
93
94
95
96
96
97
98
99
14 unmodified lines
114
115
116
117
117
118
119
120
8 unmodified lines
129
130
131
132
132
133
134
135
97 unmodified lines
233
234
235
236
236
237
238
239
240
240
241
242
243
95 unmodified lines
339
340
341
342
343
344
342
343
344
345
346
346
347
348
349
47 unmodified lines
397
398
399
400
401
400
401
402
403
404
44 unmodified lines
\
}
if ref.IsBranch() && !remote.IsURL(target) && !hasUnpushedSessionsCommon(repo, target, localRef.Hash(), ref.Short()) {
if ref.IsBranch() && !remote.IsURL(target) && !hasUnpushedBranchRef(repo, target, localRef.Hash(), ref.Short()) {
return nil
}
return doPushRef(ctx, target, ref)
}
// hasUnpushedSessionsCommon checks if the local branch differs from the remote.
// hasUnpushedBranchRef checks if the local branch differs from the remote.
// Returns true if there's any difference that needs syncing (local ahead, remote ahead, or diverged).
func hasUnpushedSessionsCommon(repo *git.Repository, remoteName string, localHash plumbing.Hash, branchName string) bool {
func hasUnpushedBranchRef(repo *git.Repository, remoteName string, localHash plumbing.Hash, branchName string) bool {
// Check for remote tracking ref: refs/remotes/<remoteName>/<branch>
remoteRefName := plumbing.NewRemoteReferenceName(remoteName, branchName)
remoteRef, err := repo.Reference(remoteRefName, true)
19 unmodified lines
// can shrink it.
var checkpointPushBudget = 2 * time.Minute
// doPushRef pushes the given ref to the target with fetch+merge recovery.
// doPushRef pushes the given ref to the target with fetch+rebase recovery.
// The target can be a remote name or a URL.
func doPushRef(ctx context.Context, target string, ref plumbing.ReferenceName) error {
ctx, cancel := context.WithTimeout(ctx, checkpointPushBudget)
6 unmodified lines
// Try pushing first
result, err := tryPushSessionsCommon(ctx, target, ref)
result, err := tryPushRefCommon(ctx, target, ref)
if err == nil {
finishPush(ctx, stop, result, target)
return nil
14 unmodified lines
stop = startProgressDots(os.Stderr)
frCtx, fetchRebaseSpan := perf.Start(ctx, "fetch_and_rebase")
syncErr := fetchAndRebaseSessionsCommon(frCtx, target, ref)
syncErr := fetchAndRebaseRefCommon(frCtx, target, ref)
fetchRebaseSpan.RecordError(syncErr)
fetchRebaseSpan.End()
if syncErr != nil {
8 unmodified lines
fmt.Fprintf(os.Stderr, "[entire] Pushing %s to %s...", refLabel, displayTarget)
stop = startProgressDots(os.Stderr)
if result, err := tryPushSessionsCommon(ctx, target, ref); err != nil {
if result, err := tryPushRefCommon(ctx, target, ref); err != nil {
stop("")
fmt.Fprintf(os.Stderr, "[entire] Warning: failed to push %s after sync: %v\n", refLabel, err)
printCheckpointRemoteHint(target)
97 unmodified lines
}
}
// tryPushSessionsCommon attempts to push a ref. No timeout of its own —
// tryPushRefCommon attempts to push a ref. No timeout of its own —
// runs under doPushRef's shared budget. Branch refs use a bare branch-name
// refSpec so existing remote-tracking works; non-branch refs use a force
// refspec ("+refs/...:refs/...") with no tracking shadow.
func tryPushSessionsCommon(ctx context.Context, remoteName string, ref plumbing.ReferenceName) (pushResult, error) {
func tryPushRefCommon(ctx context.Context, remoteName string, ref plumbing.ReferenceName) (pushResult, error) {
refSpec := ref.Short()
if !ref.IsBranch() {
refSpec = " + " + ref.String() + ":" + ref.String()
95 unmodified lines
fmt.Fprintln(w, banner)
// fetchAndRebaseSessionsCommon fetches remote sessions and rebases local commits
// on top of the remote tip. Since checkpoint shards use unique paths, rebases
// always apply cleanly.
// fetchAndRebaseRefCommon fetches a remote ref and rebases local commits on top
// of the remote tip. Since checkpoint shards use unique paths, rebases
// always apply cleanly.
// The target can be a remote name or a URL.
func fetchAndRebaseSessionsCommon(ctx context.Context, target string, ref plumbing.ReferenceName) error {
func fetchAndRebaseRefCommon(ctx context.Context, target string, ref plumbing.ReferenceName) error {
// No timeout: runs under doPushRef's shared budget.
fetchTarget, err := remote.ResolveFetchTarget(ctx, target)
if err != nil {
47 unmodified lines
// can compare fresh local vs remote. If disconnected (empty-orphan bug),
// this cherry-picks local commits onto remote tip, updating the local ref.
// If reconciliation fails, abort — proceeding to rebase on disconnected
// branches would silently combine unrelated histories.
if reconcileErr := ReconcileDisconnectedMetadataBranch(ctx, repo, fetchedRefName, os.Stderr); reconcileErr != nil {
// refs would silently combine unrelated histories.
if reconcileErr := ReconcileDisconnectedMetadataRef(ctx, repo, ref, fetchedRefName, os.Stderr); reconcileErr != nil {
return fmt.Errorf("metadata reconciliation failed: %w", reconcileErr)
}