status, doctor: finalize sessions whose agent has exited · Entire
status, doctor: finalize sessions whose agent has exited
439be1e→main·
Soph·3w ago·7 files·+300 added/-34 removed
Extract endSessionNow — the markSessionEnded + eager-condense sequence the SessionStop hook already runs — and share it with a new finalizeExitedSessions sweep, so the hook and the sweep stay in lockstep. markSessionEnded gains an optional guard so the sweep re-checks OwnerExited on the freshly-loaded state under the session-state lock, closing a TOCTOU race where a turn could revive a session between the list snapshot and the finalize.
entire status (human and --json) and entire doctor run the sweep up front, finalizing any ACTIVE session whose owner process is gone instead of leaving it "active" until the 1h StuckActiveThreshold. After finalizing, the sweep reloads each session from disk so callers see the true post-finalize state (condense is fail-open, so StepCount/ FullyCondensed are never assumed). Both surfaces also label such sessions "exited" (human output, status --json, doctor's stuck-session reason) as a fallback when finalization can't run.
Co-Authored-By: Claude Opus 4.8 (1M context) noreply@anthropic.com
Sessions
3ee2f4dbe270View transcript
Changes
7
cmd/entire/cli
Mdoctor.go+22/-6
Mlifecycle.go+44/-20
Mphase_wiring_test.go+5/-5
Asession_finalize.go+76
Asession_finalize_test.go+127
Msessions.go+1/-1
Mstatus.go+25/-2
119 unmodified lines
120
121
122
123
124
125
126
127
128
129
130
131
132
133
84 unmodified lines
218
219
220
213
214
215
216
221
218
222
223
224
225
226
227
228
229
230
231
232
233
234
235
220
236
237
238
239
119 unmodified lines
}
defer repo.Close()
// Finalize any ACTIVE session whose agent process has exited (no SessionStop
// hook fired). A gone process is unambiguous, so these are condensed on the
// spot rather than left for the interactive prompt below; the sweep marks
// them ended in place so classifySession won't re-flag them.
if n := finalizeExitedSessions(ctx, states); n > 0 {
fmt.Fprintf(cmd.OutOrStdout(), "Finalized %d exited session(s) (agent process gone).
\n", n)
}
// Identify stuck sessions
now := time.Now()
var stuck []stuckSession
84 unmodified lines
switch {
case state.Phase.IsActive():
if !state.IsStuckActive() {
return nil
}
var reason string
if state.LastInteractionTime != nil {
switch {
case state.OwnerExited():
// Detected immediately (no timeout wait): the owning agent process
// is gone. Normally finalized up front in runSessionsFix; this
// branch covers a session that couldn't be finalized there.
pid := 0
if state.Owner != nil {
pid = state.Owner.PID
}
reason = fmt.Sprintf("agent process %d exited (no longer running)", pid)
case !state.IsStuckActive():
return nil
case state.LastInteractionTime != nil:
reason = fmt.Sprintf("active, last interaction %s ago", now.Sub(*state.LastInteractionTime).Truncate(time.Minute))
} else {
default:
reason = fmt.Sprintf("active, started %s ago with no recorded interaction", now.Sub(state.StartedAt).Truncate(time.Minute))
}
Mcmd/entire/cli/doctor.go+22/-6
932 unmodified lines
933
934
935
936
936
937
938
939
940
941
939
940
944
945
946
947
948
949
950
951
952
953
954
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
209 unmodified lines
1183
1184
1185
1173
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
3 unmodified lines
1204
1205
1206
1207
1208
1209
1186
1187
1210
1211
1212
1213
1190
1214
1215
1192
1216
1217
1218
1219
932 unmodified lines
// the transcript to extract file changes. Cleanup is handled by
// `entire clean` or when the session state is fully removed.
if err := markSessionEnded(ctx, event, event.SessionID); err != nil {
if _, err := endSessionNow(ctx, event, event.SessionID, nil); err != nil {
logging.Warn(logCtx, "failed to mark session ended",
slog.String("error", err.Error()))
// Don't attempt eager condense if we couldn't even mark the session ended —
// the session state may be in an inconsistent state.
return nil
}
// Eagerly condense session data so PostCommit doesn't have to process it.
// This prevents zombie ENDED sessions from accumulating and causing O(N)
// overhead on every future commit (GitHub issue #591).
// Fail-open: if this fails, PostCommit will still process it on the next commit.
strat := GetStrategy(ctx)
if err := strat.CondenseAndMarkFullyCondensed(ctx, event.SessionID); err != nil {
logging.Warn(logCtx, "eager condense on session stop failed",
slog.String("session_id", event.SessionID),
slog.String("error", err.Error()))
}
return nil
}
// endSessionNow runs the canonical "this session is over" sequence: it marks the
// session ended (firing the SessionStop transition → PhaseEnded + EndedAt) and
// eagerly condenses its pending work so PostCommit need not. This prevents
// zombie ENDED sessions from accumulating and causing O(N) overhead on every
// future commit (GitHub issue #591). It is shared by the SessionStop hook
// (handleLifecycleSessionEnd) and the exited-session sweep
// (finalizeExitedSessions), so the two stay in lockstep.
//
// The condense is fail-open (PostCommit retries on the next commit); an error
// marking the session ended is returned so callers can react, and skips the
// condense since the state may be inconsistent. event may be nil when no hook
// event drives the end (the sweep), which skips event-metadata persistence.
// guard is forwarded to markSessionEnded (see there); when it skips the end,
// the condense is skipped too and ended is false.
func endSessionNow(ctx context.Context, event *agent.Event, sessionID string, guard func(*strategy.SessionState) bool) (ended bool, err error) {
ended, err = markSessionEnded(ctx, event, sessionID, guard)
if err != nil || !ended {
return ended, err
}
if condErr := GetStrategy(ctx).CondenseAndMarkFullyCondensed(ctx, sessionID); condErr != nil {
logging.Warn(logging.WithComponent(ctx, "lifecycle"), "eager condense on session end failed",
slog.String("session_id", sessionID),
slog.String("error", condErr.Error()))
}
return true, nil
}
// handleLifecycleSubagentStart handles subagent start: captures pre-task state.
func handleLifecycleSubagentStart(ctx context.Context, ag agent.Agent, event *agent.Event) error {
logCtx := logging.WithAgent(logging.WithComponent(ctx, "lifecycle"), ag.Name())
209 unmodified lines
// markSessionEnded transitions the session to ENDED phase via the state machine.
// If event is non-nil, hook-provided metrics are persisted to state before saving.
func markSessionEnded(ctx context.Context, event *agent.Event, sessionID string) error {
// markSessionEnded fires the SessionStop transition (PhaseEnded + EndedAt) under
// the session-state lock. When guard is non-nil and returns false on the
// freshly-loaded state, the transition is skipped — callers use it to
// re-validate a precondition that may have changed since their snapshot (the
// exited-session sweep re-checks OwnerExited under the lock so it never ends a
// session a concurrent turn just revived). It reports whether the session was
// actually ended.
func markSessionEnded(ctx context.Context, event *agent.Event, sessionID string, guard func(*strategy.SessionState) bool) (ended bool, err error) {
mutErr := strategy.MutateSessionState(ctx, sessionID, func(state *strategy.SessionState) error {
if guard != nil && !guard(state) {
return strategy.ErrMutationSkip
}
if event != nil {
persistEventMetadataToState(event, state)
}
now := time.Now()
state.EndedAt = &now
ended = true
return nil
})
if errors.Is(mutErr, strategy.ErrStateNotFound) {
return nil
if errors.Is(mutErr, strategy.ErrStateNotFound) || errors.Is(mutErr, strategy.ErrMutationSkip) {
return false, nil
}
if mutErr != nil {
return fmt.Errorf("failed to save session state: %w", mutErr)
}
return nil
return ended, nil
}
// logFileChanges logs the files modified, created, and deleted during a session.
Mcmd/entire/cli/lifecycle.go+44/-20
29 unmodified lines
30
31
32
33
33
34
35
36
23 unmodified lines
60
61
62
63
63
64
65
66
18 unmodified lines
85
86
87
88
88
89
90
91
18 unmodified lines
110
111
112
113
113
114
115
116
8 unmodified lines
125
126
127
128
128
129
130
131
29 unmodified lines
\trequire.NoError(t, err)
// Call markSessionEnded
err = markSessionEnded(context.Background(), nil, "test-session-end-1")
_, err = markSessionEnded(context.Background(), nil, "test-session-end-1", nil)
require.NoError(t, err)
// Verify phase is ENDED
23 unmodified lines
err := strategy.SaveSessionState(context.Background(), state)
require.NoError(t, err)
err = markSessionEnded(context.Background(), nil, "test-session-end-idle")
_, err = markSessionEnded(context.Background(), nil, "test-session-end-idle", nil)
require.NoError(t, err)
loaded, err := strategy.LoadSessionState(context.Background(), "test-session-end-idle")
18 unmodified lines
err := strategy.SaveSessionState(context.Background(), state)
require.NoError(t, err)
err = markSessionEnded(context.Background(), nil, "test-session-end-noop")
_, err = markSessionEnded(context.Background(), nil, "test-session-end-noop", nil)
require.NoError(t, err)
loaded, err := strategy.LoadSessionState(context.Background(), "test-session-end-noop")
18 unmodified lines
err := strategy.SaveSessionState(context.Background(), state)
require.NoError(t, err)
err = markSessionEnded(context.Background(), nil, "test-session-end-compat")
_, err = markSessionEnded(context.Background(), nil, "test-session-end-compat", nil)
require.NoError(t, err)
loaded, err := strategy.LoadSessionState(context.Background(), "test-session-end-compat")
8 unmodified lines
dir := setupGitRepoForPhaseTest(t)
t.Chdir(dir)
err = markSessionEnded(context.Background(), nil, "nonexistent-session")
_, err := markSessionEnded(context.Background(), nil, "nonexistent-session", nil)
assert.NoError(t, err, "should be a no-op when no state exists")
}
Acmd/entire/cli/session_finalize.go+76
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
package cli
import (
"context"
"log/slog"
"time"
"github.com/entireio/cli/cmd/entire/cli/logging"
"github.com/entireio/cli/cmd/entire/cli/session"
)
// finalizeExitedSessions finalizes every ACTIVE session in states whose owning
// agent process has exited (clean /exit, crash, kill, terminal close, reboot)
// without a SessionStop hook firing. Each such session is finalized exactly as a
// clean session stop would be: the session-stop transition runs (PhaseEnded +
// EndedAt) and pending work is eagerly condensed.
//
// It refreshes the matched in-memory states from disk after finalizing — so
// callers can re-filter/re-render without their own reload — and returns the
// number finalized. Each session is best-effort: a failure to mark one ended is
// logged and skipped; a condense failure is logged but the session is still
// counted (PostCommit will retry the condense later).
func finalizeExitedSessions(ctx context.Context, states []*session.State) int {
logCtx := logging.WithComponent(ctx, "session")
var store *session.StateStore // lazily created on first finalize
finalized := 0
for _, st := range states {
if !st.OwnerExited() {
continue // cheap pre-filter on the (possibly stale) list snapshot
}
// Finalize via the same path a clean SessionStop hook would take, but
// re-validate OwnerExited on the freshly-loaded state under the lock:
// a turn may have started since the snapshot and replaced the dead
// owner with a live one, in which case ended is false and we leave it be.
ended, err := endSessionNow(ctx, nil, st.SessionID, func(s *session.State) bool {
return s.OwnerExited()
})
if err != nil {
logging.Warn(logCtx, "failed to finalize exited session",
slog.String("session_id", st.SessionID),
slog.String("error", err.Error()))
continue
}
if !ended {
continue
}
// Refresh the in-memory snapshot from disk so downstream filtering and
// doctor classification see the true post-finalize state: ended, and
// condensed only if the eager condense actually succeeded (it is
// fail-open, so StepCount/FullyCondensed must not be assumed). Fall back
// to a minimal ended-marking if the reload fails — enough for the
// caller's "active" filter to drop it.
if store == nil {
if s, serr := session.NewStateStore(ctx); serr == nil {
store = s
}
}
refreshed := false
if store != nil {
if reloaded, lerr := store.Load(ctx, st.SessionID); lerr == nil && reloaded != nil {
*st = *reloaded
refreshed = true
}
}
if !refreshed {
now := time.Now()
st.Phase = session.PhaseEnded
st.EndedAt = &now
}
finalized++
}
return finalized
}
Acmd/entire/cli/session_finalize_test.go+127
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
package cli
import (
"context"
"os"
"testing"
"time"
"github.com/entireio/cli/cmd/entire/cli/proclive"
"github.com/entireio/cli/cmd/entire/cli/session"
)
// TestFinalizeExitedSessions finalizes an ACTIVE session whose owner process is
// gone, and leaves an ACTIVE session without a recorded owner untouched.
//
// Not parallel: setupAttachTestRepo uses t.Chdir.
func TestFinalizeExitedSessions(t *testing.T) {
setupAttachTestRepo(t)
ctx := context.Background()
store, err := session.NewStateStore(ctx)
if err != nil {
t.Fatal(err)
}
// Owner with a mismatched start fingerprint reads as a reused (dead) PID, a
// deterministic "agent exited" signal on linux/darwin.
exited := &session.State{
SessionID: "exited-session",
Phase: session.PhaseActive,
StartedAt: time.Now(),
Owner: &proclive.Identity{PID: os.Getpid(), Start: "bogus-start-fingerprint"},
}
// No owner recorded: must be left alone (liveness unknown → timeout fallback).
noOwner := &session.State{
SessionID: "no-owner-session",
Phase: session.PhaseActive,
StartedAt: time.Now(),
}
for _, s := range []*session.State{exited, noOwner} {
if err := store.Save(ctx, s); err != nil {
t.Fatalf("save %s: %v", s.SessionID, err)
}
}
states, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
if n := finalizeExitedSessions(ctx, states); n != 1 {
t.Fatalf("finalizeExitedSessions = %d, want 1", n)
}
// The exited session is now ended on disk.
got, err := store.Load(ctx, "exited-session")
if err != nil {
t.Fatal(err)
}
if got.EndedAt == nil {
t.Error("exited session EndedAt = nil, want set")
}
if got.Phase != session.PhaseEnded {
t.Errorf("exited session Phase = %q, want %q", got.Phase, session.PhaseEnded)
}
// The owner-less session is untouched.
got, err = store.Load(ctx, "no-owner-session")
if err != nil {
t.Fatal(err)
}
if got.EndedAt != nil {
t.Error("no-owner session EndedAt set, want nil (left active)")
}
}
// TestFinalizeExitedSessions_RevalidatesUnderLock guards against the
// time-of-check/time-of-use race: the sweep must re-check OwnerExited on the
// freshly-loaded state, not act on a stale list snapshot. Here the on-disk
// state has a LIVE owner while the snapshot passed to the sweep carries a dead
// one (as if a turn revived the session after the list was taken).
//
// Not parallel: setupAttachTestRepo uses t.Chdir.
func TestFinalizeExitedSessions_RevalidatesUnderLock(t *testing.T) {
setupAttachTestRepo(t)
ctx := context.Background()
liveOwner, ok := proclive.ResolveOwner()
if !ok {
t.Skip("no stable process owner resolvable in this environment")
}
store, err := session.NewStateStore(ctx)
if err != nil {
t.Fatal(err)
}
if err := store.Save(ctx, &session.State{
SessionID: "revived",
Phase: session.PhaseActive,
StartedAt: time.Now(),
Owner: &liveOwner, // on disk: a live owner
}); err != nil {
t.Fatal(err)
}
// Stale snapshot the sweep sees: same session, but with a dead owner.
stale := &session.State{
SessionID: "revived",
Phase: session.PhaseActive,
StartedAt: time.Now(),
Owner: &proclive.Identity{PID: os.Getpid(), Start: "bogus-start-fingerprint"},
}
if n := finalizeExitedSessions(ctx, []*session.State{stale}); n != 0 {
t.Fatalf("finalizeExitedSessions = %d, want 0 (revalidation should skip the revived session)", n)
}
got, err := store.Load(ctx, "revived")
if err != nil {
t.Fatal(err)
}
if got.EndedAt != nil {
t.Error("revived session was ended despite a live owner on disk")
}
}
}