Merge branch 'main' into feat/entire-trail-checkout-worktree · Entire
Log in
Merge branch 'main' into feat/entire-trail-checkout-worktree
cc8ab67→main·
pfleidi·4d ago·58 files·+5,742 added/-507 removed
Changes
58
M.goreleaser.yaml+8/-1
api/checkpoint
Mmetadata.go+30
cmd/entire/cli
agent
Magent.go+14
Mcapabilities.go+12
claudecode
Mdiscovery.go+24/-260
Mreviewer.go+51/-4
Mreviewer_test.go+100/-3
testdata
Astream_with_deltas.jsonl+7
codex
MAGENT.md+9/-1
Mdiscovery.go+43/-6
Mdiscovery_test.go+116/-7
Areview_tokens.go+195
Areview_tokens_test.go+423
Mreviewer.go+119/-27
Mreviewer_test.go+106/-3
cursor
Mhooks.go+43/-36
Mhooks_test.go+90
Aimages.go+295
Aimages_test.go+263
skilldiscovery
Mregistry.go+19/-6
Mregistry_test.go+17/-2
Ascan.go+257
checkpoint
Maliases.go+1
Mpersistent.go+173/-30
Apersistent_assets_test.go+356
Mpersistent_signing_test.go+1/-1
integration_test
Acodex_image_externalize_test.go+146
Acursor_image_externalize_test.go+321
Aimage_externalize_test.go+194
Mtestenv.go+3
paths
Mpaths.go+7
Mplugin.go+15/-1
Mplugin_test.go+60
review
Mcmd.go+17/-3
Mcmd_test.go+128/-5
settings
Msettings.go+27
Asettings_images_test.go+63
Msetup.go+4/-1
Msetup_github.go+122/-43
Msetup_github_test.go+222/-34
strategy
Acondense_images_test.go+165
Mhooks.go+24/-2
Mhooks_test.go+53
Mmanual_commit_condensation.go+95/-1
Mmanual_commit_hooks.go+48/-7
Mmanual_commit_opf_rewrite.go+19
Mmanual_commit_opf_rewrite_test.go+87
transcript/imageextract
Acodex_test.go+229
Aimageextract.go+375
Aimageextract_test.go+401
Mgo.mod+5/-5
Mgo.sum+10/-10
internal/remotehelper/githelper
Minvariants_test.go+61/-1
Mlist.go+2/-2
Mlist_test.go+1/-1
Mpush.go+6/-2
Arefadv_cache.go+55
Mrun.go+5/-2
99 unmodified lines
100
101
102
103
103
104
105
106
107
108
109
110
111
112
113
99 unmodified lines
prerelease: auto
scoops:
- repository:
# Name the manifest (and therefore the Scoop app directory) "entire". Without
# this, goreleaser defaults the manifest name to the project name, which
# resolves to the repo name ("cli"), so `scoop install` lands the binary in
# …\scoop\apps\cli\current\entire.exe. That mismatched app-dir name is
# surprising ("scoop install cli"?) and fed the Windows hook-path bug in
# https://github.com/entireio/cli/issues/1424.
- name: entire
repository:
owner: entireio
name: scoop-bucket
token: "{{ .Env.TAP_GITHUB_TOKEN }}"
M.goreleaser.yaml+8/-1
10 unmodified lines
11
12
13
14
15
16
17
18
19
20
21
22
23
24
16 unmodified lines
41
42
43
44
45
46
47
48
49
50
51
143 unmodified lines
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
234 unmodified lines
449
450
451
452
453
454
455
456
457
10 unmodified lines
"github.com/go-git/go-git/v6/plumbing"
)
// TranscriptAsset is a binary blob (e.g. an image) lifted out of a transcript
// and stored raw in the checkpoint, referenced by a placeholder in the log.
type TranscriptAsset struct {
Name string // stable asset filename / id, also used in the placeholder
MediaType string
Data []byte
}
// WriteOptions contains options for writing a persistent checkpoint.
type WriteOptions struct {
// CheckpointID is the stable 12-hex-char identifier
16 unmodified lines
// Must be pre-redacted (via redact.JSONLBytes or redact.AlreadyRedacted for trusted sources).
Transcript redact.RedactedBytes
// Assets are binary blobs (e.g. images) lifted out of Transcript and
// referenced by path-bearing placeholders. Stored raw under the session's
// assets/ folder. Empty for agents/transcripts with no externalized images.
Assets []TranscriptAsset
// Prompts contains the raw user prompts from the session. Run through
// redactedJoinedPrompts before persisting — the writer does this
// inside writeSessionToSubdirectory.
143 unmodified lines
// Must be pre-redacted (via redact.JSONLBytes or redact.AlreadyRedacted for trusted sources).
Transcript redact.RedactedBytes
// Assets are the externalized image blobs matching Transcript's placeholders
// (see WriteOptions.Assets). Set together with Transcript so the backfill keeps
// the stored assets/ folder consistent with the transcript; empty clears any
// previously-stored assets when Transcript is replaced.
Assets []TranscriptAsset
// PreserveAssetsWhenEmpty keeps already-stored assets instead of clearing them
// when Assets is empty. Set on the finalize path for agents whose assets come
// from a best-effort sidecar capture (e.g. Cursor's sqlite3 store read): a
// transient capture miss at finalize must not wipe images a prior condensation
// successfully stored. Left false for codec agents, where an empty set means
// "the transcript has no images" and stale asset blobs should be cleared.
PreserveAssetsWhenEmpty bool
// Prompts contains the raw user prompts (replaces existing).
// See WriteOptions.Prompts.
Prompts []string
234 unmodified lines
CompactTranscript string `json:"compact_transcript,omitempty"`
ContentHash string `json:"content_hash,omitempty"`
Prompt string `json:"prompt"`
// AssetsManifest points at assets/manifest.json when images were externalized
// out of the transcript into the session's assets/ folder. Omitted otherwise.
AssetsManifest string `json:"assets_manifest,omitempty"`
}
// CheckpointSummary is the root-level metadata.json for a checkpoint.
Mapi/checkpoint/metadata.go+30
187 unmodified lines
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
187 unmodified lines
PrepareTranscript(ctx context.Context, sessionRef string) error
}
// SidecarImageProvider is implemented by agents that keep images OUTSIDE the
// transcript Entire condenses — e.g. Cursor stores pasted images in a per-session
// SQLite blob store, not the JSONL transcript. The strategy layer calls this
// during condensation/finalize to capture those images as checkpoint assets so
// they're preserved with the session. Best-effort: returns nil (no error) when
// the sidecar store is unavailable or unreadable.
type SidecarImageProvider interface {
Agent
// SidecarImages returns images stored outside the transcript for the session
// identified by sessionRef (the transcript path).
SidecarImages(ctx context.Context, sessionRef string) ([]CompactedTranscriptAsset, error)
}
// TokenCalculator provides token usage calculation for a session.
// The framework calls this during step save and checkpoint if implemented.
type TokenCalculator interface {
Mcmd/entire/cli/agent/agent.go+14
75 unmodified lines
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
75 unmodified lines
return declaredCapability[TranscriptPreparer](ag, func(c DeclaredCaps) bool { return c.TranscriptPreparer })
}
// AsSidecarImageProvider returns the agent as SidecarImageProvider if it
// implements the interface. This is a best-effort, optional capability (image
// capture from a store outside the transcript, e.g. Cursor's SQLite blob store),
// so it resolves by type assertion alone with no DeclaredCaps gate.
func AsSidecarImageProvider(ag Agent) (SidecarImageProvider, bool) {
if ag == nil {
return nil, false
}
p, ok := ag.(SidecarImageProvider)
return p, ok
}
// AsTokenCalculator returns the agent as TokenCalculator if it both
// implements the interface and (for CapabilityDeclarer agents) has declared the capability.
func AsTokenCalculator(ag Agent) (TokenCalculator, bool) {
Mcmd/entire/cli/agent/capabilities.go+12
1 unmodified line
2
3
4
5
5
6
7
9
10
11
12
8
9
10
5 unmodified lines
16
17
18
24
25
26
27
19
20
21
22
23
24
29
30
31
32
33
25
26
27
28
29
30
31
32
33
3 unmodified lines
37
38
39
40
41
44
45
46
47
48
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
1 unmodified line
import (
"context"
"errors"
"log/slog"
"os"
"path/filepath"
"sort"
"strings"
"golang.org/x/mod/semver"
"github.com/entireio/cli/cmd/entire/cli/agent"
"github.com/entireio/cli/cmd/entire/cli/agent/skilldiscovery"
5 unmodified lines
// (nil, nil) when HOME is unreadable or directories are missing — discovery
// is best-effort.
//
// Claude Code exposes three kinds of invocable content per plugin:
// - skills: <plugin>/skills/<name>/SKILL.md (YAML frontmatter with name + description)
// - commands: <plugin>/commands/<name>.md (YAML frontmatter with description; name = filename)
// - agents: <plugin>/agents/<name>.md (YAML frontmatter with description; name = filename)
// Claude Code exposes three kinds of invocable content per plugin, all invoked
// via the same slash-prefix syntax (`/name`, `/plugin:name`):
// - skills: <plugin>/skills/<name>/SKILL.md (frontmatter: name + description)
// - commands: <plugin>/commands/<name>.md (frontmatter: description; name = filename)
// - agents: <plugin>/agents/<name>.md (frontmatter: description; name = filename)
//
// All three are walked because users invoke them via the same slash-prefix
// syntax (`/plugin:name`) and any of them can be a review tool. The
// pr-review-toolkit plugin, for example, ships its review skills as
// commands/agents (not skills/), and was silently missed by a skills-only
// walker.
// All three are walked because any can be a review tool — the pr-review-toolkit
// plugin, for example, ships its review skills as commands/agents (not skills/).
//
// The generic SKILL.md / markdown scanning, version dedupe, and frontmatter
// parsing live in the shared skilldiscovery package; this method supplies the
// Claude-specific roots and slash invocation form.
//
//nolint:unparam // error return is part of SkillDiscoverer contract; future implementations may report hard failures
func (c *ClaudeCodeAgent) DiscoverReviewSkills(ctx context.Context) ([]agent.DiscoveredSkill, error) {
3 unmodified lines
return nil, nil
}
form := skilldiscovery.SlashForm
var found []agent.DiscoveredSkill
found = append(found, scanPluginCache(ctx, filepath.Join(home, ".claude", "plugins", "cache"))...)
found = append(found, scanUserSkills(ctx, filepath.Join(home, ".claude", "skills"))...)
found = append(found, scanFlatMarkdownDir(ctx, filepath.Join(home, ".claude", "commands"), "")...)
found = append(found, scanFlatMarkdownDir(ctx, filepath.Join(home, ".claude", "agents"), "")...)
found = dedupeByInvocation(found)
found = append(found, skilldiscovery.ScanPluginCache(ctx, filepath.Join(home, ".claude", "plugins", "cache"),
func(versionRoot, pluginName string) []agent.DiscoveredSkill {
var out []agent.DiscoveredSkill
out = append(out, skilldiscovery.ScanSkillsDir(ctx, filepath.Join(versionRoot, "skills"), pluginName, form)...)
out = append(out, skilldiscovery.ScanFlatMarkdownDir(ctx, filepath.Join(versionRoot, "commands"), pluginName, form)...)
out = append(out, skilldiscovery.ScanFlatMarkdownDir(ctx, filepath.Join(versionRoot, "agents"), pluginName, form)...)
return out
})...)
found = append(found, skilldiscovery.ScanSkillsDir(ctx, filepath.Join(home, ".claude", "skills"), "", form)...)
found = append(found, skilldiscovery.ScanFlatMarkdownDir(ctx, filepath.Join(home, ".claude", "commands"), "", form)...)
found = append(found, skilldiscovery.ScanFlatMarkdownDir(ctx, filepath.Join(home, ".claude", "agents"), "", form)...)
found = skilldiscovery.DedupeByInvocation(found)
if len(found) == 0 {
return nil, nil
}
return found, nil
}
// dedupeByInvocation collapses entries sharing an invocation name. Plugins
// can ship a skill and a same-named command wrapper that forwards to it;
// scan order keeps the skill over its wrapper.
func dedupeByInvocation(in []agent.DiscoveredSkill) []agent.DiscoveredSkill {
if len(in) < 2 {
return in
}
seen := make(map[string]struct{}, len(in))
out := make([]agent.DiscoveredSkill, 0, len(in))
for _, s := range in {
if _, dup := seen[s.Name]; dup {
continue
}
seen[s.Name] = struct{}{}
out = append(out, s)
}
return out
}
// scanPluginCache walks <root>/<marketplace>/<plugin>/<version>/{skills,commands,agents}/
// One plugin can contribute through any or all three directories.
//
// Multiple version directories per plugin are common after upgrades. Walking
// every version produces duplicate skills (same invocation name, same
// description) — confusing in the picker and wasteful in the prompt. We pick
// a single version per plugin via pickLatestVersion: prefer valid semver
// (highest), fall back to lexicographic max.
func scanPluginCache(ctx context.Context, root string) []agent.DiscoveredSkill {
entries, err := os.ReadDir(root)
if err != nil {
logging.Debug(ctx, "claude-code discovery: plugin cache unreadable",
slog.String("root", root), slog.String("error", err.Error()))
return nil
}
var found []agent.DiscoveredSkill
for _, marketEntry := range entries {
if !marketEntry.IsDir() {
continue
}
marketRoot := filepath.Join(root, marketEntry.Name())
pluginEntries, err := os.ReadDir(marketRoot)
if err != nil {
continue
}
for _, pluginEntry := range pluginEntries {
if !pluginEntry.IsDir() {
continue
}
pluginName := pluginEntry.Name()
pluginRoot := filepath.Join(marketRoot, pluginName)
versionEntries, err := os.ReadDir(pluginRoot)
if err != nil {
continue
}
versionDir, ok := pickLatestVersion(versionEntries)
if !ok {
continue
}
versionRoot := filepath.Join(pluginRoot, versionDir)
found = append(found, readSkillsDir(ctx, filepath.Join(versionRoot, "skills"), pluginName)...)
found = append(found, scanFlatMarkdownDir(ctx, filepath.Join(versionRoot, "commands"), pluginName)...)
found = append(found, scanFlatMarkdownDir(ctx, filepath.Join(versionRoot, "agents"), pluginName)...)
}
}
return found
}
// pickLatestVersion returns the name of the "newest" version directory among
// entries. Strategy:
//
// - If any entry name parses as semver (with or without a leading "v"), pick
// the highest semver among those that parse. Non-semver entries are
// ignored when at least one semver entry exists.
// - Otherwise, fall back to the lexicographic max of all directory names.
// This handles the "unknown" sentinel some plugins ship and one-off names.
//
// Returns ("", false) if no usable directory entry exists.
func pickLatestVersion(entries []os.DirEntry) (string, bool) {
var dirs []string
for _, e := range entries {
if e.IsDir() {
dirs = append(dirs, e.Name())
}
}
if len(dirs) == 0 {
return "", false
}
var semverDirs []string
for _, d := range dirs {
if semver.IsValid(semverWithV(d)) {
semverDirs = append(semverDirs, d)
}
}
if len(semverDirs) > 0 {
sort.Slice(semverDirs, func(i, j int) bool {
return semver.Compare(semverWithV(semverDirs[i]), semverWithV(semverDirs[j])) > 0
})
return semverDirs[0], true
}
sort.Sort(sort.Reverse(sort.StringSlice(dirs)))
return dirs[0], true
}
// semverWithV ensures a version string has the "v" prefix that
// golang.org/x/mod/semver requires. Plugin version dirs are usually bare
// (e.g. "0.1.0"), but we tolerate either form.
func semverWithV(s string) string {
if strings.HasPrefix(s, "v") {
return s
}
return "v" + s
}
// scanUserSkills walks ~/.claude/skills/<skill>/SKILL.md.
func scanUserSkills(ctx context.Context, root string) []agent.DiscoveredSkill {
return readSkillsDir(ctx, root, "" /* no plugin prefix */)
}
// readSkillsDir reads each skill subdirectory's SKILL.md, parses frontmatter,
// and emits a DiscoveredSkill if Matches() returns true.
func readSkillsDir(ctx context.Context, dir, pluginName string) []agent.DiscoveredSkill {
entries, err := os.ReadDir(dir)
if err != nil {
return nil
}
var found []agent.DiscoveredSkill
for _, skillEntry := range entries {
if !skillEntry.IsDir() {
continue
}
skillDir := filepath.Join(dir, skillEntry.Name())
skillFile := filepath.Join(skillDir, "SKILL.md")
data, err := os.ReadFile(skillFile) //nolint:gosec // G304: skillFile is constructed from a ReadDir walk under HOME, not user input
if err != nil {
continue
}
name, description, parseErr := parseSkillFrontmatter(data)
if parseErr != nil {
logging.Debug(ctx, "claude-code discovery: skipping malformed SKILL.md",
slog.String("path", skillFile), slog.String("error", parseErr.Error()))
continue
}
if name == "" {
name = skillEntry.Name()
}
invocation := invocationName(name, pluginName)
if !skilldiscovery.Matches(invocation, description) {
continue
}
found = append(found, agent.DiscoveredSkill{
Name: invocation,
Description: description,
SourcePath: skillFile,
})
}
return found
}
// scanFlatMarkdownDir reads *.md files directly under dir (no nesting), parses
// their YAML frontmatter for `description:`, and derives the invocation name
// from the filename (stripping the .md suffix). Used for both plugin
// commands/agents and user-level ~/.claude/commands and ~/.claude/agents.
//
// Frontmatter shape differs from SKILL.md — no `name:` field, so the
// filename is the source of truth for the invocation name.
func scanFlatMarkdownDir(ctx context.Context, dir, pluginName string) []agent.DiscoveredSkill {
entries, err := os.ReadDir(dir)
if err != nil {
return nil
}
var found []agent.DiscoveredSkill
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".md") {
continue
}
baseName := strings.TrimSuffix(entry.Name(), ".md")
if strings.EqualFold(baseName, "README") {
continue
}
filePath := filepath.Join(dir, entry.Name())
data, err := os.ReadFile(filePath) //nolint:gosec // G304: filePath is constructed from a ReadDir walk under HOME, not user input
if err != nil {
continue
}
_, description, parseErr := parseSkillFrontmatter(data)
if parseErr != nil {
logging.Debug(ctx, "claude-code discovery: skipping malformed command/agent",
slog.String("path", filePath), slog.String("error", parseErr.Error()))
continue
}
invocation := invocationName(baseName, pluginName)
if !skilldiscovery.Matches(invocation, description) {
continue
}
found = append(found, agent.DiscoveredSkill{
Name: invocation,
Description: description,
SourcePath: filePath,
})
}
return found
}
// invocationName builds the slash-prefixed invocation form. Plugin-prefixed
// names use "/plugin:name"; bare names use "/name".
func invocationName(name, pluginName string) string {
if pluginName == "" {
return "/" + name
}
return "/" + pluginName + ":" + name
}
// parseSkillFrontmatter extracts `name:` and `description:` from a minimal
// YAML frontmatter block. Purpose-built for the tiny subset of YAML these
// SKILL.md / command / agent files actually use.
//
// Trims surrounding double-quotes from values so `description: "foo bar"`
// is returned as `foo bar` — the command/agent frontmatter quotes values;
// SKILL.md files usually don't.
func parseSkillFrontmatter(data []byte) (name, description string, err error) {
s := string(data)
if !strings.HasPrefix(s, "---\n") && !strings.HasPrefix(s, "---\r\n") {
return "", "", errors.New("no frontmatter delimiter")
}
body := strings.TrimPrefix(strings.TrimPrefix(s, "---\r\n"), "---\n")
end := strings.Index(body, "\n---")
if end < 0 {
return "", "", errors.New("no closing frontmatter delimiter")
}
for _, line := range strings.Split(body[:end], "\n") {
line = strings.TrimSpace(line)
switch {
case strings.HasPrefix(line, "name:"):
name = strings.Trim(strings.TrimSpace(strings.TrimPrefix(line, "name:")), `"`)
case strings.HasPrefix(line, "description:"):
description = strings.Trim(strings.TrimSpace(strings.TrimPrefix(line, "description:")), `"`)
}
}
return name, description, nil
}
Mcmd/entire/cli/agent/claudecode/discovery.go+24/-260
54 unmodified lines
55
56
57
58
59
60
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
22 unmodified lines
99
100
101
102
103
104
105
106
20 unmodified lines
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
6 unmodified lines
155
156
157
158
159
160
161
162
128
163
164
165
166
167
168
14 unmodified lines
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
54 unmodified lines
// Emits Started first, Finished{Success:...} last (success follows result.is_error).
// On a scanner error (torn stream), emits RunError then Finished{Success:false}.
//
// Tokens are emitted only at the terminal `result` envelope, not
// incrementally — claude's per-assistant `usage` fields aren't cumulative
// and summing them across messages would double-count.
// Live-token semantics: Claude's assistant envelopes carry a usage snapshot
// taken at the START of each API call — input_tokens/cache_* are populated
// but output_tokens is essentially zero (a 1–8 token "initial decision"
// count that does not update as text streams). The true output is only
// surfaced on `result` (aggregate across all calls in the run) or on the
// late `message_delta` event of --include-partial-messages mode.
//
// The Tokens contract (types/reviewer.go) is cumulative running totals, so
// the parser accumulates the input sum across unique message ids (the same
// usage block repeats verbatim on every content-block envelope of one API
// call — summing per envelope would multi-count) and emits
// `Tokens{In: <running sum>, Out: 0}` once per new message id. The running
// sum converges to the `result` aggregate, which is emitted last with the
// true {In, Out}. Out stays 0 mid-run because consumers render every Tokens
// event the same way — surfacing the 1–8 token stub would display a
// misleading real-looking output count.
//
// Package-private; called directly from this package's tests so they can
// drive raw stdout fixtures through the parser without going through the
22 unmodified lines
var sawResult bool
var resultErr bool
var resultUsage messageUsage
seenMsgIDs := map[string]struct{}{}
var cumInputTokens int
for scanner.Scan() {
line := scanner.Bytes()
if len(line) == 0 {
20 unmodified lines
out <- reviewtypes.ToolCall{Name: block.Name, Args: string(block.Input)}
}
}
// Accumulate input once per unique message id: every
// content-block envelope of one API call repeats the same
// usage snapshot, and its output_tokens is a 1–8 token stub
// (see the parser doc). Emitting the running sum keeps
// mid-run values on the cumulative Tokens contract; the
// true {In, Out} tally comes from `result` below.
in := env.Message.Usage.InputTokens +
env.Message.Usage.CacheReadInputTokens +
env.Message.Usage.CacheCreationInputTokens
if in > 0 && env.Message.ID != "" {
if _, seen := seenMsgIDs[env.Message.ID]; !seen {
seenMsgIDs[env.Message.ID] = struct{}{}
cumInputTokens += in
out <- reviewtypes.Tokens{In: cumInputTokens, Out: 0}
}
}
case "result":
sawResult = true
resultErr = env.IsError
6 unmodified lines
return
}
if sawResult {
// Gate on non-zero usage: a result envelope without a usage
// block would emit Tokens{0,0}, which only ever ERASES the
// mid-run cumulative total under the consumers'
// overwrite-not-sum semantics (mirrors the codex guard).
in := resultUsage.InputTokens + resultUsage.CacheReadInputTokens + resultUsage.CacheCreationInputTokens
out <- reviewtypes.Tokens{In: in, Out: resultUsage.OutputTokens}
if in > 0 || resultUsage.OutputTokens > 0 {
out <- reviewtypes.Tokens{In: in, Out: resultUsage.OutputTokens}
}
out <- reviewtypes.Finished{Success: !resultErr}
return
}
14 unmodified lines
}
type claudeMessage struct {
// ID is the API message id — identical across the multiple
// content-block envelopes of one API call; the parser dedupes usage
// accumulation on it.
ID string `json:"id"`
Content []claudeBlock `json:"content"`
// Usage on assistant envelopes is the per-call-START snapshot — input
// counts are populated but output_tokens reflects only the model's
// initial decision, not the streamed text. Final aggregate usage
// arrives on the `result` envelope. Reuses messageUsage (declared in
// types.go) to stay aligned with the transcript-parser usage shape.
Usage messageUsage `json:"usage"`
}
type claudeBlock struct {
Mcmd/entire/cli/agent/claudecode/reviewer.go+51/-4
238 unmodified lines
239
240
241
242
243
244
245
246
247
248
249
2 unmodified lines
252
253
254
250
251
255
256
257
258
254
259
260
261
262
127 unmodified lines
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
238 unmodified lines
t.Error("expected AssistantText carrying fixture prose 'Cats are…'")
}
// The parser emits Tokens on every assistant envelope that carries
// non-zero usage plus a terminal Tokens on the result envelope. The
// fixture's two assistant envelopes both carry usage, so expect >=2
// here (the exact count is fixture-defined and not asserted to keep
// the fixture editable).
var tokensSeen int
var tokensOut int
for _, ev := range events {
2 unmodified lines
tokensOut = tk.Out
}
}
if tokensSeen != 1 {
t.Errorf("Tokens count = %d, want 1", tokensSeen)
if tokensSeen < 2 {
t.Errorf("Tokens count = %d, want >=2 (per-assistant snapshots + result)", tokensSeen)
}
if tokensOut == 0 {
t.Error("Tokens.Out = 0, want > 0")
t.Error("final Tokens.Out = 0, want > 0")
}
}
127 unmodified lines
}
}
// TestParseClaudeOutput_EmitsCumulativeInputDuringRun captures the live-token
// contract for Claude. The `Tokens` type is documented as cumulative running
// totals (each emission replaces the previous), so mid-run emissions must be
// running sums, not per-call snapshots. Claude's assistant envelopes carry a
// usage block per API call (repeated verbatim on every content-block envelope
// of the same message id), where output_tokens is a 1–8 token "initial
// decision" stub — so the parser accumulates input across unique message ids,
// emits `Tokens{In: <running sum>, Out: 0}`, and lets the terminal `result`
// envelope deliver the true {In, Out} aggregate.
//
// Fixture is derived from real `claude -p --output-format stream-json
// --verbose` output captured against claude-haiku-4-5: six assistant
// envelopes across three API calls (message ids msg_turn1..3, with turn 1
// repeated on three envelopes), then a final result. The per-call input sums
// are 56277, 56626, and 56734 — running totals 56277, 112903, 169637 — and
// the result aggregate is exactly {In: 169637, Out: 2511}, which pins that
// accumulation converges to the final figure.
func TestParseClaudeOutput_EmitsCumulativeInputDuringRun(t *testing.T) {
t.Parallel()
f, err := os.Open("testdata/stream_with_deltas.jsonl")
if err != nil {
t.Fatal(err)
}
defer f.Close()
var events []reviewtypes.Event
for ev := range parseClaudeOutput(f) {
events = append(events, ev)
}
var tokens []reviewtypes.Tokens
sawFinished := false
for _, e := range events {
switch ev := e.(type) {
case reviewtypes.Tokens:
if sawFinished {
t.Errorf("Tokens event arrived AFTER Finished — wrong ordering")
}
tokens = append(tokens, ev)
case reviewtypes.Finished:
sawFinished = true
}
}
// One emission per unique message id (duplicate envelopes of the same
// API call must not re-emit) plus the terminal result emission.
want := []reviewtypes.Tokens{
{In: 56277, Out: 0},
{In: 112903, Out: 0},
{In: 169637, Out: 0},
{In: 169637, Out: 2511},
}
if len(tokens) != len(want) {
t.Fatalf("Tokens events = %d, want %d (one per unique message id + result): %+v", len(tokens), len(want), tokens)
}
for i, w := range want {
if tokens[i] != w {
t.Errorf("tokens[%d] = %+v, want %+v", i, tokens[i], w)
}
}
}
// TestParseClaudeOutput_UsagelessResultDoesNotClobberCumulative pins the
// terminal emission guard: a result envelope with no/zero usage must not
// emit Tokens{0,0} — under the consumers' overwrite-not-sum semantics that
// would erase the mid-run cumulative input total.
func TestParseClaudeOutput_UsagelessResultDoesNotClobberCumulative(t *testing.T) {
t.Parallel()
input := strings.Join([]string{
`{"type":"assistant","message":{"id":"msg_1","content":[{"type":"text","text":"hi"}],"usage":{"input_tokens":10,"cache_read_input_tokens":90,"cache_creation_input_tokens":0,"output_tokens":2}}}`,
`{"type":"result","subtype":"success","is_error":false}`,
"",
}, "\n")
var tokens []reviewtypes.Tokens
for ev := range parseClaudeOutput(strings.NewReader(input)) {
if tk, ok := ev.(reviewtypes.Tokens); ok {
tokens = append(tokens, tk)
}
}
if len(tokens) == 0 {
t.Fatal("expected the mid-run cumulative Tokens emission")
}
last := tokens[len(tokens)-1]
if last.In == 0 && last.Out == 0 {
t.Fatalf("final tokens = %+v — usage-less result clobbered the cumulative total", last)
}
if last.In != 100 {
t.Errorf("final tokens = %+v, want the cumulative {100, 0} to stand", last)
}
}
// collectEvents drains an event channel into a slice.
func collectEvents(ch <-chan reviewtypes.Event) []reviewtypes.Event {
var events []reviewtypes.Event
Mcmd/entire/cli/agent/claudecode/reviewer_test.go+100/-3
1
2
3
4
5
6
7
{"type":"system","subtype":"init","cwd":"/redacted/worktree","session_id":"a905e63f-aaaa-aaaa-aaaa-aaaaaaaaaaaa","model":"claude-haiku-4-5","permissionMode":"plan","output_style":"default","apiKeySource":"none","uuid":"redacted-uuid-1"}
{"type":"assistant","message":{"model":"claude-haiku-4-5-20251001","id":"msg_turn1","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"Analyzing the request..."}],"stop_reason":null,"usage":{"input_tokens":10,"cache_creation_input_tokens":56267,"cache_read_input_tokens":0,"output_tokens":6,"service_tier":"standard"}},"session_id":"a905e63f-aaaa-aaaa-aaaa-aaaaaaaaaaaa","uuid":"redacted-uuid-2"}
{"type":"assistant","message":{"model":"claude-haiku-4-5-20251001","id":"msg_turn1","type":"message","role":"assistant","content":[{"type":"text","text":"I'll outline a plan first."}],"stop_reason":null,"usage":{"input_tokens":10,"cache_creation_input_tokens":56267,"cache_read_input_tokens":0,"output_tokens":6,"service_tier":"standard"}},"session_id":"a905e63f-aaaa-aaaa-aaaa-aaaaaaaaaaaa","uuid":"redacted-uuid-3"}
{"type":"assistant","message":{"model":"claude-haiku-4-5-20251001","id":"msg_turn1","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01","name":"Write","input":{"file_path":"plan.md","content":"plan body"}}],"stop_reason":null,"usage":{"input_tokens":10,"cache_creation_input_tokens":56267,"cache_read_input_tokens":0,"output_tokens":6,"service_tier":"standard"}},"session_id":"a905e63f-aaaa-aaaa-aaaa-aaaaaaaaaaaa","uuid":"redacted-uuid-4"}
{"type":"assistant","message":{"model":"claude-haiku-4-5-20251001","id":"msg_turn2","type":"message","role":"assistant","content":[{"type":"text","text":"Plan created, ready to proceed."}],"stop_reason":null,"usage":{"input_tokens":5,"cache_creation_input_tokens":10066,"cache_read_input_tokens":46555,"output_tokens":1,"service_tier":"standard"}},"session_id":"a905e63f-aaaa-aaaa-aaaa-aaaaaaaaaaaa","uuid":"redacted-uuid-5"}
{"type":"assistant","message":{"model":"claude-haiku-4-5-20251001","id":"msg_turn3","type":"message","role":"assistant","content":[{"type":"text","text":"Found 3 issues."}],"stop_reason":null,"usage":{"input_tokens":6,"cache_creation_input_tokens":107,"cache_read_input_tokens":56621,"output_tokens":2,"service_tier":"standard"}},"session_id":"a905e63f-aaaa-aaaa-aaaa-aaaaaaaaaaaa","uuid":"redacted-uuid-6"}
{"type":"result","subtype":"success","is_error":false,"duration_ms":29272,"num_turns":3,"result":"Found 3 issues.","stop_reason":"end_turn","session_id":"a905e63f-aaaa-aaaa-aaaa-aaaaaaaaaaaa","total_cost_usd":0.105,"usage":{"input_tokens":21,"cache_creation_input_tokens":66440,"cache_read_input_tokens":103176,"output_tokens":2511,"service_tier":"standard"},"uuid":"redacted-uuid-7"}
Acmd/entire/cli/agent/claudecode/testdata/stream_with_deltas.jsonl+7
192 unmodified lines
193
194
195
196
196
197
198
199
4 unmodified lines
204
205
206
207
208
209
210
211
212
213
214
192 unmodified lines
## Gaps & Limitations
- **Hooks require feature flag:** The `hooks` feature is `default_enabled: false` (stage: UnderDevelopment). It must be enabled via `--enable hooks` CLI flag, or `features.hooks = true` in `config.toml`, or `-c features.hooks=true`. Without this, hooks.json is ignored entirely.
- **Hooks require feature flag:** The `codex_hooks` feature is `default_enabled: false` (stage: UnderDevelopment). It must be enabled via `--enable codex_hooks` CLI flag, or `features.codex_hooks = true` in `config.toml`, or `-c features.codex_hooks=true`. Without this, hooks.json is ignored entirely.
- **No SessionEnd hook:** Codex does not fire a hook when a session is completely terminated. The `Stop` hook fires at end-of-turn, not end-of-session. This is similar to some other agents — the framework handles this gracefully.
- **PreToolUse is shell-only:** Currently only fires for `Bash` tool (direct shell execution). MCP tools, stdin streaming, and other tool types are not yet hooked. PostToolUse is in review.
- **Transcript may be null:** In `--ephemeral` mode, `transcript_path` is null. The integration should handle this gracefully.
4 unmodified lines
- JSON schemas at `codex-rs/hooks/schema/generated/` in the Codex repository
- Hook config structure at `codex-rs/hooks/src/engine/config.rs` in the Codex repository
## Review integration (`entire review`)
Codex review runs via `codex exec --skip-git-repo-check --json [-m <model>] [-c model_reasoning_effort=<level>] -` (prompt on stdin). **`codex exec` fires no lifecycle hooks**, which shapes the whole integration (see CLAUDE.md → `entire review` → "Codex specifics"):
- **Skills are passed verbatim, not paraphrased.** Codex injects its installed-skill catalog into every exec session and loads the matching `SKILL.md`; configured skills use codex's `$name` / `$plugin:name` form (`DiscoverReviewSkills` in `discovery.go`). Native `codex exec review` is not used — it rejects a prompt under a scope flag and can't carry Entire's scope/per-run/checkpoint context.
- **Live tokens come from the rollout file, not stdout.** `codex exec --json` carries `usage` only on the terminal `turn.completed`, and a review is a single turn. `review_tokens.go` resolves the rollout transcript by `thread_id` (from the `thread.started` envelope), tails it (the same `~/.codex/.../rollout-*-<thread-id>.jsonl` documented under Transcript above), and emits cumulative `Tokens` per `token_count` event — the source codex's interactive UI reads.
- **No tagged review session.** Because no hook fires, codex's session is never tagged `KindAgentReview`. The fix manifest therefore sources codex from its **live run output** (`run.Buffer`), and `entire review fix` skill verification is advisory for codex (loose description match), not a hard block.
Mcmd/entire/cli/agent/codex/AGENT.md+9/-1
1 unmodified line
2
3
4
5
6
7
8
9
10
11
12
9
10
11
12
13
14
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
1 unmodified line
import (
"context"
"log/slog"
"path/filepath"
"github.com/entireio/cli/cmd/entire/cli/agent"
"github.com/entireio/cli/cmd/entire/cli/agent/skilldiscovery"
"github.com/entireio/cli/cmd/entire/cli/logging"
)
// DiscoverReviewSkills is a stub until the Codex on-disk plugin layout is
// verified against codex-rs source (see codex-rs/tui/src/slash_command.rs).
// Returns (nil, nil) so the picker treats Codex as "built-ins + install
// hint only" for Phase 1.
func (c *CodexAgent) DiscoverReviewSkills(_ context.Context) ([]agent.DiscoveredSkill, error) {
return nil, nil
// DiscoverReviewSkills walks codex's on-disk skill layout looking for
// review-adjacent skills. Returns (nil, nil) when HOME is unreadable or the
// directories are missing — discovery is best-effort.
//
// Codex exposes skills as <root>/<name>/SKILL.md (same frontmatter shape as
// Claude). Three roots contribute, mirroring codex's own injected skills
// catalog:
// - ~/.codex/skills/<name>/ → user skills ($name)
// - ~/.codex/plugins/cache/<m>/<p>/<v>/skills/<name>/ → plugin skills ($p:name)
// - ~/.codex/superpowers/skills/<name>/ → superpowers ($superpowers:name)
//
// Skills are emitted in codex's dollar invocation form ($name / $plugin:name) —
// the literal token a user types to invoke the skill in the codex CLI — so the
// review prompt names skills exactly the way codex's skill system expects,
// loading the real SKILL.md rather than relying on a loose description match.
//
//nolint:unparam // error return is part of SkillDiscoverer contract; future implementations may report hard failures
func (c *CodexAgent) DiscoverReviewSkills(ctx context.Context) ([]agent.DiscoveredSkill, error) {
// resolveCodexHome is the agent's canonical config-tree resolution
// (honors CODEX_HOME) — discovery must see the same skills codex runs.
codexHome, err := resolveCodexHome()
if err != nil {
logging.Debug(ctx, "codex discovery: resolve codex home failed", slog.String("error", err.Error()))
return nil, nil
}
form := skilldiscovery.DollarForm
var found []agent.DiscoveredSkill
found = append(found, skilldiscovery.ScanSkillsDir(ctx, filepath.Join(codexHome, "skills"), "", form)...)
found = append(found, skilldiscovery.ScanPluginCache(ctx, filepath.Join(codexHome, "plugins", "cache"),
func(versionRoot, pluginName string) []agent.DiscoveredSkill {
return skilldiscovery.ScanSkillsDir(ctx, filepath.Join(versionRoot, "skills"), pluginName, form)
})...)
found = append(found, skilldiscovery.ScanSkillsDir(ctx, filepath.Join(codexHome, "superpowers", "skills"), "superpowers", form)...)
found = skilldiscovery.DedupeByInvocation(found)
if len(found) == 0 {
return nil, nil
}
return found, nil
}
Mcmd/entire/cli/agent/codex/discovery.go+43/-6
1 unmodified line
2
3
4
5
6
7
8
9
3 unmodified lines
13
14
15
14
15
16
17
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
19
44
45
21
22
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
1 unmodified line
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/entireio/cli/cmd/entire/cli/agent"
3 unmodified lines
// Compile-time pin: CodexAgent must satisfy SkillDiscoverer.
var _ agent.SkillDiscoverer = (*codex.CodexAgent)(nil)
func TestCodexAgent_DiscoverReviewSkills_Stub(t *testing.T) {
t.Parallel()
a := &codex.CodexAgent{}
skills, err := a.DiscoverReviewSkills(context.Background())
// withFakeHome points HOME at a temp dir so discovery walks an empty,
// controlled ~/.codex tree. Uses t.Setenv, so callers must NOT t.Parallel.
func withFakeHome(t *testing.T) string {
t.Helper()
home := t.TempDir()
t.Setenv("HOME", home)
t.Setenv("CODEX_HOME", "") // hermetic: a dev shell's CODEX_HOME must not leak in
return home
}
// writeSkill creates <root>/<name>/SKILL.md with the given frontmatter name
// and description.
func writeSkill(t *testing.T, root, dir, name, description string) {
t.Helper()
skillDir := filepath.Join(root, dir)
if err := os.MkdirAll(skillDir, 0o755); err != nil {
t.Fatal(err)
}
content := "---\nname: " + name + "\ndescription: " + description + "\n---\n\nbody\n"
if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
func discover(t *testing.T) []agent.DiscoveredSkill {
t.Helper()
skills, err := (&codex.CodexAgent{}).DiscoverReviewSkills(context.Background())
if err != nil {
t.Fatalf("stub should not error; got %v", err)
t.Fatalf("unexpected error: %v", err)
}
if skills != nil {
t.Errorf("stub should return nil skills; got %+v", skills)
return skills
}
func nameOf(skills []agent.DiscoveredSkill, want string) bool {
for _, s := range skills {
if s.Name == want {
return true
}
}
return false
}
func TestCodexAgent_DiscoverReviewSkills_NoSkillsReturnsNilNil(t *testing.T) {
// Cannot t.Parallel — uses t.Setenv.
withFakeHome(t)
if skills := discover(t); skills != nil {
t.Errorf("skills = %v, want nil", skills)
}
}
func TestCodexAgent_DiscoverReviewSkills_FindsUserSkillInDollarForm(t *testing.T) {
home := withFakeHome(t)
writeSkill(t, filepath.Join(home, ".codex", "skills"), "code-reviewer", "code-reviewer",
"Review code changes with an emphasis on correctness.")
skills := discover(t)
if len(skills) != 1 {
t.Fatalf("skills count = %d, want 1: %+v", len(skills), skills)
}
if skills[0].Name != "$code-reviewer" {
t.Errorf("Name = %q, want $code-reviewer", skills[0].Name)
}
}
func TestCodexAgent_DiscoverReviewSkills_FindsPluginSkillNamespaced(t *testing.T) {
home := withFakeHome(t)
// Opaque (non-semver) version dir, like codex's content-hash versions.
writeSkill(t,
filepath.Join(home, ".codex", "plugins", "cache", "openai-curated", "github", "fef63ecf", "skills"),
"gh-review", "gh-review", "Review a GitHub pull request.")
skills := discover(t)
if !nameOf(skills, "$github:gh-review") {
t.Errorf("missing $github:gh-review; got %+v", skills)
}
}
func TestCodexAgent_DiscoverReviewSkills_FindsSuperpowersSkill(t *testing.T) {
home := withFakeHome(t)
writeSkill(t, filepath.Join(home, ".codex", "superpowers", "skills"),
"receiving-code-review", "receiving-code-review", "Receive code review feedback.")
skills := discover(t)
if !nameOf(skills, "$superpowers:receiving-code-review") {
t.Errorf("missing $superpowers:receiving-code-review; got %+v", skills)
}
}
func TestCodexAgent_DiscoverReviewSkills_SkipsNonReviewSkill(t *testing.T) {
home := withFakeHome(t)
skillsRoot := filepath.Join(home, ".codex", "skills")
writeSkill(t, skillsRoot, "code-reviewer", "code-reviewer", "Review code changes.")
// "committer" has no review keyword in its name → filtered by Matches.
writeSkill(t, skillsRoot, "committer", "committer", "Prepare clear commit messages.")
skills := discover(t)
if len(skills) != 1 || skills[0].Name != "$code-reviewer" {
t.Errorf("want only $code-reviewer; got %+v", skills)
}
}
// TestCodexAgent_DiscoverReviewSkills_HonorsCodexHome pins discovery to the
// agent's canonical home resolution: the rest of the codex agent resolves its
// config tree through resolveCodexHome (which honors CODEX_HOME), so skills
// installed under a custom codex home must be discoverable too — otherwise
// saved $skills fail spawn-time validation as "not installed" even though
// codex itself finds and runs them.
func TestCodexAgent_DiscoverReviewSkills_HonorsCodexHome(t *testing.T) {
// Cannot t.Parallel — uses t.Setenv.
withFakeHome(t) // HOME points at an empty dir; the skill lives elsewhere
codexHome := t.TempDir()
t.Setenv("CODEX_HOME", codexHome)
writeSkill(t, codexHome, "skills/code-review", "code-review", "Reviews code.")
if !nameOf(discover(t), "$code-review") {
t.Fatal("skill under CODEX_HOME not discovered — discovery must use resolveCodexHome, not ~/.codex")
}
}
Mcmd/entire/cli/agent/codex/discovery_test.go+116/-7
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
package codex
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"os"
"sync/atomic"
"time"
"github.com/entireio/cli/cmd/entire/cli/logging"
reviewtypes "github.com/entireio/cli/cmd/entire/cli/review/types"
)
// Polling/tailing cadence for the rollout token tailer.
const (
rolloutPollInterval = 300 * time.Millisecond
rolloutPollAttempts = 100 // ~30s for codex to create the rollout file
rolloutTailInterval = 400 * time.Millisecond
rolloutReadChunk = 8192
)
// tailRolloutTokens resolves the codex rollout transcript for threadID and
// tails it, emitting a cumulative reviewtypes.Tokens event for every
// token_count codex writes (~once per model turn). codex's `exec --json`
// stdout only carries usage on turn.completed envelopes, and a review is
// usually a single turn — so without this, consumers see no token movement
// until the run ends. The rollout file is the same source codex's
// interactive UI reads for its live token counter.
//
// token_count.total_token_usage is a running SESSION total (not per-turn
// scale like turn.completed usage), so each emission is an absolute count —
// matching consumers' overwrite-not-sum semantics. Duplicate totals are
// suppressed so we only emit on real movement. emitted is set after the
// first successful send; the parser uses it to suppress its per-turn-scale
// stdout emissions so a single source stays authoritative.
//
// Returns when stop is closed (the stdout stream ended) — after one final
// catch-up drain of the file, so the last token_count codex wrote is not
// lost to tick timing — or when the rollout file never appears. The caller
// must wait for this to return before closing the event channel (see
// parseCodexOutputBuf), and the run contract guarantees the consumer drains
// events until close, so sends here can neither race a close nor deadlock.
func tailRolloutTokens(threadID string, out chan<- reviewtypes.Event, stop <-chan struct{}, emitted *atomic.Bool) {
ctx := context.Background()
sessionDir, err := (&CodexAgent{}).GetSessionDir("")
if err != nil {
logging.Debug(ctx, "codex token tail: session dir unresolved", slog.String("error", err.Error()))
return
}
path := waitForRollout(ctx, sessionDir, threadID, stop)
if path == "" {
return
}
f, err := os.Open(path) //nolint:gosec // path is a glob match under codex's session dir, not user input
if err != nil {
logging.Debug(ctx, "codex token tail: open rollout failed", slog.String("error", err.Error()))
return
}
defer f.Close()
// Tail via os.File.Read rather than bufio.Reader: bufio is sticky on EOF
// and would never observe lines codex appends after we first catch up.
tail := rolloutTail{f: f, out: out, emitted: emitted, lastIn: -1, lastOut: -1}
ticker := time.NewTicker(rolloutTailInterval)
defer ticker.Stop()
for {
if err := tail.drain(); err != nil {
logging.Debug(ctx, "codex token tail: read rollout failed", slog.String("error", err.Error()))
return
}
select {
case <-stop:
// Final catch-up: codex may have flushed the terminal
// token_count between our last drain and stream end.
if err := tail.drain(); err != nil {
logging.Debug(ctx, "codex token tail: final drain failed", slog.String("error", err.Error()))
}
// Re-emit the last totals unconditionally (bypassing dedup):
// a per-turn stdout emission can race past the parser's
// tailerEmitted check in the instant before this tailer's
// first send is observed, and this re-send guarantees the
// session-cumulative value is the final Tokens regardless.
if tail.lastIn >= 0 {
out <- reviewtypes.Tokens{In: tail.lastIn, Out: tail.lastOut}
}
return
case <-ticker.C:
}
}
}
// rolloutTail holds the incremental read state for one rollout file.
type rolloutTail struct {
f *os.File
out chan<- reviewtypes.Event
emitted *atomic.Bool
pending []byte
lastIn int
lastOut int
}
// drain reads the file to EOF, emitting Tokens for every complete
// token_count line with new totals. Returns a non-nil error only for
// non-EOF read failures (deleted file, I/O error) — persistent failures
// must stop the tailer instead of silently re-polling forever.
func (t *rolloutTail) drain() error {
chunk := make([]byte, rolloutReadChunk)
for {
n, readErr := t.f.Read(chunk)
if n > 0 {
t.pending = append(t.pending, chunk[:n]...)
for {
idx := bytes.IndexByte(t.pending, '\n')
if idx < 0 {
break
}
line := t.pending[:idx]
t.pending = t.pending[idx+1:]
in, outTok, ok := parseRolloutTokenCount(line)
if !ok || (in == t.lastIn && outTok == t.lastOut) {
continue
}
t.lastIn, t.lastOut = in, outTok
// Unconditional send is safe: the parser waits for the
// tailer before closing the channel, and the run contract
// guarantees the consumer drains until close.
t.out <- reviewtypes.Tokens{In: in, Out: outTok}
t.emitted.Store(true)
}
}
if readErr != nil {
if errors.Is(readErr, io.EOF) {
return nil // caught up — wait for the file to grow
}
return fmt.Errorf("read rollout: %w", readErr)
}
}
}
// waitForRollout polls for the rollout file matching threadID until it
// appears or stop fires — never giving up while the review is running, since
// a rollout that materialises late (slow codex startup, unusual layout
// timing) should still get live tokens for the rest of the run. After the
// expected-quickly window it debug-logs once (the likely signature of a
// codex release changing the rollout layout, which would otherwise silently
// disable live tokens) and backs off to a slower poll.
func waitForRollout(ctx context.Context, sessionDir, threadID string, stop <-chan struct{}) string {
return pollForRollout(ctx, sessionDir, threadID, stop, rolloutPollAttempts, rolloutPollInterval)
}
func pollForRollout(ctx context.Context, sessionDir, threadID string, stop <-chan struct{}, window int, interval time.Duration) string {
for attempt := 0; ; attempt++ {
if path := findRolloutBySessionID(sessionDir, threadID); path != "" {
return path
}
wait := interval
if attempt >= window {
if attempt == window {
logging.Debug(ctx, "codex token tail: rollout file still missing; continuing to poll",
slog.String("session_dir", sessionDir), slog.String("thread_id", threadID))
}
wait = interval * 8 // ~2.4s at production cadence — cheap for a minutes-long run
}
select {
case <-stop:
return ""
case <-time.After(wait):
}
}
}
// parseRolloutTokenCount extracts cumulative input/output token totals from one
// rollout JSONL line. ok is false for any line that isn't a token_count event
// carrying total_token_usage. Reuses the rolloutLine/eventMsgPayload/
// tokenCountInfo shapes from transcript.go so the two readers can't drift.
func parseRolloutTokenCount(data []byte) (in, out int, ok bool) {
var line rolloutLine
if json.Unmarshal(data, &line) != nil || line.Type != "event_msg" {
return 0, 0, false
}
var evt eventMsgPayload
if json.Unmarshal(line.Payload, &evt) != nil || evt.Type != "token_count" || len(evt.Info) == 0 {
return 0, 0, false
}
var info tokenCountInfo
if json.Unmarshal(evt.Info, &info) != nil || info.TotalTokenUsage == nil {
return 0, 0, false
}
return info.TotalTokenUsage.InputTokens, info.TotalTokenUsage.OutputTokens, true
}
Acmd/entire/cli/agent/codex/review_tokens.go+195
package codex
import ( "context" "io" "os" "path/filepath" "strconv" "sync/atomic" "testing" "time"
reviewtypes "github.com/entireio/cli/cmd/entire/cli/review/types" )
const tailTestThreadID = "019e8d8f-9d70-7021-b8fe-2c13802e3443"
func tokenLine(in, out int) string {
return {"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage": +
{"input_tokens": + strconv.Itoa(in) + ,"output_tokens": + strconv.Itoa(out) + }}}} + "\n"
}
func TestParseRolloutTokenCount(t *testing.T) {
t.Parallel()
in, out, ok := parseRolloutTokenCount([]byte(tokenLine(25338, 595)))
if !ok || in != 25338 || out != 595 {
t.Fatalf("token_count line: got in=%d out=%d ok=%v, want 25338/595/true", in, out, ok)
}
// Non-token_count lines are ignored.
for _, line := range []string{
{"type":"response_item","payload":{"type":"reasoning"}},
{"type":"event_msg","payload":{"type":"agent_message"}},
not json,
``,
} {
if _, _, ok := parseRolloutTokenCount([]byte(line)); ok {
t.Errorf("expected ok=false for %q", line)
}
}
}
// TestTailRolloutTokens_TailsAppendedLines is the core behavior: the tailer // must emit Tokens for token_count lines that codex appends after the tailer // has already caught up to EOF (a plain bufio.Reader would miss these). func TestTailRolloutTokens_TailsAppendedLines(t *testing.T) { // Cannot t.Parallel — uses t.Setenv. dir := t.TempDir() t.Setenv("ENTIRE_TEST_CODEX_SESSION_DIR", dir)
rollout := filepath.Join(dir, "rollout-2026-06-03T08-57-39-"+tailTestThreadID+".jsonl") if err := os.WriteFile(rollout, []byte(tokenLine(25338, 595)), 0o644); err != nil { t.Fatal(err) }
out := make(chan reviewtypes.Event, 16) stop := make(chan struct{}) done := make(chan struct{}) go func() { tailRolloutTokens(tailTestThreadID, out, stop, new(atomic.Bool)) close(done) }() defer func() { close(stop) <-done }()
first := awaitTokens(t, out) if first.In != 25338 || first.Out != 595 { t.Fatalf("first tokens = %+v, want {25338, 595}", first) }
// Append a second token_count after the tailer caught up — it must see it. f, err := os.OpenFile(rollout, os.O_APPEND|os.O_WRONLY, 0o644) if err != nil { t.Fatal(err) } if _, err := f.WriteString(tokenLine(52798, 1123)); err != nil { t.Fatal(err) } _ = f.Close()
second := awaitTokens(t, out) if second.In != 52798 || second.Out != 1123 { t.Fatalf("second tokens = %+v, want {52798, 1123} (appended line not tailed)", second) } }
// awaitTokens waits for the next Tokens event or fails on timeout. func awaitTokens(t *testing.T, out <-chan reviewtypes.Event) reviewtypes.Tokens { t.Helper() timeout := time.After(5 * time.Second) for { select { case ev := <-out: if tk, ok := ev.(reviewtypes.Tokens); ok { return tk } case <-timeout: t.Fatal("timed out waiting for a Tokens event") } } }
// startTailerFixture writes a rollout file for tailTestThreadID, starts the // parser on a pipe, sends thread.started, and waits for the tailer's first // Tokens. Returns the pipe writer, the event channel, and the rollout path. func startTailerFixture(t *testing.T, firstLine string, wantIn, wantOut int) (*io.PipeWriter, <-chan reviewtypes.Event, string) { t.Helper() dir := t.TempDir() t.Setenv("ENTIRE_TEST_CODEX_SESSION_DIR", dir) rollout := filepath.Join(dir, "rollout-2026-06-03T08-57-39-"+tailTestThreadID+".jsonl") if err := os.WriteFile(rollout, []byte(firstLine), 0o644); err != nil { t.Fatal(err) }
pr, pw := io.Pipe()
events := parseCodexOutput(pr)
// Inline write is safe: the parser goroutine is already draining pr.
if _, err := pw.Write([]byte({"type":"thread.started","thread_id":" + tailTestThreadID + "} + "\n")); err != nil {
t.Fatalf("write thread.started: %v", err)
}
// The tailer (not stdout — no turn.completed was written yet) must // deliver Tokens while the stream is still open. tk := awaitTokens(t, events) if tk.In != wantIn || tk.Out != wantOut { t.Fatalf("tailer tokens = %+v, want {%d, %d}", tk, wantIn, wantOut) } return pw, events, rollout }
// collectUntilClose drains events until the channel closes, failing the test // if it doesn't close within 5s. func collectUntilClose(t *testing.T, events <-chan reviewtypes.Event) []reviewtypes.Event { t.Helper() var got []reviewtypes.Event drained := make(chan struct{}) go func() { for ev := range events { got = append(got, ev) } close(drained) }() select { case <-drained: case <-time.After(5 * time.Second): t.Fatal("event channel did not close — tailer not stopped") } return got }
// TestParseCodexOutput_StartsRolloutTailerOnThreadStarted locks the wiring: // the parser launches the rollout tailer when thread.started carries a // thread_id, so Tokens flow from the rollout file between turn boundaries, // and the parser stops the tailer and waits for it before closing the event // channel (no send-on-closed-channel race). func TestParseCodexOutput_StartsRolloutTailerOnThreadStarted(t *testing.T) { // Cannot t.Parallel — uses t.Setenv. pw, events, _ := startTailerFixture(t, tokenLine(11111, 22), 11111, 22) _ = pw.Close() collectUntilClose(t, events) }
// TestParseCodexOutput_FinishedIsLastEvenWithPendingTailerLines pins the // parser contract that Finished is the final event: the tailer must be // stopped and awaited BEFORE the terminal emissions, not in a defer that // runs after them — otherwise a tailer with unread rollout lines keeps // sending Tokens after Finished. func TestParseCodexOutput_FinishedIsLastEvenWithPendingTailerLines(t *testing.T) { // Cannot t.Parallel — uses t.Setenv. pw, events, rollout := startTailerFixture(t, tokenLine(1000, 50), 1000, 50)
// Append a large backlog, then wait until the tailer is actively // draining it (a few backlog Tokens observed) before signalling EOF — // that pins the tailer mid-send exactly when the parser emits its // terminal events. f, err := os.OpenFile(rollout, os.O_APPEND|os.O_WRONLY, 0o644) if err != nil { t.Fatal(err) } for i := 1; i <= 2000; i++ { if _, err := f.WriteString(tokenLine(1000+i, 50+i)); err != nil { t.Fatal(err) } } _ = f.Close() for range 3 { awaitTokens(t, events) } _ = pw.Close() // EOF with tailer mid-backlog
got := collectUntilClose(t, events) if len(got) == 0 { t.Fatal("no events after EOF") } last := got[len(got)-1] if _, ok := last.(reviewtypes.Finished); !ok { t.Fatalf("last event = %#v, want Finished (Tokens after Finished violates the parser contract)", last) } }
// TestParseCodexOutput_UsagelessTurnCompletedDoesNotClobberTailerTokens pins // the backstop behavior: a terminal turn.completed WITHOUT a usage block // must not emit Tokens{0,0} — under overwrite-not-sum consumer semantics // that would erase the rollout tailer's genuine totals. func TestParseCodexOutput_UsagelessTurnCompletedDoesNotClobberTailerTokens(t *testing.T) { // Cannot t.Parallel — uses t.Setenv. pw, events, _ := startTailerFixture(t, tokenLine(1000, 50), 1000, 50)
if _, err := pw.Write([]byte({"type":"turn.completed"} + "\n")); err != nil {
t.Fatalf("write turn.completed: %v", err)
}
_ = pw.Close()
got := collectUntilClose(t, events) // The tailer's {1000, 50} was already consumed by startTailerFixture; // it must remain the final observed value — any later Tokens (in // particular {0,0} from the old backstop) would clobber it under the // consumers' overwrite semantics. lastTokens := reviewtypes.Tokens{In: 1000, Out: 50} finishedOK := false for _, ev := range got { switch e := ev.(type) { case reviewtypes.Tokens: if e.In == 0 && e.Out == 0 { t.Fatalf("observed Tokens{0,0} — clobbers the tailer's totals") } lastTokens = e case reviewtypes.Finished: finishedOK = e.Success } } if !finishedOK { t.Error("turn.completed present: want Finished{Success:true}") } if lastTokens.In != 1000 || lastTokens.Out != 50 { t.Errorf("final tokens = %+v, want tailer's {1000, 50} to stand", lastTokens) } }
// TestParseCodexOutput_TailerSuppressesPerTurnStdoutTokens pins single-source // authority: rollout token_count totals are session-cumulative while // turn.completed usage is per-turn scale, so once the tailer has emitted, // per-turn stdout values must be suppressed — mixing the two makes the live // counter flap between scales and the final value nondeterministic. func TestParseCodexOutput_TailerSuppressesPerTurnStdoutTokens(t *testing.T) { // Cannot t.Parallel — uses t.Setenv. pw, events, rollout := startTailerFixture(t, tokenLine(1000, 50), 1000, 50)
// The session-cumulative rollout advances to 2000/150... f, err := os.OpenFile(rollout, os.O_APPEND|os.O_WRONLY, 0o644) if err != nil { t.Fatal(err) } if _, err := f.WriteString(tokenLine(2000, 150)); err != nil { t.Fatal(err) } _ = f.Close()
// ...then a per-turn-scale turn.completed arrives on stdout.
if _, err := pw.Write([]byte({"type":"turn.completed","usage":{"input_tokens":500,"output_tokens":30}} + "\n")); err != nil {
t.Fatalf("write turn.completed: %v", err)
}
_ = pw.Close()
got := collectUntilClose(t, events) var lastTokens reviewtypes.Tokens for _, ev := range got { switch e := ev.(type) { case reviewtypes.Tokens: if e.In == 500 && e.Out == 30 { t.Fatalf("per-turn stdout Tokens{500,30} emitted despite active tailer — scale flap") } lastTokens = e case reviewtypes.Finished: if !e.Success { t.Error("want Finished{Success:true}") } } } if lastTokens.In != 2000 || lastTokens.Out != 150 { t.Errorf("final tokens = %+v, want the tailer's cumulative {2000, 150}", lastTokens) } }
// TestTailRolloutTokens_PartialLineAppend covers the hand-rolled line buffer: // a token_count written in two partial chunks must be parsed exactly once, // when the newline completes it. func TestTailRolloutTokens_PartialLineAppend(t *testing.T) { // Cannot t.Parallel — uses t.Setenv. dir := t.TempDir() t.Setenv("ENTIRE_TEST_CODEX_SESSION_DIR", dir) rollout := filepath.Join(dir, "rollout-2026-06-03T08-57-39-"+tailTestThreadID+".jsonl") line := tokenLine(31337, 42) half := len(line) / 2 if err := os.WriteFile(rollout, []byte(line[:half]), 0o644); err != nil { t.Fatal(err) }
// Give the tailer a moment on the partial line, then complete it. select { case ev := <-out: t.Fatalf("event %#v emitted from a partial line", ev) case <-time.After(600 * time.Millisecond): } f, err := os.OpenFile(rollout, os.O_APPEND|os.O_WRONLY, 0o644) if err != nil { t.Fatal(err) } if _, err := f.WriteString(line[half:]); err != nil { t.Fatal(err) } _ = f.Close()
tk := awaitTokens(t, out) if tk.In != 31337 || tk.Out != 42 { t.Fatalf("tokens = %+v, want {31337, 42}", tk) } }
// TestTailRolloutTokens_ReemitsLastTotalsOnStop pins the TOCTOU hardening: // on stop, after the final catch-up drain, the tailer re-emits its last // known totals. This guarantees the tailer's session-cumulative value is the // final Tokens even if a per-turn stdout emission raced past the parser's // tailerEmitted check in the instant before the tailer's first Store(true). func TestTailRolloutTokens_ReemitsLastTotalsOnStop(t *testing.T) { // Cannot t.Parallel — uses t.Setenv. dir := t.TempDir() t.Setenv("ENTIRE_TEST_CODEX_SESSION_DIR", dir) rollout := filepath.Join(dir, "rollout-2026-06-03T08-57-39-"+tailTestThreadID+".jsonl") if err := os.WriteFile(rollout, []byte(tokenLine(7000, 300)), 0o644); err != nil { t.Fatal(err) }
out := make(chan reviewtypes.Event, 16) stop := make(chan struct{}) done := make(chan struct{}) go func() { tailRolloutTokens(tailTestThreadID, out, stop, new(atomic.Bool)) close(done) }()
first := awaitTokens(t, out) if first.In != 7000 || first.Out != 300 { t.Fatalf("first tokens = %+v, want {7000, 300}", first) }
close(stop) <-done // The stop path must have re-emitted the last totals (dedup bypassed). select { case ev := <-out: tk, ok := ev.(reviewtypes.Tokens) if !ok || tk.In != 7000 || tk.Out != 300 { t.Fatalf("post-stop event = %#v, want re-emitted Tokens{7000, 300}", ev) } default: t.Fatal("no re-emitted Tokens after stop — TOCTOU window unguarded") } }
func TestTailRolloutTokens_ReturnsOnStopWhenNoRollout(t *testing.T) { // Cannot t.Parallel — uses t.Setenv. t.Setenv("ENTIRE_TEST_CODEX_SESSION_DIR", t.TempDir()) out := make(chan reviewtypes.Event, 4) stop := make(chan struct{}) done := make(chan struct{}) go func() { tailRolloutTokens(tailTestThreadID, out, stop, new(atomic.Bool)) close(done) }() close(stop) select { case <-done: case <-time.After(5 * time.Second): t.Fatal("tailRolloutTokens did not return promptly after stop with no rollout file") } }
// TestPollForRollout_KeepsLookingPastTheWindow pins that the poll never // gives up while stop is open: a rollout that materialises after the // expected-quickly window must still be found (previously the poll returned // "" after ~30s and live tokens were lost for the rest of the run). func TestPollForRollout_KeepsLookingPastTheWindow(t *testing.T) { // Cannot t.Parallel — uses t.Setenv. dir := t.TempDir() t.Setenv("ENTIRE_TEST_CODEX_SESSION_DIR", dir) rollout := filepath.Join(dir, "rollout-2026-06-03T08-57-39-"+tailTestThreadID+".jsonl")
stop := make(chan struct{}) defer close(stop) got := make(chan string, 1) go func() { got <- pollForRollout(context.Background(), dir, tailTestThreadID, stop, 3, 10*time.Millisecond) }()
// Create the file well after the 3-attempt window has elapsed. time.Sleep(200 * time.Millisecond) if err := os.WriteFile(rollout, []byte(tokenLine(1, 1)), 0o644); err != nil { t.Fatal(err) }
select { case path := <-got: if path != rollout { t.Fatalf("pollForRollout = %q, want %q (gave up instead of continuing past the window)", path, rollout) } case <-time.After(5 * time.Second): t.Fatal("pollForRollout did not find the late rollout") } }
Acmd/entire/cli/agent/codex/review\_tokens\_test.go+423
9 unmodified lines
10 11 12 13 14 15 16 17 16 unmodified lines
34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 37 54 55 56 57 4 unmodified lines
62 63 64 48 49 50 51 52 53 65 66 56 67 68 69 70 71 72 59 60 73 74 75 76 77 17 unmodified lines
95 96 97 84 85 98 99 100 101 102 103 104 105 106 107 108 109 110 111 17 unmodified lines
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 112 151 152 153 154 18 unmodified lines
173 174 175 137 138 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 8 unmodified lines
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 3 unmodified lines
250 251 252 167 168 169 170 171 172 173 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 10 unmodified lines
278 279 280 190 191 192 193 194 281 282 283 284 285 286 287 288 289
9 unmodified lines
"os" "os/exec" "strings" "sync" "sync/atomic"
"github.com/entireio/cli/cmd/entire/cli/logging" "github.com/entireio/cli/cmd/entire/cli/review" 16 unmodified lines
// buildCodexReviewCmd builds the exec.Cmd for a codex review run.
// Exposed at package level for test inspection of argv, stdin, and env.
// buildCodexReviewCmd builds the exec.Cmd for a codex review run.
//
// Configured skills are passed through in codex's native $name form — NOT
// paraphrased. Codex's skill system injects a catalog of installed skills
// into every exec session and loads the matching SKILL.md when the prompt
// names one, so the agent runs the real configured workflow. A previous
// version silently REPLACED /review with a generic 28-word instruction: the
// configured skill never ran (the codex sibling of the claude -p
// slash-expansion bug, where the built-in /review hijacked the prompt).
//
// Native codex exec review is intentionally NOT used: it rejects an extra
// prompt when a scope flag is set, and codex hooks don't fire during it —
// leaving no channel for Entire's scope enumeration, per-run prompt, and
// checkpoint context. Plain codex exec - with the composed prompt on stdin
// runs the same skill while carrying our arguments.
func buildCodexReviewCmd(ctx context.Context, cfg reviewtypes.RunConfig) *exec.Cmd {
promptCfg := cfg
promptCfg.Skills = expandCodexBuiltinReview(cfg.Skills)
promptCfg.Skills = codexNativeSkillInvocations(cfg.Skills)
args := []string{codexExecCommand, "--skip-git-repo-check", "--json"}
args = review.AppendModelFlag(args, cfg.Model)
args = append(args, "-")
4 unmodified lines
return cmd }
// Codex's native exec review --base <branch> rejects an additional prompt,
// so expand /review into text and run normal codex exec -. That preserves
// Entire's scoped base clause, per-run instructions, and checkpoint context.
const codexBuiltinReviewPrompt = "Review the current branch changes and report actionable findings. " +
"Prioritize correctness, regressions, security, and missing test coverage. Do not make code changes."
const codexExecCommand = "exec"
func expandCodexBuiltinReview(skills []string) []string { // codexNativeSkillInvocations rewrites slash-form skill invocations (the // agent-portable form profiles are configured with) into codex's native // $name form. Non-slash entries (plain instruction text) pass verbatim. func codexNativeSkillInvocations(skills []string) []string { out := make([]string, 0, len(skills)) for _, skill := range skills { if skill == "/review" { out = append(out, codexBuiltinReviewPrompt) if rest, ok := strings.CutPrefix(skill, "/"); ok && rest != "" { out = append(out, "$"+rest) continue } out = append(out, skill) 17 unmodified lines
// On a scanner error or a missing turn.completed envelope, emits RunError
// (scanner) or Finished{Success: false} (missing turn) accordingly.
//
// Tokens are emitted only at the terminal turn.completed envelope, not
// incrementally — codex's usage fields land once at end-of-turn.
// Live-token semantics: codex's --json output carries usage ONLY on
// turn.completed envelopes. Verified against codex-cli 0.130.0 stdout
// for both short and long prompts — no intermediate envelope
// (item.started, item.completed, etc.) carries a usage block. Codex's
// on-disk session log (the event_msg{type:"token_count"} shape the
// transcript parser consumes) is a separate format, not surfaced
// through exec --json — the rollout tailer (review_tokens.go) reads
// it for live counts between turn boundaries.
//
// Tokens are emitted at every turn.completed envelope so multi-turn
// runs show iterative updates.
//
// Package-private; called directly from this package's tests so they can
// drive raw stdout fixtures through the parser without going through the
17 unmodified lines
out := make(chan reviewtypes.Event, 32) go func() { defer close(out) // The rollout token tailer (started on thread.started) runs concurrently // and also sends on out. It must be stopped and awaited BEFORE the // terminal Tokens/RunError/Finished emissions — Finished is contractually // the last event, and a lagging tailer tick would otherwise overwrite the // final recorded totals after completion. stopTailer is called explicitly // on every exit path ahead of the terminal sends; the deferred call is a // safety net (sync.Once) that also guarantees no send can hit the closed // channel. stop := make(chan struct{}) var tailWG sync.WaitGroup var tailerEmitted atomic.Bool stopTailer := sync.OnceFunc(func() { close(stop) tailWG.Wait() }) defer stopTailer() out <- reviewtypes.Started{} scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, min(1024*1024, maxBuf)), maxBuf) var seenTurnComplete bool var seenTurnComplete, emittedTokens, tailerStarted bool var turnUsage codexUsage var failureMsg string for scanner.Scan() { 18 unmodified lines
// default arm logs unknown types at Debug so drift can be // triaged via ENTIRE_LOG_LEVEL=debug. switch env.Type { case "thread.started", "turn.started": // Session/turn markers — no event emitted. case "thread.started": // Launch the rollout token tailer once. codex's exec --json stdout // only carries usage on turn.completed, so we tail the rollout // file (located by thread_id) for live per-turn token totals — // the same source codex's interactive UI reads. if !tailerStarted && env.ThreadID != "" { tailerStarted = true tailWG.Add(1) go func(id string) { defer tailWG.Done() tailRolloutTokens(id, out, stop, &tailerEmitted) }(env.ThreadID) } case "turn.started": // Turn marker — no event emitted. case "item.started": if env.Item.Type == "command_execution" { out <- reviewtypes.ToolCall{Name: "exec", Args: env.Item.Command} 8 unmodified lines
case "turn.completed": seenTurnComplete = true turnUsage = env.Usage // Emit Tokens at every turn boundary so multi-turn reviews // show iterative updates — but only while the rollout tailer // hasn't produced values: turn.completed usage is treated as // per-turn scale, the tailer's token_count totals are // session-cumulative, and mixing the two in one // overwrite-not-sum slot makes the counter flap between // scales. Once the tailer has emitted, it is the single // authoritative source. // // Scale caveat: whether turn.completed usage is per-turn or // session-cumulative is unverified against real MULTI-turn // codex output — exec-mode reviews are single-turn, where // the two are identical and this code is exact. In the rare // multi-turn no-rollout fallback, the recorded total is the // last turn's usage (an under-count if per-turn); when the // rollout tailer runs — the normal case — its cumulative // totals win regardless. // // codex reports cached_input_tokens as a subset of // input_tokens and reasoning_output_tokens as a subset of // output_tokens (matching OpenAI's chat-completions usage // shape), so do NOT sum the subset fields — that would // double-count. if !tailerEmitted.Load() && (env.Usage.InputTokens > 0 || env.Usage.OutputTokens > 0) { out <- reviewtypes.Tokens{ In: env.Usage.InputTokens, Out: env.Usage.OutputTokens, } emittedTokens = true } default: logging.Debug(context.Background(), "codex parser: unknown envelope type", slog.String("type", env.Type)) } } // Stream over — stop the tailer BEFORE any terminal emission so // Finished stays the last event and no lagging tailer tick can // overwrite the final recorded totals. stopTailer() if err := scanner.Err(); err != nil { out <- reviewtypes.RunError{Err: fmt.Errorf("read stdout: %w", err)} out <- reviewtypes.Finished{Success: false} 3 unmodified lines
out <- reviewtypes.RunError{Err: fmt.Errorf("codex: %s", failureMsg)}
}
if seenTurnComplete {
// codex reports cached_input_tokens as a subset of input_tokens
// and reasoning_output_tokens as a subset of output_tokens
// (matching OpenAI's chat-completions usage shape), so do NOT
// sum the subset fields — that would double-count.
out <- reviewtypes.Tokens{
In: turnUsage.InputTokens,
Out: turnUsage.OutputTokens,
// Defensive backstop for a stream whose turn.completed carried
// usage that never got emitted (can't happen today — the
// per-turn arm emits whenever usage is non-zero and no tailer
// value exists). Gated on non-zero usage: emitting {0,0} here
// would only ever ERASE the tailer's genuine totals under the
// consumers' overwrite-not-sum semantics.
if !emittedTokens && !tailerEmitted.Load() &&
(turnUsage.InputTokens > 0 || turnUsage.OutputTokens > 0) {
out <- reviewtypes.Tokens{
In: turnUsage.InputTokens,
Out: turnUsage.OutputTokens,
}
}
// Success is hard-coded true here because codex's turn.completed
// envelope has no turn-level error field in 0.130.0. If a future
10 unmodified lines
}
type codexEnvelope struct {
Type string json:"type"
Item codexItem json:"item"
Usage codexUsage json:"usage"
Message string json:"message"
Error codexErrorField json:"error"
Type string json:"type"
ThreadID string json:"thread_id" // present on thread.started; locates the rollout file
Item codexItem json:"item"
Usage codexUsage json:"usage"
Message string json:"message"
Error codexErrorField json:"error"
}
// codexErrorField captures the nested error message shape some codex envelopes
Mcmd/entire/cli/agent/codex/reviewer.go+119/-27
121 unmodified lines
122 123 124 125 125 126 127 128 128 129 130 131 214 unmodified lines
346 347 348 349 349 350 351 352 353 354 355 75 unmodified lines
431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 29 unmodified lines
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568
121 unmodified lines
prompt := readCodexCmdStdin(t, cmd) if strings.Contains(prompt, "/review") { t.Fatalf("builtin review prompt should not include raw /review:\n%s", prompt) t.Fatalf("slash-form skill must be rewritten to codex's $ form:\n%s", prompt) } for _, wantText := range []string{ "Review the current branch changes and report actionable findings.", "$review", "Focus on auth regressions.", "Scope: review the commits unique to this branch vs main, plus any uncommitted changes in the working tree. Ignore code outside this scope.", "Commits in scope (newest first):", 214 unmodified lines
}
func TestParseCodexOutput_NoTurnCompletedMeansFailed(t *testing.T) {
t.Parallel()
// Cannot t.Parallel — uses t.Setenv. The thread.started envelope
// launches the rollout tailer; without the session-dir override it
// would glob the real ~/.codex/sessions.
t.Setenv("ENTIRE_TEST_CODEX_SESSION_DIR", t.TempDir())
// A truncated session: thread starts and an item completes, but no
// turn.completed envelope ever arrives. The parser must surface
// this as Finished{Success: false}.
75 unmodified lines
} }
// TestParseCodexOutput_EmitsTokensAtEveryTurnCompleted locks the live-token
// contract for codex: its --json output carries usage on every
// turn.completed envelope, and the parser emits Tokens at each turn
// boundary so multi-turn reviews show iterative updates. Captured by
// running real codex-cli 0.130.0 — no item.* envelope ever carried a
// usage field, so emission stays anchored to turn.completed.
func TestParseCodexOutput_EmitsTokensAtEveryTurnCompleted(t *testing.T) {
// Cannot t.Parallel — uses t.Setenv. The thread.started envelope
// launches the rollout tailer; without the session-dir override it
// would glob the real ~/.codex/sessions, and a matching rollout could
// inject Tokens into the exact-count assertions below.
t.Setenv("ENTIRE_TEST_CODEX_SESSION_DIR", t.TempDir())
// Synthetic multi-turn stream (real envelope shapes, invented usage
// numbers) with a turn.completed at every turn boundary and NO rollout
// file — the no-tailer fallback path. The parser emits each turn's
// usage as it arrives. NOTE: turn.completed usage is treated as
// per-turn scale (see the parser doc); exec-mode reviews are single
// turn in practice, where per-turn and cumulative are identical, so
// multi-turn fallback totals are a documented approximation (the last
// turn's usage), not a verified cumulative sum.
input := strings.Join([]string{
{"type":"thread.started","thread_id":"tid-1"},
{"type":"turn.started"},
{"type":"item.started","item":{"id":"item_0","type":"command_execution","command":"ls","aggregated_output":"","exit_code":null,"status":"in_progress"}},
{"type":"item.completed","item":{"id":"item_0","type":"command_execution","command":"ls","aggregated_output":"a\nb\nc","exit_code":0,"status":"completed"}},
{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"Found three files."}},
{"type":"turn.completed","usage":{"input_tokens":34317,"cached_input_tokens":19712,"output_tokens":240,"reasoning_output_tokens":114}},
{"type":"turn.started"},
{"type":"item.started","item":{"id":"item_2","type":"command_execution","command":"cat a","aggregated_output":"","exit_code":null,"status":"in_progress"}},
{"type":"item.completed","item":{"id":"item_2","type":"command_execution","command":"cat a","aggregated_output":"hello","exit_code":0,"status":"completed"}},
{"type":"item.completed","item":{"id":"item_3","type":"agent_message","text":"Done."}},
{"type":"turn.completed","usage":{"input_tokens":35820,"cached_input_tokens":20114,"output_tokens":401,"reasoning_output_tokens":160}},
"",
}, "\n")
var tokens []reviewtypes.Tokens for ev := range parseCodexOutput(strings.NewReader(input)) { if tk, ok := ev.(reviewtypes.Tokens); ok { tokens = append(tokens, tk) } }
if len(tokens) != 2 { t.Fatalf("Tokens count = %d, want exactly 2 (one per turn.completed); got events: %+v", len(tokens), tokens) } if tokens[0].In != 34317 || tokens[0].Out != 240 { t.Errorf("tokens[0] = %+v, want {In:34317, Out:240}", tokens[0]) } if tokens[1].In != 35820 || tokens[1].Out != 401 { t.Errorf("tokens[1] = %+v, want {In:35820, Out:401}", tokens[1]) } }
func collectCodexEvents(ch <-chan reviewtypes.Event) []reviewtypes.Event { var events []reviewtypes.Event for ev := range ch { 29 unmodified lines
} return m }
// TestBuildCodexReviewCmd_SkillsPassNativelyNotParaphrased locks the fix for // codex skill invocation: configured skills reach codex in its native $name // form so codex's skill system loads the real SKILL.md, instead of /review // being silently REPLACED with a generic 28-word paraphrase (which meant the // configured skill never ran — the codex sibling of the claude -p // slash-expansion bug). func TestBuildCodexReviewCmd_SkillsPassNativelyNotParaphrased(t *testing.T) { t.Parallel() cmd := buildCodexReviewCmd(context.Background(), reviewtypes.RunConfig{ Skills: []string{"/review", "/pr-review-toolkit:review-pr", "plain instruction line"}, }) stdin, err := io.ReadAll(cmd.Stdin) if err != nil { t.Fatal(err) } prompt := string(stdin) for _, want := range []string{"$review", "$pr-review-toolkit:review-pr", "plain instruction line"} { if !strings.Contains(prompt, want) { t.Errorf("prompt missing native skill invocation %q:\n%s", want, prompt) } } if strings.Contains(prompt, "Review the current branch changes and report actionable findings") { t.Errorf("prompt still contains the generic paraphrase:\n%s", prompt) } if strings.Contains(prompt, "/review\n") || strings.HasSuffix(prompt, "/review") { t.Errorf("slash-form skill leaked through untransformed:\n%s", prompt) } }
// TestBuildCodexReviewCmd_PromptOverrideVerbatim ensures the $-form transform // never touches a verbatim prompt override. func TestBuildCodexReviewCmd_PromptOverrideVerbatim(t *testing.T) { t.Parallel() cmd := buildCodexReviewCmd(context.Background(), reviewtypes.RunConfig{ Skills: []string{"/review"}, PromptOverride: "/review exactly as written", }) stdin, err := io.ReadAll(cmd.Stdin) if err != nil { t.Fatal(err) } if got := string(stdin); got != "/review exactly as written" { t.Errorf("PromptOverride modified: %q", got) } }
Mcmd/entire/cli/agent/codex/reviewer\_test.go+106/-3
129 unmodified lines
130 131 132 133 134 135 136 137 138 139 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 175 unmodified lines
343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363
129 unmodified lines
subagentStartCmd := cmdPrefix + HookNameSubagentStart
subagentEndCmd := cmdPrefix + HookNameSubagentStop
if !localDev {
sessionStartCmd = agent.WrapProductionSilentHookCommand(sessionStartCmd)
sessionEndCmd = agent.WrapProductionSilentHookCommand(sessionEndCmd)
beforeSubmitPromptCmd = agent.WrapProductionSilentHookCommand(beforeSubmitPromptCmd)
stopCmd = agent.WrapProductionSilentHookCommand(stopCmd)
preCompactCmd = agent.WrapProductionSilentHookCommand(preCompactCmd)
subagentStartCmd = agent.WrapProductionSilentHookCommand(subagentStartCmd)
subagentEndCmd = agent.WrapProductionSilentHookCommand(subagentEndCmd)
// Cursor spawns hook commands through the native OS shell (cmd.exe on
// Windows), so a sh -c '…' wrapper silently fails to launch on a
// Windows host without a working POSIX sh — no hook fires and, because
// this is the silent wrapper, no error surfaces (issue #1424).
// UseWindowsProductionHooks probes for a runnable sh and only swaps in
// the native cmd.exe wrapper when one is absent, so this is a no-op on
// hosts (incl. all non-Windows) where the sh wrapper already works.
useWindowsHooks := agent.UseWindowsProductionHooks(ctx, localDev)
sessionStartCmd = agent.WrapProductionSilentHookCommandForOS(sessionStartCmd, useWindowsHooks)
sessionEndCmd = agent.WrapProductionSilentHookCommandForOS(sessionEndCmd, useWindowsHooks)
beforeSubmitPromptCmd = agent.WrapProductionSilentHookCommandForOS(beforeSubmitPromptCmd, useWindowsHooks)
stopCmd = agent.WrapProductionSilentHookCommandForOS(stopCmd, useWindowsHooks)
preCompactCmd = agent.WrapProductionSilentHookCommandForOS(preCompactCmd, useWindowsHooks)
subagentStartCmd = agent.WrapProductionSilentHookCommandForOS(subagentStartCmd, useWindowsHooks)
subagentEndCmd = agent.WrapProductionSilentHookCommandForOS(subagentEndCmd, useWindowsHooks)
}
count := 0
// Add hooks if they don't exist if !hookCommandExists(sessionStart, sessionStartCmd) { sessionStart = append(sessionStart, CursorHookEntry{Command: sessionStartCmd}) count++ } if !hookCommandExists(sessionEnd, sessionEndCmd) { sessionEnd = append(sessionEnd, CursorHookEntry{Command: sessionEndCmd}) count++ } if !hookCommandExists(beforeSubmitPrompt, beforeSubmitPromptCmd) { beforeSubmitPrompt = append(beforeSubmitPrompt, CursorHookEntry{Command: beforeSubmitPromptCmd}) count++ } if !hookCommandExists(stop, stopCmd) { stop = append(stop, CursorHookEntry{Command: stopCmd}) count++ } if !hookCommandExists(preCompact, preCompactCmd) { preCompact = append(preCompact, CursorHookEntry{Command: preCompactCmd}) count++ } if !hookCommandExists(subagentStart, subagentStartCmd) { subagentStart = append(subagentStart, CursorHookEntry{Command: subagentStartCmd}) count++ } if !hookCommandExists(subagentStop, subagentEndCmd) { subagentStop = append(subagentStop, CursorHookEntry{Command: subagentEndCmd}) count++ } // Sync each hook to its desired command. syncEntireHook replaces any // stale-form Entire hook (e.g. an sh-wrapped entry from a previous install) // with the current command even without --force, so a wrapper-form change — // notably the sh↔cmd.exe migration driven by UseWindowsProductionHooks when // a Windows host gains or loses a working POSIX sh — cleanly replaces rather // than leaving a dead duplicate entry that could double-fire (issue #1424). sessionStart, count = syncEntireHook(sessionStart, sessionStartCmd, count) sessionEnd, count = syncEntireHook(sessionEnd, sessionEndCmd, count) beforeSubmitPrompt, count = syncEntireHook(beforeSubmitPrompt, beforeSubmitPromptCmd, count) stop, count = syncEntireHook(stop, stopCmd, count) preCompact, count = syncEntireHook(preCompact, preCompactCmd, count) subagentStart, count = syncEntireHook(subagentStart, subagentStartCmd, count) subagentStop, count = syncEntireHook(subagentStop, subagentEndCmd, count)
if count == 0 { return 0, nil 175 unmodified lines
// Helper functions for hook management
// syncEntireHook ensures entries contains exactly the given Entire hook command // for this hook type. If command is already present it is a no-op. Otherwise any // existing Entire hook (in any wrapper form) is removed before appending command, // so a changed wrapper form replaces the stale one rather than duplicating it. // Non-Entire entries are preserved. count is incremented when a change is made. func syncEntireHook(entries []CursorHookEntry, command string, count int) ([]CursorHookEntry, int) { if hookCommandExists(entries, command) { return entries, count } if hasEntireHook(entries) { entries = removeEntireHooks(entries) } return append(entries, CursorHookEntry{Command: command}), count + 1 }
func hookCommandExists(entries []CursorHookEntry, command string) bool { for _, entry := range entries { if entry.Command == command {
Mcmd/entire/cli/agent/cursor/hooks.go+43/-36
4 unmodified lines
5 6 7 8 9 10 11 52 unmodified lines
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
4 unmodified lines
"encoding/json" "os" "path/filepath" "strings" "testing"
"github.com/entireio/cli/cmd/entire/cli/agent" 52 unmodified lines
assertEntryCommand(t, hooksFile.Hooks.SubagentStop, agent.WrapProductionSilentHookCommand("entire hooks cursor subagent-stop")) }
// TestInstallHooks_WindowsProbeSuccessKeepsShWrappers verifies that on a // Windows host where a POSIX sh is runnable, Cursor keeps the sh-based wrappers // (parity with non-Windows). Mutates the shared probe, so no t.Parallel(). func TestInstallHooks_WindowsProbeSuccessKeepsShWrappers(t *testing.T) { t.Cleanup(agent.SetWindowsHookProbeForTesting("windows", func(context.Context, string) bool { return true // sh works }))
tempDir := t.TempDir() t.Chdir(tempDir)
ag := &CursorAgent{} if _, err := ag.InstallHooks(context.Background(), false, false); err != nil { t.Fatalf("InstallHooks() error = %v", err) }
hooksFile := readHooksFile(t, tempDir) assertEntryCommand(t, hooksFile.Hooks.SessionStart, agent.WrapProductionSilentHookCommand("entire hooks cursor session-start")) assertEntryCommand(t, hooksFile.Hooks.Stop, agent.WrapProductionSilentHookCommand("entire hooks cursor stop")) }
// TestInstallHooks_WindowsProbeFailureUsesCmdWrappers verifies that on a Windows // host with no runnable POSIX sh, Cursor installs the native cmd.exe wrappers so // hooks actually fire (issue #1424). Mutates the shared probe, so no t.Parallel(). func TestInstallHooks_WindowsProbeFailureUsesCmdWrappers(t *testing.T) { t.Cleanup(agent.SetWindowsHookProbeForTesting("windows", func(context.Context, string) bool { return false // no working sh }))
tempDir := t.TempDir() t.Chdir(tempDir)
ag := &CursorAgent{} if _, err := ag.InstallHooks(context.Background(), false, false); err != nil { t.Fatalf("InstallHooks() error = %v", err) }
hooksFile := readHooksFile(t, tempDir) assertEntryCommand(t, hooksFile.Hooks.SessionStart, agent.WrapWindowsProductionSilentHookCommand("entire hooks cursor session-start")) assertEntryCommand(t, hooksFile.Hooks.Stop, agent.WrapWindowsProductionSilentHookCommand("entire hooks cursor stop")) assertEntryCommand(t, hooksFile.Hooks.SubagentStop, agent.WrapWindowsProductionSilentHookCommand("entire hooks cursor subagent-stop")) }
// TestInstallHooks_WindowsProbeFlipMigratesCleanly verifies that when a host's // sh availability changes between installs, a non-force reinstall REPLACES the // stale sh-wrapped hooks with cmd.exe ones rather than leaving both (which would // double-fire). Mirrors the codex migration test. Mutates the shared probe, so // no t.Parallel(). func TestInstallHooks_WindowsProbeFlipMigratesCleanly(t *testing.T) { shWorks := true t.Cleanup(agent.SetWindowsHookProbeForTesting("windows", func(context.Context, string) bool { return shWorks }))
tempDir := t.TempDir() t.Chdir(tempDir) ag := &CursorAgent{}
// First install with a working sh → sh-based wrappers. if _, err := ag.InstallHooks(context.Background(), false, false); err != nil { t.Fatalf("first InstallHooks() error = %v", err) }
// sh stops working; reinstall WITHOUT force. shWorks = false if _, err := ag.InstallHooks(context.Background(), false, false); err != nil { t.Fatalf("second InstallHooks() error = %v", err) }
hooksFile := readHooksFile(t, tempDir) // Exactly one entry per type — the stale sh entry must be gone, not duplicated. if len(hooksFile.Hooks.Stop) != 1 { t.Errorf("Stop hooks = %d after wrapper migration, want 1 (no duplicate)", len(hooksFile.Hooks.Stop)) } if len(hooksFile.Hooks.SessionStart) != 1 { t.Errorf("SessionStart hooks = %d after wrapper migration, want 1 (no duplicate)", len(hooksFile.Hooks.SessionStart)) } assertEntryCommand(t, hooksFile.Hooks.Stop, agent.WrapWindowsProductionSilentHookCommand("entire hooks cursor stop"))
// No sh-based Entire wrapper may survive the migration. data, err := os.ReadFile(filepath.Join(tempDir, ".cursor", HooksFileName)) if err != nil { t.Fatalf("failed to read hooks file: %v", err) } if strings.Contains(string(data), "sh -c") || strings.Contains(string(data), "command -v entire") { t.Errorf("stale sh-based wrapper survived migration:\n%s", data) } }
func TestInstallHooks_Idempotent(t *testing.T) { tempDir := t.TempDir() t.Chdir(tempDir)
Mcmd/entire/cli/agent/cursor/hooks\_test.go+90
package cursor
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/entireio/cli/cmd/entire/cli/agent"
"github.com/entireio/cli/cmd/entire/cli/logging"
)
// Compile-time interface assertion.
var _ agent.SidecarImageProvider = (*CursorAgent)(nil)
// cursorChatsDirEnv overrides the base directory that holds Cursor's per-session
// SQLite blob stores. Used by tests and mock environments.
const cursorChatsDirEnv = "ENTIRE_TEST_CURSOR_CHATS_DIR"
const (
// maxStoreDBBytes bounds the work: a store.db larger than this is skipped
// (best-effort no-op). sqlite3's hex() output is ~2x the blob size and is
// buffered in memory, so this caps peak memory. A normal Cursor store holding
// screenshots is well under this.
maxStoreDBBytes = 64 << 20 // 64MB
// sqlite3Timeout bounds the sidecar read so a locked, huge, or malformed
// store.db can never hang the git commit / stop hook it runs inside.
sqlite3Timeout = 30 * time.Second
)
// storeDBBlobQuery selects the hex encoding of every blob whose leading bytes
// match a known image magic number (JPEG, PNG, GIF, or RIFF/WEBP). sqlite3's
// hex() returns uppercase, so the literals are uppercase.
const storeDBBlobQuery = "SELECT hex(data) FROM blobs WHERE " +
"substr(hex(data),1,6)='FFD8FF' OR " + // JPEG
"substr(hex(data),1,8)='89504E47' OR " + // PNG
"substr(hex(data),1,8)='47494638' OR " + // GIF
"(substr(hex(data),1,8)='52494646' AND substr(hex(data),17,8)='57454250');" // RIFF....WEBP
// SidecarImages captures images that Cursor stores outside the JSONL transcript.
// Cursor keeps pasted/generated images in a per-session SQLite blob store
// (~/.cursor/chats/<workspace>/<session>/store.db), not the transcript Entire
// condenses, so they would otherwise be lost from the checkpoint. This locates
// that store for the session, shells out to the sqlite3 binary to read the image
// blobs, and returns them as checkpoint assets.
//
// It is best-effort: when the store, the sqlite3 binary, or the expected schema
// is absent, or the store is too large, it returns no images and no error.
// sessionRef is the transcript path.
func (c *CursorAgent) SidecarImages(ctx context.Context, sessionRef string) ([]agent.CompactedTranscriptAsset, error) {
logCtx := logging.WithComponent(ctx, "agent.cursor")
sessionID := sessionIDFromTranscriptPath(sessionRef)
if sessionID == "" {
return nil, nil
}
dbPaths, err := findStoreDBs(sessionID)
if err != nil {
return nil, fmt.Errorf("locate cursor store.db: %w", err)
}
if len(dbPaths) == 0 {
return nil, nil // no sidecar store for this session
}
if !sqlite3Available() {
logging.Debug(logCtx, "sqlite3 not found; skipping cursor sidecar image capture")
return nil, nil
}
assets := make([]agent.CompactedTranscriptAsset, 0)
seen := make(map[string]struct{})
for _, dbPath := range dbPaths {
hexBlobs, err := readImageBlobs(logCtx, dbPath)
if err != nil {
return nil, fmt.Errorf("read cursor store.db blobs: %w", err)
}
for _, h := range hexBlobs {
data, err := hex.DecodeString(h)
if err != nil {
logging.Debug(logCtx, "skipping undecodable cursor blob", slog.String("error", err.Error()))
continue
}
if len(data) > agent.MaxChunkSize {
// A blob this large would become an unpushable git object; drop it.
logging.Debug(logCtx, "skipping oversized cursor image", slog.Int("bytes", len(data)))
continue
}
mediaType, ext := detectImageType(data)
if mediaType == "" {
continue // not an image after all
}
sum := sha256.Sum256(data)
name := fmt.Sprintf("img-%s.%s", hex.EncodeToString(sum[:16]), ext)
if _, dup := seen[name]; dup {
continue // identical image already captured
}
seen[name] = struct{}{}
assets = append(assets, agent.CompactedTranscriptAsset{
Name: name,
MediaType: mediaType,
Data: data,
})
}
}
if len(assets) > 0 {
logging.Debug(logCtx, "captured cursor sidecar images",
slog.Int("count", len(assets)), slog.String("session", sessionID))
}
return assets, nil
}
// sessionIDFromTranscriptPath extracts the Cursor session id from a transcript
// path. Both the nested (<id>/<id>.jsonl) and flat (<id>.jsonl) layouts name the
// file after the session id, so the base name without extension is the id.
// Returns "" for a path whose base resolves to "." or ".." (never a real id).
func sessionIDFromTranscriptPath(transcriptPath string) string {
if transcriptPath == "" {
return ""
}
base := filepath.Base(transcriptPath)
id := strings.TrimSuffix(base, filepath.Ext(base))
if id == "." || id == ".." {
return ""
}
return id
}
// findStoreDBs locates every SQLite blob store for a session. Cursor lays these
// out as <chats>/<workspace-hash>/<session-id>/store.db; the workspace hash is
// not derivable from the session id, so we enumerate workspaces and check each.
//
// Only the workspace level is globbed; the session id is joined as a LITERAL
// path component (checked with os.Stat), so glob metacharacters in the id can't
// widen the match to a different session's store. Returns all matches (a session
// id is a UUID, so normally exactly one) — callers union + dedup the images,
// which avoids silently dropping images when a session resolves under more than
// one workspace directory.
func findStoreDBs(sessionID string) ([]string, error) {
base := os.Getenv(cursorChatsDirEnv)
if base == "" {
home, err := os.UserHomeDir()
if err != nil {
return nil, fmt.Errorf("get home directory: %w", err)
}
base = filepath.Join(home, ".cursor", "chats")
}
workspaces, err := filepath.Glob(filepath.Join(base, "*"))
if err != nil {
return nil, fmt.Errorf("glob cursor workspaces: %w", err)
}
var dbs []string
for _, ws := range workspaces {
p := filepath.Join(ws, sessionID, "store.db")
if fileExists(p) {
dbs = append(dbs, p)
}
}
return dbs, nil
}
// readImageBlobs copies the store to a temp location (so a live Cursor session
// cannot lock or mutate it mid-read, and any WAL is applied) and shells out to
// sqlite3 to select image blobs as hex. Returns one hex string per image blob.
//
// Best-effort: a store larger than maxStoreDBBytes, or one whose schema is not
// the expected blobs(data) shape, returns (nil, nil) — an expected miss, not an
// error, so it never spams a warning on every checkpoint.
func readImageBlobs(ctx context.Context, dbPath string) ([]string, error) {
if info, err := os.Stat(dbPath); err == nil && info.Size() > maxStoreDBBytes {
logging.Debug(ctx, "cursor store.db too large; skipping sidecar capture",
slog.Int64("bytes", info.Size()))
return nil, nil
}
tmpDir, err := os.MkdirTemp("", "entire-cursor-store-")
if err != nil {
return nil, fmt.Errorf("create temp dir: %w", err)
}
defer func() { _ = os.RemoveAll(tmpDir) }()
tmpDB := filepath.Join(tmpDir, "store.db")
if err := copyFile(dbPath, tmpDB); err != nil {
return nil, fmt.Errorf("copy store.db: %w", err)
}
// Copy the WAL/SHM sidecars if present so committed-but-not-checkpointed
// pages are applied when sqlite3 opens the copy. Best-effort: a missing or
// uncopyable sidecar just means we read the main db as-is.
for _, suffix := range []string{"-wal", "-shm"} {
src := dbPath + suffix
if !fileExists(src) {
continue
}
if err := copyFile(src, tmpDB+suffix); err != nil {
logging.Debug(ctx, "skipping cursor store.db sidecar copy",
slog.String("file", src), slog.String("error", err.Error()))
}
}
cctx, cancel := context.WithTimeout(ctx, sqlite3Timeout)
defer cancel()
cmd := exec.CommandContext(cctx, "sqlite3", tmpDB, storeDBBlobQuery)
out, err := cmd.Output()
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
stderr := strings.TrimSpace(string(exitErr.Stderr))
if isSchemaMismatch(stderr) {
logging.Debug(ctx, "cursor store.db schema not recognized; skipping",
slog.String("detail", stderr))
return nil, nil
}
return nil, fmt.Errorf("sqlite3 query failed: %w: %s", err, stderr)
}
return nil, fmt.Errorf("sqlite3 query failed: %w", err)
}
var blobs []string
for _, line := range strings.Split(string(out), "\n") {
if line = strings.TrimSpace(line); line != "" {
blobs = append(blobs, line)
}
}
return blobs, nil
}
// isSchemaMismatch reports whether a sqlite3 error is a benign schema-shape
// mismatch (a store version whose blob table/columns differ from what the query
// assumes) rather than a genuine failure. Such stores are treated as an expected
// no-op, not an error.
func isSchemaMismatch(stderr string) bool {
s := strings.ToLower(stderr)
return strings.Contains(s, "no such table") || strings.Contains(s, "no such column")
}
// detectImageType returns the media type and file extension for known image
// magic bytes, or ("", "") when the bytes are not a recognized image.
func detectImageType(data []byte) (mediaType, ext string) {
switch {
case len(data) >= 8 && string(data[:8]) == "\x89PNG\r\n\x1a\n":
return "image/png", "png"
case len(data) >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF:
return "image/jpeg", "jpg"
case len(data) >= 6 && string(data[:6]) == "GIF89a", len(data) >= 6 && string(data[:6]) == "GIF87a":
return "image/gif", "gif"
case len(data) >= 12 && string(data[:4]) == "RIFF" && string(data[8:12]) == "WEBP":
return "image/webp", "webp"
default:
return "", ""
}
}
func sqlite3Available() bool {
_, err := exec.LookPath("sqlite3")
return err == nil
}
func fileExists(path string) bool {
// path is built from a workspace glob result plus a filepath.Base-sanitized
// session id (separators stripped, "."/".." rejected), so no traversal.
info, err := os.Stat(path) //nolint:gosec // G703 false positive: path is sanitized (see above)
return err == nil && !info.IsDir()
}
func copyFile(src, dst string) error {
in, err := os.Open(src) //nolint:gosec // path is an internal, non-user-controlled store location
if err != nil {
return fmt.Errorf("open source: %w", err)
}
defer func() { _ = in.Close() }()
out, err := os.Create(dst) //nolint:gosec // dst is a temp file we created
if err != nil {
return fmt.Errorf("create destination: %w", err)
}
if _, err := io.Copy(out, in); err != nil {
_ = out.Close()
return fmt.Errorf("copy contents: %w", err)
}
if err := out.Close(); err != nil {
return fmt.Errorf("close destination: %w", err)
}
return nil
}
Acmd/entire/cli/agent/cursor/images.go+295
package cursor
import ( "context" "encoding/hex" "os" "os/exec" "path/filepath" "strings" "testing" )
// pngBytes returns a minimal byte slice with a valid PNG magic header, padded so // it is unambiguously an image. func pngBytes(payload string) []byte { return append([]byte("\x89PNG\r\n\x1a\n"), []byte(payload)...) }
func jpegBytes(payload string) []byte { return append([]byte{0xFF, 0xD8, 0xFF, 0xE0}, []byte(payload)...) }
// webpBytes returns a minimal RIFF/WEBP container (RIFF....WEBP) padded past the // header so the store query's magic-byte filter matches it. func webpBytes(payload string) []byte { return append([]byte("RIFF____WEBP"), []byte(payload)...) }
// buildStoreDB writes a Cursor-style store.db at path with a blobs(id, data) // table populated from the given blobs. It shells out to sqlite3 (the same // binary the code under test uses). func buildStoreDB(t *testing.T, path string, blobs map[string][]byte) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatalf("mkdir: %v", err) } var sb strings.Builder sb.WriteString("CREATE TABLE blobs(id TEXT PRIMARY KEY, data BLOB);\n") for id, data := range blobs { sb.WriteString("INSERT INTO blobs(id,data) VALUES('" + id + "', x'" + hex.EncodeToString(data) + "');\n") } cmd := exec.CommandContext(context.Background(), "sqlite3", path, sb.String()) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("build store.db: %v: %s", err, out) } }
// setupChatsDir creates
func requireSqlite3(t *testing.T) { t.Helper() if _, err := exec.LookPath("sqlite3"); err != nil { t.Skip("sqlite3 not installed; skipping cursor store.db test") } }
func TestSidecarImages_CapturesImageBlobs(t *testing.T) { requireSqlite3(t)
img := pngBytes("cursor-sidecar-image-payload-aaaaaaaaaaaaaaaaaaaa") transcriptPath := setupChatsDir(t, "sess-img", map[string][]byte{ "img1": img, "txt1": []byte("this is just some message text, not an image at all"), })
assets, err := (&CursorAgent{}).SidecarImages(context.Background(), transcriptPath)
if err != nil {
t.Fatalf("SidecarImages: %v", err)
}
if len(assets) != 1 {
t.Fatalf("expected 1 image asset, got %d", len(assets))
}
if assets[0].MediaType != "image/png" {
t.Errorf("media type = %q, want image/png", assets[0].MediaType)
}
if string(assets[0].Data) != string(img) {
t.Error("captured bytes do not match the stored image blob")
}
if !strings.HasPrefix(assets[0].Name, "img-") || !strings.HasSuffix(assets[0].Name, ".png") {
t.Errorf("asset name %q is not img-
func TestSidecarImages_MixedImageTypes(t *testing.T) { requireSqlite3(t)
transcriptPath := setupChatsDir(t, "sess-mixed", map[string][]byte{ "a": pngBytes(strings.Repeat("p", 40)), "b": jpegBytes(strings.Repeat("j", 40)), "c": []byte("not an image"), })
assets, err := (&CursorAgent{}).SidecarImages(context.Background(), transcriptPath) if err != nil { t.Fatalf("SidecarImages: %v", err) } if len(assets) != 2 { t.Fatalf("expected 2 image assets, got %d", len(assets)) } types := map[string]bool{} for _, a := range assets { types[a.MediaType] = true } if !types["image/png"] || !types["image/jpeg"] { t.Errorf("expected png and jpeg, got %v", types) } }
func TestSidecarImages_CapturesWebp(t *testing.T) { requireSqlite3(t)
img := webpBytes(strings.Repeat("w", 40)) transcriptPath := setupChatsDir(t, "sess-webp", map[string][]byte{"w1": img})
assets, err := (&CursorAgent{}).SidecarImages(context.Background(), transcriptPath) if err != nil { t.Fatalf("SidecarImages: %v", err) } if len(assets) != 1 { t.Fatalf("expected 1 webp asset (end-to-end through the SQL magic filter), got %d", len(assets)) } if assets[0].MediaType != "image/webp" || !strings.HasSuffix(assets[0].Name, ".webp") { t.Errorf("got %q / %q, want image/webp / *.webp", assets[0].MediaType, assets[0].Name) } }
// A store whose schema is not the expected blobs(data) shape (a future/older // Cursor version) must be a silent no-op, not an error that would log a warning // on every checkpoint. func TestSidecarImages_UnknownSchemaIsNoOp(t *testing.T) { requireSqlite3(t)
chats := t.TempDir()
dbPath := filepath.Join(chats, "workspace-hash", "sess-schema", "store.db")
if err := os.MkdirAll(filepath.Dir(dbPath), 0o755); err != nil {
t.Fatalf("mkdir: %v", err)
}
// No blobs table at all — a different schema shape.
cmd := exec.CommandContext(context.Background(), "sqlite3", dbPath,
"CREATE TABLE messages(id TEXT, body TEXT); INSERT INTO messages VALUES('a','hi');")
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("build store.db: %v: %s", err, out)
}
t.Setenv(cursorChatsDirEnv, chats)
transcriptPath := filepath.Join(t.TempDir(), "sess-schema.jsonl")
assets, err := (&CursorAgent{}).SidecarImages(context.Background(), transcriptPath) if err != nil { t.Fatalf("unexpected error for unrecognized schema (should be a silent no-op): %v", err) } if len(assets) != 0 { t.Fatalf("expected no assets from an unrecognized schema, got %d", len(assets)) } }
func TestSidecarImages_DedupsIdenticalImages(t *testing.T) { requireSqlite3(t)
img := pngBytes(strings.Repeat("dedup", 20)) transcriptPath := setupChatsDir(t, "sess-dup", map[string][]byte{ "one": img, "two": img, // identical content under a different blob id })
assets, err := (&CursorAgent{}).SidecarImages(context.Background(), transcriptPath) if err != nil { t.Fatalf("SidecarImages: %v", err) } if len(assets) != 1 { t.Fatalf("expected identical images deduped to 1, got %d", len(assets)) } }
func TestSidecarImages_TextOnlyStoreReturnsNothing(t *testing.T) { requireSqlite3(t)
transcriptPath := setupChatsDir(t, "sess-text", map[string][]byte{ "m1": []byte("first message"), "m2": []byte("second message"), })
assets, err := (&CursorAgent{}).SidecarImages(context.Background(), transcriptPath) if err != nil { t.Fatalf("SidecarImages: %v", err) } if len(assets) != 0 { t.Fatalf("expected no assets from a text-only store, got %d", len(assets)) } }
func TestSidecarImages_NoStoreDBIsNoOp(t *testing.T) { // Point at an empty chats dir: no store.db for any session. t.Setenv(cursorChatsDirEnv, t.TempDir()) transcriptPath := filepath.Join(t.TempDir(), "missing-session.jsonl")
assets, err := (&CursorAgent{}).SidecarImages(context.Background(), transcriptPath) if err != nil { t.Fatalf("SidecarImages: %v", err) } if assets != nil { t.Fatalf("expected nil assets when no store.db exists, got %d", len(assets)) } }
func TestSidecarImages_EmptySessionRefIsNoOp(t *testing.T) { assets, err := (&CursorAgent{}).SidecarImages(context.Background(), "") if err != nil { t.Fatalf("SidecarImages: %v", err) } if assets != nil { t.Fatal("expected nil assets for empty session ref") } }
func TestSessionIDFromTranscriptPath(t *testing.T) { t.Parallel() cases := map[string]string{ "/home/u/.cursor/projects/p/agent-transcripts/abc-123.jsonl": "abc-123", "/home/u/.cursor/projects/p/agent-transcripts/abc-123/abc-123.jsonl": "abc-123", "": "", "bare.jsonl": "bare", } for in, want := range cases { if got := sessionIDFromTranscriptPath(in); got != want { t.Errorf("sessionIDFromTranscriptPath(%q) = %q, want %q", in, got, want) } } }
func TestDetectImageType(t *testing.T) { t.Parallel() cases := []struct { name string data []byte mediaType string ext string }{ {"png", pngBytes("x"), "image/png", "png"}, {"jpeg", jpegBytes("x"), "image/jpeg", "jpg"}, {"gif89", []byte("GIF89a...."), "image/gif", "gif"}, {"gif87", []byte("GIF87a...."), "image/gif", "gif"}, {"webp", append([]byte("RIFF____WEBP"), []byte("data")...), "image/webp", "webp"}, {"text", []byte("hello world not an image"), "", ""}, {"tooShort", []byte{0x89, 0x50}, "", ""}, } for _, tc := range cases { mt, ext := detectImageType(tc.data) if mt != tc.mediaType || ext != tc.ext { t.Errorf("%s: detectImageType = (%q,%q), want (%q,%q)", tc.name, mt, ext, tc.mediaType, tc.ext) } } }
Acmd/entire/cli/agent/cursor/images\_test.go+263
30 unmodified lines
31 32 33 34 34 35 36 37 38 39 40 41 42 4 unmodified lines
47 48 49 50 51 52 53 54 55 56 48 57 58 59 60 1 unmodified line
62 63 64 56 65 66 67 68 69 70 62 63 71 72 73 74 75 76 77 78 79 80 68 81 82 83 84
30 unmodified lines
{Name: "/security-review", Desc: "Scan git diff for security issues"},
{Name: "/simplify", Desc: "Review recent changes for code quality"},
},
"codex": {{Name: "/review", Desc: "Review current changes and find issues"}},
// Codex has no binary-bundled review command usable from codex exec:
// built-in slash commands like /review only fire in the interactive TUI,
// not when piped through exec. Codex's review skills (code-reviewer,
// review-swarm, …) live on disk and are surfaced by DiscoverReviewSkills in
// $name form, so there are no curated built-ins to hardcode here.
"codex": {},
"gemini": {},
}
4 unmodified lines
// Install commands below are placeholders until marketplace URLs are pinned.
// Tests do not assert on Message text — only on ProvidesAny semantics — so
// prose revisions do not break the suite.
//
// Messages must stay backtick-free: the picker renders them through huh, which
// treats the text as markdown and mangles backtick-wrapped code spans in the
// terminal. Use plain text / colons to set off commands instead.
var installHints = map[string][]InstallHint{
"claude-code": {
{
Message: "Install pr-review-toolkit via claude plugin install entireio/pr-review-toolkit",
Message: "Install pr-review-toolkit: claude plugin install entireio/pr-review-toolkit",
ProvidesAny: []string{
"/pr-review-toolkit:review-pr",
"/pr-review-toolkit:code-reviewer",
1 unmodified line
},
},
{
Message: "Install test-auditor via the superpowers plugin",
Message: "Install test-auditor via the superpowers plugin",
ProvidesAny: []string{"/test-auditor"},
},
},
"codex": {
{
Message: "Install codex-review-pack via codex plugins add <url>",
ProvidesAny: []string{"/codex:adversarial-review"},
Message: "Install codex-review-pack: codex plugins add gemini-code-review via gemini extensions install <url>",
Message: "Install gemini-code-review: gemini extensions install
Mcmd/entire/cli/agent/skilldiscovery/registry.go+19/-6
11 unmodified lines
12 13 14 15 16 17 16 17 18 19 20 21 22 49 unmodified lines
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
11 unmodified lines
if len(claude) != 3 {
t.Fatalf("claude-code built-ins: got %d entries, want 3", len(claude))
}
// Codex has no binary-bundled review command usable from codex exec;
// its review skills are discovered on disk in $name form instead.
codex := skilldiscovery.CuratedBuiltinsFor("codex")
if len(codex) != 1 || codex[0].Name != "/review" {
t.Errorf("codex built-ins: got %+v, want 1x /review", codex)
if len(codex) != 0 {
t.Errorf("codex built-ins: got %+v, want 0 (discovery-driven)", codex)
}
gemini := skilldiscovery.CuratedBuiltinsFor("gemini")
if len(gemini) != 0 {
49 unmodified lines
t.Error("unknown agent should not be eligible") } }
// TestActiveInstallHintsFor_CodexFingerprintMatchesDollarFormDiscovery pins
// the suppression fingerprint to the invocation form codex discovery actually
// produces: DiscoverReviewSkills emits $plugin:name, so a slash-form
// ProvidesAny entry could never intersect the discovered set and the hint
// would show forever even with the plugin installed.
func TestActiveInstallHintsFor_CodexFingerprintMatchesDollarFormDiscovery(t *testing.T) {
t.Parallel()
discovered := map[string]struct{}{"$codex:adversarial-review": {}}
if hints := skilldiscovery.ActiveInstallHintsFor("codex", discovered); len(hints) != 0 {
t.Fatalf("codex hint not suppressed by $-form discovery; got %d hints: %+v", len(hints), hints)
}
}
Mcmd/entire/cli/agent/skilldiscovery/registry\_test.go+17/-2
package skilldiscovery
import (
"context"
"errors"
"log/slog"
"os"
"path/filepath"
"sort"
"strings"
"golang.org/x/mod/semver"
"github.com/entireio/cli/cmd/entire/cli/agent"
"github.com/entireio/cli/cmd/entire/cli/logging"
)
// InvocationForm builds an agent's invocation string for a discovered skill.
// The only thing that differs between agents is the prefix and namespace
// joiner: Claude Code uses slash form (`/name`, `/plugin:name`), codex uses
// dollar form (`$name`, `$plugin:name`) — the literal token a user types to
// invoke the skill in that CLI. Discovery emits Name already in this form so
// downstream prompt composition stays agent-agnostic and joins verbatim.
type InvocationForm func(name, pluginName string) string
// SlashForm is Claude Code's invocation syntax: "/name" or "/plugin:name".
func SlashForm(name, pluginName string) string {
if pluginName == "" {
return "/" + name
}
return "/" + pluginName + ":" + name
}
// DollarForm is codex's invocation syntax: "$name" or "$plugin:name". This is
// the explicit "use this skill" token from codex's own injected skills
// catalog ("name a skill with $SkillName or plain text").
func DollarForm(name, pluginName string) string {
if pluginName == "" {
return "$" + name
}
return "$" + pluginName + ":" + name
}
// DedupeByInvocation collapses entries sharing an invocation name, keeping the
// first occurrence. Plugins can ship a skill and a same-named wrapper that
// forwards to it; scan order decides which wins.
func DedupeByInvocation(in []agent.DiscoveredSkill) []agent.DiscoveredSkill {
if len(in) < 2 {
return in
}
seen := make(map[string]struct{}, len(in))
out := make([]agent.DiscoveredSkill, 0, len(in))
for _, s := range in {
if _, dup := seen[s.Name]; dup {
continue
}
seen[s.Name] = struct{}{}
out = append(out, s)
}
return out
}
// ScanPluginCache walks <root>/<marketplace>/<plugin>/<version>/ and invokes
// scanVersion once per plugin, for the single version directory chosen by
// PickLatestVersion. The callback receives the chosen version root and the
// plugin name (used as the invocation namespace). Both Claude Code and codex
// use this same market/plugin/version cache layout; they differ only in which
// subdirectories under the version root they scan and their invocation form.
func ScanPluginCache(ctx context.Context, root string, scanVersion func(versionRoot, pluginName string) []agent.DiscoveredSkill) []agent.DiscoveredSkill {
entries, err := os.ReadDir(root)
if err != nil {
logging.Debug(ctx, "skill discovery: plugin cache unreadable",
slog.String("root", root), slog.String("error", err.Error()))
return nil
}
var found []agent.DiscoveredSkill
for _, marketEntry := range entries {
if !marketEntry.IsDir() {
continue
}
marketRoot := filepath.Join(root, marketEntry.Name())
pluginEntries, err := os.ReadDir(marketRoot)
if err != nil {
continue
}
for _, pluginEntry := range pluginEntries {
if !pluginEntry.IsDir() {
continue
}
pluginName := pluginEntry.Name()
pluginRoot := filepath.Join(marketRoot, pluginName)
versionEntries, err := os.ReadDir(pluginRoot)
if err != nil {
continue
}
versionDir, ok := PickLatestVersion(versionEntries)
if !ok {
continue
}
found = append(found, scanVersion(filepath.Join(pluginRoot, versionDir), pluginName)...)
}
}
return found
}
// PickLatestVersion returns the "newest" version directory name among entries:
//
// - If any entry parses as semver (with or without a leading "v"), pick the
// highest semver; non-semver entries are ignored when a semver exists.
// - Otherwise fall back to the lexicographic max of all directory names.
// This handles the "unknown" sentinel some plugins ship, and the opaque
// content-hash version dirs codex plugins use (e.g. "fef63ecf").
//
// Returns ("", false) if no usable directory entry exists.
func PickLatestVersion(entries []os.DirEntry) (string, bool) {
var dirs []string
for _, e := range entries {
if e.IsDir() {
dirs = append(dirs, e.Name())
}
}
if len(dirs) == 0 {
return "", false
}
var semverDirs []string
for _, d := range dirs {
if semver.IsValid(semverWithV(d)) {
semverDirs = append(semverDirs, d)
}
}
if len(semverDirs) > 0 {
sort.Slice(semverDirs, func(i, j int) bool {
return semver.Compare(semverWithV(semverDirs[i]), semverWithV(semverDirs[j])) > 0
})
return semverDirs[0], true
}
sort.Sort(sort.Reverse(sort.StringSlice(dirs)))
return dirs[0], true
}
// semverWithV ensures a version string has the "v" prefix golang.org/x/mod/semver
// requires. Plugin version dirs are usually bare (e.g. "0.1.0").
func semverWithV(s string) string {
if strings.HasPrefix(s, "v") {
return s
}
return "v" + s
}
// ScanSkillsDir reads each <dir>/<name>/SKILL.md, parses its frontmatter, and
// emits a DiscoveredSkill (in invoke's form) when Matches() returns true.
// pluginName is the namespace ("" for un-namespaced user skills). Missing dirs
// yield nil — discovery is best-effort.
func ScanSkillsDir(ctx context.Context, dir, pluginName string, invoke InvocationForm) []agent.DiscoveredSkill {
entries, err := os.ReadDir(dir)
if err != nil {
return nil
}
var found []agent.DiscoveredSkill
for _, skillEntry := range entries {
if !skillEntry.IsDir() {
continue
}
skillFile := filepath.Join(dir, skillEntry.Name(), "SKILL.md")
data, err := os.ReadFile(skillFile) //nolint:gosec // G304: skillFile is built from a ReadDir walk under HOME, not user input
if err != nil {
continue
}
name, description, parseErr := ParseSkillFrontmatter(data)
if parseErr != nil {
logging.Debug(ctx, "skill discovery: skipping malformed SKILL.md",
slog.String("path", skillFile), slog.String("error", parseErr.Error()))
continue
}
if name == "" {
name = skillEntry.Name()
}
invocation := invoke(name, pluginName)
if !Matches(invocation, description) {
continue
}
found = append(found, agent.DiscoveredSkill{
Name: invocation,
Description: description,
SourcePath: skillFile,
})
}
return found
}
// ScanFlatMarkdownDir reads *.md files directly under dir (no nesting), parses
// their frontmatter for `description:`, and derives the invocation name from
// the filename (minus .md). Used by Claude Code for plugin/user commands and
// agents, whose frontmatter has no `name:` field. README.md is skipped.
func ScanFlatMarkdownDir(ctx context.Context, dir, pluginName string, invoke InvocationForm) []agent.DiscoveredSkill {
entries, err := os.ReadDir(dir)
if err != nil {
return nil
}
var found []agent.DiscoveredSkill
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".md") {
continue
}
baseName := strings.TrimSuffix(entry.Name(), ".md")
if strings.EqualFold(baseName, "README") {
continue
}
filePath := filepath.Join(dir, entry.Name())
data, err := os.ReadFile(filePath) //nolint:gosec // G304: filePath is built from a ReadDir walk under HOME, not user input
if err != nil {
continue
}
_, description, parseErr := ParseSkillFrontmatter(data)
if parseErr != nil {
logging.Debug(ctx, "skill discovery: skipping malformed command/agent",
slog.String("path", filePath), slog.String("error", parseErr.Error()))
continue
}
invocation := invoke(baseName, pluginName)
if !Matches(invocation, description) {
continue
}
found = append(found, agent.DiscoveredSkill{
Name: invocation,
Description: description,
SourcePath: filePath,
})
}
return found
}
// ParseSkillFrontmatter extracts `name:` and `description:` from a minimal YAML
// frontmatter block — the tiny subset these SKILL.md / command / agent files
// use. Surrounding double-quotes are trimmed so `description: "foo"` returns
// `foo`.
func ParseSkillFrontmatter(data []byte) (name, description string, err error) {
s := string(data)
if !strings.HasPrefix(s, "---\n") && !strings.HasPrefix(s, "---\r\n") {
return "", "", errors.New("no frontmatter delimiter")
}
body := strings.TrimPrefix(strings.TrimPrefix(s, "---\r\n"), "---\n")
end := strings.Index(body, "\n---")
if end < 0 {
return "", "", errors.New("no closing frontmatter delimiter")
}
for _, line := range strings.Split(body[:end], "\n") {
line = strings.TrimSpace(line)
switch {
case strings.HasPrefix(line, "name:"):
name = strings.Trim(strings.TrimSpace(strings.TrimPrefix(line, "name:")), `"`)
case strings.HasPrefix(line, "description:"):
description = strings.Trim(strings.TrimSpace(strings.TrimPrefix(line, "description:")), `"`)
}
}
return name, description, nil
}
Acmd/entire/cli/agent/skilldiscovery/scan.go+257
21 unmodified lines
22
23
24
25
26
27
28
21 unmodified lines
CheckpointInfo = apicheckpoint.CheckpointInfo
SessionContent = apicheckpoint.SessionContent
SessionFilePaths = apicheckpoint.SessionFilePaths
TranscriptAsset = apicheckpoint.TranscriptAsset
SessionMetrics = apicheckpoint.SessionMetrics
Summary = apicheckpoint.Summary
LearningsSummary = apicheckpoint.LearningsSummary
Mcmd/entire/cli/checkpoint/aliases.go+1
3 unmodified lines
4
5
6
7
8
9
10
17 unmodified lines
28
29
30
31
32
33
34
363 unmodified lines
398
399
400
399
401
402
403
404
405
403
404
405
406
407
408
409
410
411
412
413
414
406
407
408
409
410
411
412
413
414
415
416
417
416
418
419
418
419
420
420
421
422
423
424
425
426
427
428
429
422
423
424
425
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
258 unmodified lines
708
709
710
711
712
713
714
715
716
717
718
719
720
781 unmodified lines
1502
1503
1504
1477
1505
1506
1507
1508
332 unmodified lines
1841
1842
1843
1816
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
17 unmodified lines
1980
1981
1982
1840
1983
1984
1985
1986
14 unmodified lines
2001
2002
2003
1861
2004
2005
2006
2007
2008
2009
1867
2010
2011
2012
2013
16 unmodified lines
2030
2031
2032
1890
2033
2034
2035
2036
28 unmodified lines
2065
2066
2067
1925
2068
2069
2070
1928
2071
2072
2073
2074
391 unmodified lines
2466
2467
2468
2326
2469
2470
2471
2472
3 unmodified lines
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
17 unmodified lines
"github.com/entireio/cli/cmd/entire/cli/settings"
"github.com/entireio/cli/cmd/entire/cli/trailers"
transcriptcompact "github.com/entireio/cli/cmd/entire/cli/transcript/compact"
"github.com/entireio/cli/cmd/entire/cli/transcript/imageextract"
"github.com/entireio/cli/cmd/entire/cli/validation"
"github.com/entireio/cli/cmd/entire/cli/vercelconfig"
"github.com/entireio/cli/cmd/entire/cli/versioninfo"
363 unmodified lines
agentType = sessionMeta.Agent
}
}
if err := s.replaceTranscript(ctx, opts.Transcript, agentType, startLine, opts.PrecomputedBlobs, sessionDir, entries); err != nil {
rewrote, err := s.replaceTranscript(ctx, opts.Transcript, agentType, startLine, opts.PrecomputedBlobs, sessionDir, entries)
if err != nil {
return plumbing.ZeroHash, fmt.Errorf("failed to replace transcript: %w", err)
}
// Keep the root metadata.json compact_transcript pointer consistent with
// the finalized tree. replaceTranscript may have written transcript.jsonl
// that the initial write lacked (e.g. compaction was skipped then and
// succeeds now), so re-derive the pointer from the tree entry and rewrite
// the root summary when it changed.
compactPath := ""
if _, ok := entries[checkpointSubtreePath(sessionDir, paths.CompactTranscriptFileName)]; ok {
compactPath = "/" + checkpointSubtreePath(sessionDir, paths.CompactTranscriptFileName)
}
if checkpointSummary.Sessions[sessionIndex].CompactTranscript != compactPath {
checkpointSummary.Sessions[sessionIndex].CompactTranscript = compactPath
summaryJSON, err := jsonutil.MarshalIndentWithNewline(checkpointSummary, "", " ")
// Only touch assets and the root pointers when the transcript was actually
// rewritten. If replaceTranscript short-circuited (identical content), the
// stored transcript, compact, and assets are all unchanged and already
// consistent — clearing/rewriting assets here would strip the blobs a
// still-present placeholder depends on, leaving a dangling placeholder.
if rewrote {
// Keep the externalized image assets consistent with the replaced
// transcript: write the new set (clearing any stale ones), so a finalize
// that re-externalizes matches its placeholders and one that produces an
// inline transcript leaves no orphaned blobs.
manifestPath, err := s.writeAssetsForBackfill(opts, sessionDir, entries)
if err != nil {
return plumbing.ZeroHash, fmt.Errorf("failed to marshal checkpoint summary: %w", err)
return plumbing.ZeroHash, fmt.Errorf("failed to write assets: %w", err)
}
summaryHash, err := CreateBlobFromContent(s.repo, summaryJSON)
if err != nil {
return plumbing.ZeroHash, fmt.Errorf("failed to create checkpoint summary blob: %w", err)
// Keep the root metadata.json compact_transcript and assets_manifest
// pointers consistent with the finalized tree. replaceTranscript may have
// written transcript.jsonl that the initial write lacked (e.g. compaction
// was skipped then and succeeds now), so re-derive both pointers from the
// tree and rewrite the root summary once when either changed.
compactPath := ""
if _, ok := entries[checkpointSubtreePath(sessionDir, paths.CompactTranscriptFileName)]; ok {
compactPath = "/" + checkpointSubtreePath(sessionDir, paths.CompactTranscriptFileName)
}
entries[rootMetadataPath] = object.TreeEntry{
Name: rootMetadataPath,
Mode: filemode.Regular,
Hash: summaryHash,
sess := &checkpointSummary.Sessions[sessionIndex]
if sess.CompactTranscript != compactPath || sess.AssetsManifest != manifestPath {
sess.CompactTranscript = compactPath
sess.AssetsManifest = manifestPath
summaryJSON, err := jsonutil.MarshalIndentWithNewline(checkpointSummary, "", " ")
if err != nil {
return plumbing.ZeroHash, fmt.Errorf("failed to marshal checkpoint summary: %w", err)
}
summaryHash, err := CreateBlobFromContent(s.repo, summaryJSON)
if err != nil {
return plumbing.ZeroHash, fmt.Errorf("failed to create checkpoint summary blob: %w", err)
}
entries[rootMetadataPath] = object.TreeEntry{
Name: rootMetadataPath,
Mode: filemode.Regular,
Hash: summaryHash,
}
}
}
}
258 unmodified lines
}
}
// Write externalized image assets (raw binary blobs + manifest), when present.
manifestPath, err := s.writeAssets(opts.Assets, sessionDir, entries)
if err != nil {
return filePaths, err
}
filePaths.AssetsManifest = manifestPath
// Write prompts via the 7-layer pipeline. OPF runs only in the
// pre-push rewrite path (manual_commit_opf_rewrite.go).
if len(opts.Prompts) > 0 {
781 unmodified lines
// Read transcript (auto-fetches blobs if needed)
if transcript, transcriptErr := readTranscriptFromTree(ctx, sessionTree, agentType); transcriptErr == nil && transcript != nil {
result.Transcript = transcript
result.Transcript = reinjectAssets(sessionTree, agentType, transcript)
result.TranscriptBlobHashes = transcriptBlobHashesFromTreeEntries(sessionTree.RawEntries())
}
332 unmodified lines
// reuse precomputed blobs: each checkpoint in a turn shares the full
// transcript but has its own start offset, so the compact content differs per
// checkpoint.
func (s *treeWriter) replaceTranscript(ctx context.Context, transcript redact.RedactedBytes, agentType types.AgentType, startLine int, precomputed *PrecomputedTranscriptBlobs, sessionDir string, entries map[string]object.TreeEntry) error {
// assetManifestEntry describes one externalized asset in assets/manifest.json.
// Size and SHA256 are descriptive metadata for external tooling and audits; they
// are not used on reinject (git content-addresses the blobs, which already
// guarantees their integrity on read).
type assetManifestEntry struct {
Name string `json:"name"`
MediaType string `json:"media_type,omitempty"`
Size int `json:"size"`
SHA256 string `json:"sha256"`
}
// writeAssetsForBackfill writes the update's assets, but preserves any
// already-stored assets when the update carries none AND the update opts into
// preservation (UpdateOptions.PreserveAssetsWhenEmpty). This guards a best-effort
// sidecar capture (e.g. Cursor's sqlite3 store read) that transiently yields
// nothing at finalize from wiping images a prior CondenseSession successfully
// stored: leaving the existing assets/ subtree untouched is strictly safer than
// clearing it. Returns the (possibly pre-existing) manifest path.
func (s *treeWriter) writeAssetsForBackfill(opts UpdateOptions, sessionDir string, entries map[string]object.TreeEntry) (string, error) {
if len(opts.Assets) == 0 && opts.PreserveAssetsWhenEmpty {
manifestKey := checkpointSubtreePath(sessionDir, paths.AssetsManifestFile)
if _, ok := entries[manifestKey]; ok {
return "/" + manifestKey, nil
}
return "", nil
}
return s.writeAssets(opts.Assets, sessionDir, entries)
}
// writeAssets stores each externalized transcript asset as a raw binary blob
// under the session's assets/ folder, plus an assets/manifest.json index, in the
// same tree. Returns the manifest path ("" when there are no assets). git
// content-addresses the blobs, so identical images dedupe across checkpoints.
//
// It first clears any assets already present under the session's assets/ folder,
// so a re-write (backfill/finalize) replaces rather than accumulates, and an
// empty asset set leaves no orphaned blobs behind a now-inline transcript.
func (s *treeWriter) writeAssets(assets []TranscriptAsset, sessionDir string, entries map[string]object.TreeEntry) (string, error) {
assetsPrefix := checkpointSubtreePath(sessionDir, paths.AssetsDirName) + "/"
for key := range entries {
if strings.HasPrefix(key, assetsPrefix) {
delete(entries, key)
}
}
if len(assets) == 0 {
return "", nil
}
manifest := struct {
Version int `json:"version"`
Assets []assetManifestEntry `json:"assets"`
}{Version: 1}
for _, a := range assets {
blobHash, err := CreateBlobFromContent(s.repo, a.Data)
if err != nil {
return "", err
}
p := checkpointSubtreePath(sessionDir, paths.AssetsDirName, a.Name)
entries[p] = object.TreeEntry{Name: p, Mode: filemode.Regular, Hash: blobHash}
sum := sha256.Sum256(a.Data)
manifest.Assets = append(manifest.Assets, assetManifestEntry{
Name: a.Name, MediaType: a.MediaType, Size: len(a.Data), SHA256: hex.EncodeToString(sum[:]),
})
}
manifestJSON, err := jsonutil.MarshalIndentWithNewline(manifest, "", " ")
if err != nil {
return "", fmt.Errorf("marshal assets manifest: %w", err)
}
manifestHash, err := CreateBlobFromContent(s.repo, manifestJSON)
if err != nil {
return "", err
}
mp := checkpointSubtreePath(sessionDir, paths.AssetsManifestFile)
entries[mp] = object.TreeEntry{Name: mp, Mode: filemode.Regular, Hash: manifestHash}
return "/" + mp, nil
}
// reinjectAssets restores externalized images into a transcript on read, so the
// returned bytes match what was stored. Best-effort and gated on placeholder
// presence, not on any config flag: an asset it can't load is left as a
// placeholder rather than failing the read.
func reinjectAssets(sessionTree *FetchingTree, agentType types.AgentType, transcript []byte) []byte {
if !imageextract.HasPlaceholders(transcript) {
return transcript
}
codec := imageextract.CodecFor(agentType)
if codec == nil {
return transcript
}
// No blob-integrity check is needed here: git content-addresses every asset
// blob, so a corrupt/truncated fetch fails object verification and Contents()
// errors out (leaving the placeholder). The manifest's sha256 is external
// metadata, not a second integrity gate — and since writeAssets derives both
// the blob and the sha256 from the same bytes, they can never disagree.
out, err := codec.ReinjectImages(transcript, func(name string) (agent.CompactedTranscriptAsset, bool) {
f, ferr := sessionTree.File(paths.AssetsDir + name)
if ferr != nil {
return agent.CompactedTranscriptAsset{}, false
}
content, cerr := f.Contents()
if cerr != nil {
return agent.CompactedTranscriptAsset{}, false
}
return agent.CompactedTranscriptAsset{Name: name, Data: []byte(content)}, true
})
if err != nil {
return transcript
}
return out
}
// replaceTranscript rewrites the session transcript (full.jsonl chunks +
// content_hash + compact) in entries. It reports whether it actually rewrote:
// false means the content-hash matched and everything was left as-is (the
// caller must then leave coupled artifacts like assets untouched too, so they
// stay consistent with the unchanged transcript).
func (s *treeWriter) replaceTranscript(ctx context.Context, transcript redact.RedactedBytes, agentType types.AgentType, startLine int, precomputed *PrecomputedTranscriptBlobs, sessionDir string, entries map[string]object.TreeEntry) (bool, error) {
// Ignore precompute if invariants are violated — fall back to fresh chunking.
if precomputed != nil && !precomputed.IsUsable() {
precomputed = nil
17 unmodified lines
existingHash, readErr := io.ReadAll(rdr)
_ = rdr.Close()
if readErr == nil && string(existingHash) == newContentHash {
return nil
return false, nil
}
}
}
14 unmodified lines
} else {
chunks, err := chunkTranscript(ctx, transcript.Bytes(), agentType)
if err != nil {
return fmt.Errorf("failed to chunk transcript: %w", err)
return false, fmt.Errorf("failed to chunk transcript: %w", err)
}
chunkHashes = make([]plumbing.Hash, len(chunks))
for i, chunk := range chunks {
blobHash, err := CreateBlobFromContent(s.repo, chunk)
if err != nil {
return fmt.Errorf("failed to create transcript blob: %w", err)
return false, fmt.Errorf("failed to create transcript blob: %w", err)
}
chunkHashes[i] = blobHash
}
16 unmodified lines
} else {
h, err := CreateBlobFromContent(s.repo, []byte(newContentHash))
if err != nil {
return fmt.Errorf("failed to create content hash blob: %w", err)
return false, fmt.Errorf("failed to create content hash blob: %w", err)
}
hashBlob = h
}
28 unmodified lines
// transcript.jsonl: record the new boundary when one was produced, or clear
// it (nil) when the compact transcript was dropped above.
if err := s.setCompactTranscriptStart(sessionDir, compactStart, entries); err != nil {
return fmt.Errorf("failed to update compact transcript start: %w", err)
return false, fmt.Errorf("failed to update compact transcript start: %w", err)
}
return nil
return true, nil
}
// PrecomputeTranscriptBlobs chunks the given transcript and writes each chunk
391 unmodified lines
}
defer r.Close()
sig, err := signer.Sign(r)
sig, err := signer.Sign(ctx, r)
if err != nil {
logging.Warn(ctx, "failed to sign commit", slog.String("error", err.Error()))
return
Mcmd/entire/cli/checkpoint/persistent.go+173/-30
package checkpoint
import ( "context" "encoding/base64" "strings" "testing"
"github.com/entireio/cli/cmd/entire/cli/agent" "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" "github.com/entireio/cli/cmd/entire/cli/paths" "github.com/entireio/cli/cmd/entire/cli/transcript/imageextract" "github.com/entireio/cli/redact" )
// claudeTranscriptWithImage returns a Claude Code JSONL transcript whose first
// line embeds an inline base64 image, followed by an ordinary assistant reply.
// It returns the raw (image-inline) bytes plus the base64 string so tests can
// assert on both the extracted and reinjected forms.
func claudeTranscriptWithImage(t *testing.T) (raw []byte, b64 string) {
t.Helper()
b64 = base64.StdEncoding.EncodeToString([]byte("\x89PNG\r\n\x1a\nround-trip-fixture-bytes-long-enough-to-be-externalized\x00\x01\x02\x03"))
lines := []string{
{"type":"user","uuid":"u1","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":[ +
{"type":"text","text":"look at this"}, +
{"type":"image","source":{"type":"base64","media_type":"image/png","data":" + b64 + "}} +
]}},
{"type":"assistant","uuid":"a1","timestamp":"2026-01-01T00:00:01Z","message":{"id":"msg_1","role":"assistant","content":[{"type":"text","text":"nice screenshot"}],"usage":{"input_tokens":5,"output_tokens":7}}},
}
return []byte(strings.Join(lines, "\n") + "\n"), b64
}
// claudeImagePayload builds a one-image Claude Code transcript from a distinct
// payload, returning the raw inline bytes and the base64 string.
func claudeImagePayload(t *testing.T, payload string) (raw []byte, b64 string) {
t.Helper()
b64 = base64.StdEncoding.EncodeToString([]byte(payload + "-padded-so-the-base64-clears-the-externalize-threshold"))
line := {"type":"user","message":{"role":"user","content":[ +
{"type":"text","text":"look"}, +
{"type":"image","source":{"type":"base64","media_type":"image/png","data":" + b64 + "}} +
]}}
return []byte(line + "\n"), b64
}
// externalize runs the codec the way the condensation/finalize paths do. func externalize(t *testing.T, raw []byte) (rewritten []byte, assets []TranscriptAsset) { t.Helper() codec := imageextract.CodecFor(agent.AgentTypeClaudeCode) rw, ex, err := codec.ExtractImages(raw) if err != nil { t.Fatalf("ExtractImages: %v", err) } out := make([]TranscriptAsset, len(ex)) for i, a := range ex { out[i] = TranscriptAsset{Name: a.Name, MediaType: a.MediaType, Data: a.Data} } return rw, out }
// TestAssets_BackfillReExternalizesAndReplacesAssets is the S1 regression: the // stop-hook finalize path (backfillTranscript / SessionTranscript) must persist a // newly-externalized transcript and its assets, replacing the condense-time // assets rather than orphaning them or re-inlining the images. func TestAssets_BackfillReExternalizesAndReplacesAssets(t *testing.T) { t.Parallel() repo, _ := setupTestRepo(t) store := NewGitStore(repo, DefaultV1Refs()) cpID := id.MustCheckpointID("a55e70000010") sessionPath := cpID.Path() + "/0/"
// Condense: first (mid-turn) externalized write. rawA, _ := claudeImagePayload(t, "condense-image") rewrittenA, assetsA := externalize(t, rawA) if len(assetsA) != 1 { t.Fatalf("want 1 asset from condense, got %d", len(assetsA)) } if err := store.Write(context.Background(), Session{ CheckpointID: cpID, SessionID: "s-backfill", Strategy: "manual-commit", Transcript: redact.AlreadyRedacted(rewrittenA), Assets: assetsA, Agent: agent.AgentTypeClaudeCode, AuthorName: "T", AuthorEmail: "t@t.com", }); err != nil { t.Fatalf("condense Write: %v", err) }
// Finalize: backfill with a different, longer externalized transcript. rawB, b64B := claudeImagePayload(t, "finalize-different-image-with-more-bytes") rewrittenB, assetsB := externalize(t, rawB) if err := store.Write(context.Background(), SessionTranscript{ CheckpointID: cpID, SessionID: "s-backfill", Transcript: redact.AlreadyRedacted(rewrittenB), Assets: assetsB, Agent: agent.AgentTypeClaudeCode, }); err != nil { t.Fatalf("backfill Write: %v", err) }
// Stored full.jsonl carries B's placeholder, not raw base64; the old asset // blob is gone and B's is present. stored, ok := readBranchFile(t, store, sessionPath+paths.TranscriptFileName) if !ok { t.Fatal("full.jsonl missing") } if strings.Contains(stored, b64B) { t.Error("stored transcript still contains raw base64 after backfill") } if !strings.Contains(stored, "entire-asset:assets/"+assetsB[0].Name) { t.Error("stored transcript missing backfilled placeholder") } if _, ok := readBranchFile(t, store, sessionPath+paths.AssetsDir+assetsA[0].Name); ok { t.Error("stale condense-time asset blob was not cleared on backfill") } if _, ok := readBranchFile(t, store, sessionPath+paths.AssetsDir+assetsB[0].Name); !ok { t.Error("backfilled asset blob missing") }
// Manifest pointer updated; restore round-trips to B byte-exact. summary := readSummaryFromBranch(t, repo, cpID) if summary.Sessions[0].AssetsManifest != "/"+sessionPath+paths.AssetsManifestFile { t.Errorf("assets_manifest pointer = %q, want set", summary.Sessions[0].AssetsManifest) } content, err := store.ReadSessionContent(context.Background(), cpID, 0) if err != nil { t.Fatalf("ReadSessionContent: %v", err) } if string(content.Transcript) != string(rawB) { t.Fatalf("backfill round-trip not byte-exact:\n got: %s\nwant: %s", content.Transcript, rawB) } }
// TestAssets_BackfillIdenticalTranscriptKeepsAssets is the short-circuit // regression: a backfill whose transcript is byte-identical to what is stored // (so replaceTranscript short-circuits) must NOT clear the assets, even if it is // called with empty Assets — the still-present placeholder must keep round-tripping. func TestAssets_BackfillIdenticalTranscriptKeepsAssets(t *testing.T) { t.Parallel() repo, _ := setupTestRepo(t) store := NewGitStore(repo, DefaultV1Refs()) cpID := id.MustCheckpointID("a55e70000012") sessionPath := cpID.Path() + "/0/"
rawA, _ := claudeImagePayload(t, "shortcircuit-image") rewrittenA, assetsA := externalize(t, rawA) if err := store.Write(context.Background(), Session{ CheckpointID: cpID, SessionID: "s1", Strategy: "manual-commit", Transcript: redact.AlreadyRedacted(rewrittenA), Assets: assetsA, Agent: agent.AgentTypeClaudeCode, AuthorName: "T", AuthorEmail: "t@t.com", }); err != nil { t.Fatalf("first Write: %v", err) }
// Backfill with the identical transcript (short-circuit) and NO assets. if err := store.Write(context.Background(), SessionTranscript{ CheckpointID: cpID, SessionID: "s1", Transcript: redact.AlreadyRedacted(rewrittenA), Agent: agent.AgentTypeClaudeCode, }); err != nil { t.Fatalf("second Write: %v", err) }
// Assets survive; the placeholder still round-trips to the original image. if _, ok := readBranchFile(t, store, sessionPath+paths.AssetsDir+assetsA[0].Name); !ok { t.Error("asset blob was cleared by an identical-transcript backfill") } content, err := store.ReadSessionContent(context.Background(), cpID, 0) if err != nil { t.Fatalf("ReadSessionContent: %v", err) } if strings.Contains(string(content.Transcript), "entire-asset:assets/") { t.Errorf("dangling placeholder after identical-transcript backfill: %s", content.Transcript) } if string(content.Transcript) != string(rawA) { t.Errorf("restore did not round-trip after identical-transcript backfill") } }
// TestAssets_BackfillInlineClearsStaleAssets covers the flag-off-at-finalize case: // a backfill with an inline transcript and no assets must clear the assets stored // at condense time (no orphans) and clear the manifest pointer. func TestAssets_BackfillInlineClearsStaleAssets(t *testing.T) { t.Parallel() repo, _ := setupTestRepo(t) store := NewGitStore(repo, DefaultV1Refs()) cpID := id.MustCheckpointID("a55e70000011") sessionPath := cpID.Path() + "/0/"
rawA, _ := claudeImagePayload(t, "condense-image") rewrittenA, assetsA := externalize(t, rawA) if err := store.Write(context.Background(), Session{ CheckpointID: cpID, SessionID: "s-inline", Strategy: "manual-commit", Transcript: redact.AlreadyRedacted(rewrittenA), Assets: assetsA, Agent: agent.AgentTypeClaudeCode, AuthorName: "T", AuthorEmail: "t@t.com", }); err != nil { t.Fatalf("condense Write: %v", err) }
// Backfill inline (as if externalization were off at finalize): no Assets. rawB, b64B := claudeImagePayload(t, "condense-image") // same content, inline if err := store.Write(context.Background(), SessionTranscript{ CheckpointID: cpID, SessionID: "s-inline", Transcript: redact.AlreadyRedacted(rawB), Agent: agent.AgentTypeClaudeCode, }); err != nil { t.Fatalf("backfill Write: %v", err) }
if _, ok := readBranchFile(t, store, sessionPath+paths.AssetsDir+assetsA[0].Name); ok { t.Error("stale asset blob not cleared when backfill went inline") } if _, ok := readBranchFile(t, store, sessionPath+paths.AssetsManifestFile); ok { t.Error("manifest not cleared when backfill went inline") } summary := readSummaryFromBranch(t, repo, cpID) if summary.Sessions[0].AssetsManifest != "" { t.Errorf("assets_manifest pointer = %q, want empty", summary.Sessions[0].AssetsManifest) } stored, _ := readBranchFile(t, store, sessionPath+paths.TranscriptFileName) if !strings.Contains(stored, b64B) { t.Error("inline backfill should store raw base64") } content, err := store.ReadSessionContent(context.Background(), cpID, 0) if err != nil { t.Fatalf("ReadSessionContent: %v", err) } if string(content.Transcript) != string(rawB) { t.Errorf("inline backfill restore mismatch") } }
// TestAssets_StoreRestoreRoundTrip is the end-to-end contract for image // externalization at the persistent-store layer: a Claude Code transcript with an // inline base64 image is externalized before the write, stored as a placeholder // plus an assets/ blob and manifest, and reinjected byte-exactly on read. func TestAssets_StoreRestoreRoundTrip(t *testing.T) { t.Parallel() repo, _ := setupTestRepo(t) store := NewGitStore(repo, DefaultV1Refs()) cpID := id.MustCheckpointID("a55e70000001")
raw, b64 := claudeTranscriptWithImage(t)
// Externalize exactly as the condensation path does, then store the // placeholder-bearing transcript with its assets. codec := imageextract.CodecFor(agent.AgentTypeClaudeCode) if codec == nil { t.Fatal("expected a Claude Code image codec") } rewritten, assets, err := codec.ExtractImages(raw) if err != nil { t.Fatalf("ExtractImages() error = %v", err) } if len(assets) != 1 { t.Fatalf("expected 1 externalized asset, got %d", len(assets)) } writeAssets := make([]TranscriptAsset, len(assets)) for i, a := range assets { writeAssets[i] = TranscriptAsset{Name: a.Name, MediaType: a.MediaType, Data: a.Data} }
if err := store.Write(context.Background(), Session{ CheckpointID: cpID, SessionID: "session-assets-001", Strategy: "manual-commit", Transcript: redact.AlreadyRedacted(rewritten), Assets: writeAssets, Prompts: []string{"look at this"}, Agent: agent.AgentTypeClaudeCode, AuthorName: "Test", AuthorEmail: "test@test.com", }); err != nil { t.Fatalf("Write() error = %v", err) }
sessionPath := cpID.Path() + "/0/"
// Stored full.jsonl carries the placeholder, not the raw base64. stored, ok := readBranchFile(t, store, sessionPath+paths.TranscriptFileName) if !ok { t.Fatal("full.jsonl missing from checkpoint tree") } if strings.Contains(stored, b64) { t.Error("stored full.jsonl still contains raw base64 image data") } if !strings.Contains(stored, "entire-asset:assets/"+assets[0].Name) { t.Errorf("stored full.jsonl missing placeholder for %s", assets[0].Name) }
// The asset blob and manifest are written under assets/.
if _, ok := readBranchFile(t, store, sessionPath+paths.AssetsDir+assets[0].Name); !ok {
t.Errorf("asset blob %s missing from checkpoint tree", assets[0].Name)
}
manifest, ok := readBranchFile(t, store, sessionPath+paths.AssetsManifestFile)
if !ok {
t.Fatal("assets/manifest.json missing from checkpoint tree")
}
if !strings.Contains(manifest, assets[0].Name) || !strings.Contains(manifest, "media_type": "image/png") {
t.Errorf("manifest missing expected asset entry: %s", manifest)
}
// Session metadata points at the manifest. summary := readSummaryFromBranch(t, repo, cpID) if len(summary.Sessions) != 1 { t.Fatalf("session count = %d, want 1", len(summary.Sessions)) } wantManifest := "/" + sessionPath + paths.AssetsManifestFile if summary.Sessions[0].AssetsManifest != wantManifest { t.Errorf("sessions[0].assets_manifest = %q, want %q", summary.Sessions[0].AssetsManifest, wantManifest) }
// Read back: the image is reinjected byte-exactly, reproducing the original. content, err := store.ReadSessionContent(context.Background(), cpID, 0) if err != nil { t.Fatalf("ReadSessionContent() error = %v", err) } if strings.Contains(string(content.Transcript), "entire-asset:assets/") { t.Error("restored transcript still contains a placeholder") } if !strings.Contains(string(content.Transcript), b64) { t.Error("restored transcript missing reinjected base64 image") } if string(content.Transcript) != string(raw) { t.Fatalf("round-trip not byte-exact:\n got: %s\nwant: %s", content.Transcript, raw) } }
// TestAssets_NoExternalizationWritesNoManifest confirms the default (no assets) // path is unchanged: no assets/ folder and an empty AssetsManifest pointer. func TestAssets_NoExternalizationWritesNoManifest(t *testing.T) { t.Parallel() repo, _ := setupTestRepo(t) store := NewGitStore(repo, DefaultV1Refs()) cpID := id.MustCheckpointID("a55e70000002")
if err := store.Write(context.Background(), Session{ CheckpointID: cpID, SessionID: "session-assets-002", Strategy: "manual-commit", Transcript: redact.AlreadyRedacted(claudeStyleTranscript()), Prompts: []string{"hello one"}, Agent: agent.AgentTypeClaudeCode, AuthorName: "Test", AuthorEmail: "test@test.com", }); err != nil { t.Fatalf("Write() error = %v", err) }
sessionPath := cpID.Path() + "/0/" if _, ok := readBranchFile(t, store, sessionPath+paths.AssetsManifestFile); ok { t.Error("assets/manifest.json should not be written when there are no assets") } summary := readSummaryFromBranch(t, repo, cpID) if len(summary.Sessions) != 1 { t.Fatalf("session count = %d, want 1", len(summary.Sessions)) } if summary.Sessions[0].AssetsManifest != "" { t.Errorf("sessions[0].assets_manifest = %q, want empty", summary.Sessions[0].AssetsManifest) } }
Acmd/entire/cli/checkpoint/persistent\_assets\_test.go+356
20 unmodified lines
21 22 23 24 24 25 26 27
20 unmodified lines
err error }
func (s *stubSigner) Sign(_ io.Reader) ([]byte, error) { func (s *stubSigner) Sign(_ context.Context, _ io.Reader) ([]byte, error) { return s.sig, s.err }
Mcmd/entire/cli/checkpoint/persistent\_signing\_test.go+1/-1
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
//go:build integration
package integration
import ( "context" "encoding/base64" "encoding/json" "os" "path/filepath" "strings" "testing"
"github.com/entireio/cli/cmd/entire/cli/checkpoint" "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" "github.com/entireio/cli/cmd/entire/cli/gitrepo" "github.com/entireio/cli/cmd/entire/cli/paths" )
// TestCodexImageExternalization_FullHookFlow is the Codex end-to-end proof: it // drives the real Codex hook binary (user-prompt-submit -> apply_patch // post-tool-use -> mid-turn commit condensation -> stop finalize) on a Codex // rollout transcript that embeds an image as a data-URI, with externalization // enabled via settings.local.json. It then asserts the actual // entire/checkpoints/v1 ref stores a placeholder (not the raw base64) that // survives Codex's SanitizePortableTranscript, writes the asset blob + manifest, // and that ReadSessionContent reinjects the image byte-exactly. func TestCodexImageExternalization_FullHookFlow(t *testing.T) { env := NewFeatureBranchEnv(t)
localSettings := filepath.Join(env.RepoDir, ".entire", "settings.local.json")
if err := os.WriteFile(localSettings, []byte({"redaction":{"externalize_images":true}}), 0o644); err != nil {
t.Fatalf("write settings.local.json: %v", err)
}
// A real, minimal PNG padded past the externalization length threshold. imgBytes := []byte("\x89PNG\r\n\x1a\n" + strings.Repeat("codex-real-e2e-image-payload-", 4)) b64 := base64.StdEncoding.EncodeToString(imgBytes)
sessionID := "codex-image-e2e" transcriptPath := filepath.Join(env.RepoDir, ".entire", "tmp", "codex-rollout.jsonl")
// A Codex rollout: session meta, then a user message with an inline image
// data-URI (the confirmed real format), then an assistant reply.
rollout := strings.Join([]string{
{"timestamp":"2026-01-01T00:00:00Z","type":"session_meta","payload":{"id":" + sessionID + ","cwd":" + env.RepoDir + "}},
{"timestamp":"2026-01-01T00:00:01Z","type":"response_item","payload":{"type":"message","role":"user","content":[ +
{"type":"input_text","text":"add feature.txt and look at this screenshot"}, +
{"type":"input_image","image_url":"data:image/png;base64, + b64 + "} +
]}},
{"timestamp":"2026-01-01T00:00:02Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"done"}]}},
}, "\n") + "\n"
if err := os.MkdirAll(filepath.Dir(transcriptPath), 0o755); err != nil {
t.Fatalf("mkdir: %v", err)
}
if err := os.WriteFile(transcriptPath, []byte(rollout), 0o644); err != nil {
t.Fatalf("write rollout: %v", err)
}
runner := NewCodexHookRunner(env.RepoDir, t) hook := func(name string, extra map[string]any) { t.Helper() in := map[string]any{ "session_id": sessionID, "transcript_path": transcriptPath, "cwd": env.RepoDir, "model": "gpt-5", "permission_mode": "default", } for k, v := range extra { in[k] = v } b, err := json.Marshal(in) if err != nil { t.Fatalf("marshal %s input: %v", name, err) } if err := runner.runCodexHook(name, b); err != nil { t.Fatalf("codex hook %s: %v", name, err) } }
// Turn start (creates the Codex session), then a file-mutating tool use so the // commit has attributable content. hook("user-prompt-submit", map[string]any{"prompt": "add feature.txt and look at this screenshot", "hook_event_name": "UserPromptSubmit"}) patch := "*** Begin Patch\n*** Add File: feature.txt\n+hi\n*** End Patch\n" hook("post-tool-use", map[string]any{ "hook_event_name": "PostToolUse", "tool_name": "apply_patch", "tool_use_id": "call_1", "tool_input": map[string]string{"command": patch}, "tool_response": "Success.", })
// Mid-turn commit -> post-commit condensation externalizes; stop -> finalize. env.WriteFile("feature.txt", "hi\n") env.GitCommitWithShadowHooks("add feature.txt", "feature.txt") hook("stop", map[string]any{"hook_event_name": "Stop"})
if !env.BranchExists(paths.MetadataBranchName) { t.Fatal("entire/checkpoints/v1 should exist after Codex condensation") } cpID := env.GetLatestCheckpointIDFromHistory() if cpID == "" { t.Fatal("no checkpoint id in history") } sessionPath := ShardedCheckpointPath(cpID) + "/0/"
full, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.TranscriptFileName) if !ok { t.Fatalf("full.jsonl missing at %s", sessionPath) } if strings.Contains(full, b64) { t.Error("stored full.jsonl still contains the raw base64 image (externalization did not persist)") } if !strings.Contains(full, "entire-asset:assets/") { t.Error("stored full.jsonl has no image placeholder") } if !strings.Contains(full, "data:image/png;base64,entire-asset:assets/") { t.Error("expected the placeholder inside the data-URI (prefix preserved)") } if _, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.AssetsManifestFile); !ok { t.Error("assets/manifest.json missing") }
// Restore reinjects the image byte-exactly. repo, err := gitrepo.OpenPath(env.RepoDir) if err != nil { t.Fatalf("open repo: %v", err) } defer repo.Close() stores, err := checkpoint.Open(context.Background(), repo, checkpoint.OpenOptions{}) if err != nil { t.Fatalf("open stores: %v", err) } checkpointID, err := id.NewCheckpointID(cpID) if err != nil { t.Fatalf("parse checkpoint id: %v", err) } content, err := stores.Persistent.ReadSessionContent(context.Background(), checkpointID, 0) if err != nil { t.Fatalf("ReadSessionContent: %v", err) } if strings.Contains(string(content.Transcript), "entire-asset:assets/") { t.Error("restored transcript still has a placeholder (reinjection failed)") } if !strings.Contains(string(content.Transcript), b64) { t.Error("restored transcript is missing the reinjected base64 image") } }
Acmd/entire/cli/integration\_test/codex\_image\_externalize\_test.go+146
//go:build integration
package integration
import (
"context"
"encoding/hex"
"encoding/json"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/entireio/cli/cmd/entire/cli/agent"
"github.com/entireio/cli/cmd/entire/cli/paths"
"github.com/stretchr/testify/require"
)
// TestCursorImageExternalization_SidecarCapture is the Cursor end-to-end proof.
// Cursor keeps pasted images in a per-session SQLite blob store (store.db), NOT
// the JSONL transcript Entire condenses, so the transcript codec used for Claude
// and Codex cannot reach them. This drives the real Cursor hook flow (session
// start -> before-submit-prompt -> mid-turn commit condensation -> stop finalize)
// with a store.db that holds an image, externalization enabled, and asserts the
// checkpoint captures the image as an asset (blob + manifest) even though the
// transcript never contained it and carries no placeholder.
func TestCursorImageExternalization_SidecarCapture(t *testing.T) {
t.Parallel()
if _, err := exec.LookPath("sqlite3"); err != nil {
t.Skip("sqlite3 not installed; skipping cursor store.db capture test")
}
env := NewFeatureBranchEnv(t)
env.InitEntireWithAgent(agent.AgentNameCursor)
localSettings := filepath.Join(env.RepoDir, ".entire", "settings.local.json")
require.NoError(t, os.WriteFile(localSettings, []byte(`{"redaction":{"externalize_images":true}}`), 0o644))
cursorProjectDir := t.TempDir()
if resolved, err := filepath.EvalSymlinks(cursorProjectDir); err == nil {
cursorProjectDir = resolved
}
chatsDir := t.TempDir()
// Propagate the cursor project + chats dirs to BOTH the stop-hook subprocess
// (via cliEnv) and the git-hook condensation subprocess (via gitHookEnv).
env.ExtraEnv = append(env.ExtraEnv,
"ENTIRE_TEST_CURSOR_PROJECT_DIR="+cursorProjectDir,
"ENTIRE_TEST_CURSOR_CHATS_DIR="+chatsDir,
)
const conversationID = "cursor-image-e2e"
// Transcript is text-only — Cursor never inlines the image here.
transcriptDir := filepath.Join(cursorProjectDir, conversationID)
require.NoError(t, os.MkdirAll(transcriptDir, 0o755))
transcriptPath := filepath.Join(transcriptDir, conversationID+".jsonl")
require.NoError(t, os.WriteFile(transcriptPath,
[]byte(`{"type":"user","text":"look at this screenshot and add a feature"}`+"\n"+
`{"type":"assistant","text":"done"}`+"\n"), 0o600))
// The image lives only in Cursor's SQLite store, keyed by conversation id at
// <chats>/<workspace-hash>/<conversationID>/store.db.
img := append([]byte("\x89PNG\r\n\x1a\n"), []byte(strings.Repeat("cursor-real-sidecar-image-payload-", 8))...)
storeDBPath := filepath.Join(chatsDir, "workspace-hash", conversationID, "store.db")
require.NoError(t, os.MkdirAll(filepath.Dir(storeDBPath), 0o755))
buildCursorStoreDB(t, storeDBPath, map[string][]byte{
"img-blob": img,
"text-blob": []byte("this is a message body, not an image, and should be ignored"),
})
runCursorHook(t, env, cursorProjectDir, "session-start", map[string]any{
"conversation_id": conversationID,
"transcript_path": transcriptPath,
"model": "cursor-default",
})
runCursorHook(t, env, cursorProjectDir, "before-submit-prompt", map[string]any{
"conversation_id": conversationID,
"transcript_path": transcriptPath,
"prompt": "look at this screenshot and add a feature",
})
env.WriteFile("feature.go", "package main\n// new feature\n")
// Stop ends the turn; the commit's condensation then creates the checkpoint
// and captures the sidecar image (Cursor has no mid-turn tool hooks, so the
// checkpoint is born at commit time, not updated by a later finalize).
runCursorHook(t, env, cursorProjectDir, "stop", map[string]any{
"conversation_id": conversationID,
"transcript_path": transcriptPath,
"model": "cursor-default",
"loop_count": 1,
})
env.GitCommitWithShadowHooks("Add feature", "feature.go")
cpID := env.TryGetLatestCheckpointID()
require.NotEmpty(t, cpID, "expected a condensed checkpoint after commit")
sessionPath := ShardedCheckpointPath(cpID) + "/0/"
// The transcript is untouched: no placeholder, no image bytes (there were none).
full, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.TranscriptFileName)
require.True(t, ok, "full.jsonl missing at %s", sessionPath)
require.NotContains(t, full, "entire-asset:", "cursor transcript must not carry a placeholder")
// The manifest indexes the captured image.
manifest, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.AssetsManifestFile)
require.True(t, ok, "assets/manifest.json missing — sidecar image was not captured")
var manifestDoc struct {
Version int `json:"version"`
Assets []struct {
Name string `json:"name"`
MediaType string `json:"media_type"`
} `json:"assets"`
}
require.NoError(t, json.Unmarshal([]byte(manifest), &manifestDoc))
require.Len(t, manifestDoc.Assets, 1, "expected exactly one captured image in the manifest")
entry := manifestDoc.Assets[0]
require.Equal(t, "image/png", entry.MediaType)
require.True(t, strings.HasPrefix(entry.Name, "img-") && strings.HasSuffix(entry.Name, ".png"),
"asset name %q is not img-<hash>.png", entry.Name)
// The asset blob is stored byte-exact.
blob, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.AssetsDir+entry.Name)
require.True(t, ok, "asset blob %s missing", entry.Name)
require.Equal(t, string(img), blob, "stored asset bytes differ from the store.db image")
}
// TestCursorImageExternalization_SurvivesFinalizeRewrite guards against the
// finalize-wipe regression: when a mid-turn commit's condensation captures a
// Cursor sidecar image and a later stop finalizes the checkpoint with a grown
// (rewritten) transcript, writeAssets clears the whole assets/ folder before
// re-writing. If finalize omitted the sidecar images from its asset set, the
// captured image would be permanently dropped. This drives that exact sequence
// and asserts the image survives finalize.
func TestCursorImageExternalization_SurvivesFinalizeRewrite(t *testing.T) {
t.Parallel()
if _, err := exec.LookPath("sqlite3"); err != nil {
t.Skip("sqlite3 not installed; skipping cursor store.db capture test")
}
env := NewFeatureBranchEnv(t)
env.InitEntireWithAgent(agent.AgentNameCursor)
cursorProjectDir := t.TempDir()
if resolved, err := filepath.EvalSymlinks(cursorProjectDir); err == nil {
cursorProjectDir = resolved
}
chatsDir := t.TempDir()
env.ExtraEnv = append(env.ExtraEnv,
"ENTIRE_TEST_CURSOR_PROJECT_DIR="+cursorProjectDir,
"ENTIRE_TEST_CURSOR_CHATS_DIR="+chatsDir,
)
const conversationID = "cursor-finalize-wipe"
transcriptDir := filepath.Join(cursorProjectDir, conversationID)
require.NoError(t, os.MkdirAll(transcriptDir, 0o755))
transcriptPath := filepath.Join(transcriptDir, conversationID+".jsonl")
// v1: what condensation stores at the mid-turn commit.
require.NoError(t, os.WriteFile(transcriptPath,
[]byte(`{"type":"user","text":"look at this screenshot and add a feature"}`+"\n"), 0o600))
img := append([]byte("\x89PNG\r\n\x1a\n"), []byte(strings.Repeat("cursor-finalize-image-payload-", 8))...)
storeDBPath := filepath.Join(chatsDir, "workspace-hash", conversationID, "store.db")
require.NoError(t, os.MkdirAll(filepath.Dir(storeDBPath), 0o755))
buildCursorStoreDB(t, storeDBPath, map[string][]byte{"img-blob": img})
// Mid-turn commit while the session is ACTIVE: condensation creates the
// checkpoint + captures the sidecar image, and PostCommit records it in
// TurnCheckpointIDs so the later stop finalize runs over it. AsAgent takes the
// no-TTY active-session fast path (a human mid-turn commit path differs).
env.WriteFile("feature.go", "package main\n// new feature\n")
env.GitCommitWithShadowHooksAsAgent("Add feature", "feature.go")
cpID := env.TryGetLatestCheckpointID()
require.NotEmpty(t, cpID, "expected a condensed checkpoint after the mid-turn commit")
sessionPath := ShardedCheckpointPath(cpID) + "/0/"
_, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.AssetsManifestFile)
require.True(t, ok, "PRECONDITION: condensation should have captured the sidecar image")
// Grow the transcript so the finalized full transcript differs from what
// condensation stored -> replaceTranscript reports rewrote==true, the exact
// condition under which finalize rewrites (and previously wiped) the assets.
require.NoError(t, os.WriteFile(transcriptPath,
[]byte(`{"type":"user","text":"look at this screenshot and add a feature"}`+"\n"+
`{"type":"assistant","text":"added the feature"}`+"\n"), 0o600))
runCursorHook(t, env, cursorProjectDir, "stop", map[string]any{
"conversation_id": conversationID, "transcript_path": transcriptPath,
"model": "cursor-default", "loop_count": 1,
})
// Regression assertion: the image asset must STILL be present after finalize.
manifest, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.AssetsManifestFile)
require.True(t, ok, "assets/manifest.json missing after finalize — sidecar image was wiped")
var manifestDoc struct {
Assets []struct {
Name string `json:"name"`
} `json:"assets"`
}
require.NoError(t, json.Unmarshal([]byte(manifest), &manifestDoc))
require.Len(t, manifestDoc.Assets, 1, "expected the captured image to survive finalize")
blob, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.AssetsDir+manifestDoc.Assets[0].Name)
require.True(t, ok, "asset blob missing after finalize")
require.Equal(t, string(img), blob, "asset bytes changed after finalize")
}
// TestCursorImageExternalization_PreservesImagesOnFinalizeCaptureMiss guards the
// best-effort edge: condensation captures a Cursor image, but the sidecar
// re-capture at finalize yields nothing (e.g. sqlite3 locked/timed out, or — as
// simulated here — the store.db is momentarily gone). A rewriting finalize must
// then PRESERVE the images condensation stored rather than clearing the assets/
// folder for the now-empty asset set.
func TestCursorImageExternalization_PreservesImagesOnFinalizeCaptureMiss(t *testing.T) {
t.Parallel()
if _, err := exec.LookPath("sqlite3"); err != nil {
t.Skip("sqlite3 not installed; skipping cursor store.db capture test")
}
env := NewFeatureBranchEnv(t)
env.InitEntireWithAgent(agent.AgentNameCursor)
const conversationID = "cursor-finalize-miss"
transcriptDir := filepath.Join(cursorProjectDir, conversationID)
require.NoError(t, os.MkdirAll(transcriptDir, 0o755))
transcriptPath := filepath.Join(transcriptDir, conversationID+".jsonl")
require.NoError(t, os.WriteFile(transcriptPath,
[]byte(`{"type":"user","text":"look at this screenshot and add a feature"}`+"\n"), 0o600))
img := append([]byte("\x89PNG\r\n\x1a\n"), []byte(strings.Repeat("cursor-preserve-image-payload-", 8))...)
storeDBPath := filepath.Join(chatsDir, "workspace-hash", conversationID, "store.db")
require.NoError(t, os.MkdirAll(filepath.Dir(storeDBPath), 0o755))
buildCursorStoreDB(t, storeDBPath, map[string][]byte{"img-blob": img})
// Mid-turn commit: condensation captures the image into the checkpoint.
env.WriteFile("feature.go", "package main\n// new feature\n")
env.GitCommitWithShadowHooksAsAgent("Add feature", "feature.go")
// Grow the transcript so finalize rewrites (rewrote=true), AND remove the
// store.db so the finalize re-capture yields nothing — the transient-miss case.
require.NoError(t, os.WriteFile(transcriptPath,
[]byte(`{"type":"user","text":"look at this screenshot and add a feature"}`+"\n"+
`{"type":"assistant","text":"added the feature"}`+"\n"), 0o600))
require.NoError(t, os.Remove(storeDBPath))
// The image captured at condensation must survive the finalize rewrite even
// though the re-capture found nothing.
manifest, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.AssetsManifestFile)
require.True(t, ok, "assets/manifest.json missing after finalize — sidecar image was wiped on a capture miss")
var manifestDoc struct {
Assets []struct {
Name string `json:"name"`
} `json:"assets"`
}
require.NoError(t, json.Unmarshal([]byte(manifest), &manifestDoc))
require.Len(t, manifestDoc.Assets, 1, "expected the captured image to survive a finalize capture miss")
blob, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.AssetsDir+manifestDoc.Assets[0].Name)
require.True(t, ok, "asset blob missing after finalize")
require.Equal(t, string(img), blob, "asset bytes changed after finalize")
}
// buildCursorStoreDB writes a Cursor-style store.db with a blobs(id, data) table
// populated from the given blobs, by shelling out to sqlite3.
func buildCursorStoreDB(t *testing.T, path string, blobs map[string][]byte) {
t.Helper()
var sb strings.Builder
sb.WriteString("CREATE TABLE blobs(id TEXT PRIMARY KEY, data BLOB);\n")
for id, data := range blobs {
sb.WriteString("INSERT INTO blobs(id,data) VALUES('" + id + "', x'" + hex.EncodeToString(data) + "');\n")
}
cmd := exec.CommandContext(context.Background(), "sqlite3", path, sb.String())
out, err := cmd.CombinedOutput()
require.NoErrorf(t, err, "build store.db: %s", out)
}
Acmd/entire/cli/integration_test/cursor_image_externalize_test.go+321
//go:build integration
package integration
import ( "context" "encoding/base64" "os" "path/filepath" "strings" "testing"
// TestImageExternalization_FullHookFlow is the real end-to-end proof: it drives // the actual entire hook binary (mid-turn commit -> condensation, then Stop -> // finalize) on a Claude Code session whose transcript embeds an inline base64 // image, with externalization enabled via settings.local.json (also exercising // the local-settings-merge fix). It then inspects the real entire/checkpoints/v1 // ref and confirms: // - full.jsonl carries the placeholder, not the raw base64 (survives finalize) // - the asset blob + manifest.json were written and decode to the exact image // - ReadSessionContent (the restore path) reinjects the image byte-exactly func TestImageExternalization_FullHookFlow(t *testing.T) { // Uses settings/env that must be stable across the hook subprocesses; no t.Parallel. env := NewFeatureBranchEnv(t)
// Enable externalization via the gitignored local settings file (the natural
// rollout opt-in, and the path the merge fix restored).
localSettings := filepath.Join(env.RepoDir, ".entire", "settings.local.json")
if err := os.WriteFile(localSettings, []byte({"redaction":{"externalize_images":true}}), 0o644); err != nil {
t.Fatalf("write settings.local.json: %v", err)
}
// A real, minimal PNG (valid magic bytes), padded so its base64 clears the // externalization length threshold. imgBytes := []byte("\x89PNG\r\n\x1a\n" + strings.Repeat("entire-real-e2e-image-payload-", 4)) b64 := base64.StdEncoding.EncodeToString(imgBytes)
session := env.NewSession()
// Author a Claude Code transcript: prompt, a user turn with an inline image,
// a file-writing tool use (so the commit has attributable content), result.
transcript := strings.Join([]string{
{"uuid":"u1","type":"user","message":{"role":"user","content":"add feature and look at this"},"timestamp":"2026-01-01T00:00:00Z"},
{"uuid":"u2","type":"user","message":{"role":"user","content":[{"type":"text","text":"screenshot"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":" + b64 + "}}]},"timestamp":"2026-01-01T00:00:01Z"},
{"uuid":"a1","type":"assistant","message":{"content":[{"type":"tool_use","id":"toolu_1","name":"Write","input":{"file_path":"feature.go","content":"package main\n"}}]},"timestamp":"2026-01-01T00:00:02Z"},
{"uuid":"u3","type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"Success"}]},"timestamp":"2026-01-01T00:00:03Z"},
{"uuid":"a2","type":"assistant","message":{"content":[{"type":"text","text":"done"}]},"timestamp":"2026-01-01T00:00:04Z"},
}, "\n") + "\n"
if err := os.WriteFile(session.TranscriptPath, []byte(transcript), 0o644); err != nil {
t.Fatalf("write transcript: %v", err)
}
if err := env.SimulateUserPromptSubmitWithPromptAndTranscriptPath(session.ID, "add feature and look at this", session.TranscriptPath); err != nil { t.Fatalf("UserPromptSubmit: %v", err) }
// Mid-turn commit -> post-commit condensation externalizes. env.WriteFile("feature.go", "package main\n") env.GitCommitWithShadowHooks("add feature", "feature.go")
// Stop -> finalize rewrites each turn checkpoint with the full transcript. This // is where the (fixed) re-inlining bug lived: assert externalization survives it. if err := env.SimulateStop(session.ID, session.TranscriptPath); err != nil { t.Fatalf("Stop: %v", err) }
if !env.BranchExists(paths.MetadataBranchName) { t.Fatal("entire/checkpoints/v1 should exist after condensation") } cpID := env.GetLatestCheckpointIDFromHistory() if cpID == "" { t.Fatal("no checkpoint id found in history") } sessionPath := ShardedCheckpointPath(cpID) + "/0/"
// full.jsonl: placeholder present, raw base64 gone (externalized, and it stuck // through finalize). full, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.TranscriptFileName) if !ok { t.Fatalf("full.jsonl missing at %s", sessionPath) } if strings.Contains(full, b64) { t.Error("stored full.jsonl still contains the raw base64 image (externalization did not persist)") } if !strings.Contains(full, "entire-asset:assets/") { t.Error("stored full.jsonl has no image placeholder") }
// manifest.json written; the asset blob decodes to the exact original image.
manifest, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.AssetsManifestFile)
if !ok {
t.Fatal("assets/manifest.json missing")
}
if !strings.Contains(manifest, "media_type": "image/png") {
t.Errorf("manifest missing png entry: %s", manifest)
}
// Restore path: ReadSessionContent reinjects the image byte-exactly. repo, err := gitrepo.OpenPath(env.RepoDir) if err != nil { t.Fatalf("open repo: %v", err) } defer repo.Close() stores, err := checkpoint.Open(context.Background(), repo, checkpoint.OpenOptions{}) if err != nil { t.Fatalf("open stores: %v", err) } checkpointID, err := id.NewCheckpointID(cpID) if err != nil { t.Fatalf("parse checkpoint id %q: %v", cpID, err) } content, err := stores.Persistent.ReadSessionContent(context.Background(), checkpointID, 0) if err != nil { t.Fatalf("ReadSessionContent: %v", err) } if strings.Contains(string(content.Transcript), "entire-asset:assets/") { t.Error("restored transcript still has a placeholder (reinjection failed)") } if !strings.Contains(string(content.Transcript), b64) { t.Error("restored transcript is missing the reinjected base64 image") } }
// TestImageExternalization_FinalizeWithFlagOffPreservesAssets guards the // config-drift case: externalization is ON at condensation (placeholders + // assets stored) but OFF at finalize (env override not inherited by the hook // process, or settings toggled mid-session). Extraction then doesn't run at // finalize and finalizeAssets is empty — that must mean "didn't run", not // "no images": the previously-stored asset blobs must survive the rewrite // (the re-inlined base64 in the finalized transcript is destroyed by // redaction, so clearing the assets would lose the images permanently). func TestImageExternalization_FinalizeWithFlagOffPreservesAssets(t *testing.T) { env := NewFeatureBranchEnv(t)
imgBytes := []byte("\x89PNG\r\n\x1a\n" + strings.Repeat("entire-flag-drift-image-payload-", 4)) b64 := base64.StdEncoding.EncodeToString(imgBytes)
session := env.NewSession()
transcript := strings.Join([]string{
{"uuid":"u1","type":"user","message":{"role":"user","content":"add feature and look at this"},"timestamp":"2026-01-01T00:00:00Z"},
{"uuid":"u2","type":"user","message":{"role":"user","content":[{"type":"text","text":"screenshot"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":" + b64 + "}}]},"timestamp":"2026-01-01T00:00:01Z"},
{"uuid":"a1","type":"assistant","message":{"content":[{"type":"tool_use","id":"toolu_1","name":"Write","input":{"file_path":"feature.go","content":"package main\n"}}]},"timestamp":"2026-01-01T00:00:02Z"},
{"uuid":"u3","type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"Success"}]},"timestamp":"2026-01-01T00:00:03Z"},
{"uuid":"a2","type":"assistant","message":{"content":[{"type":"text","text":"done"}]},"timestamp":"2026-01-01T00:00:04Z"},
}, "\n") + "\n"
if err := os.WriteFile(session.TranscriptPath, []byte(transcript), 0o644); err != nil {
t.Fatalf("write transcript: %v", err)
}
if err := env.SimulateUserPromptSubmitWithPromptAndTranscriptPath(session.ID, "add feature and look at this", session.TranscriptPath); err != nil {
t.Fatalf("UserPromptSubmit: %v", err)
}
// Mid-turn commit with the flag ON: condensation stores placeholder + asset. env.WriteFile("feature.go", "package main\n") env.GitCommitWithShadowHooks("add feature", "feature.go")
cpID := env.GetLatestCheckpointIDFromHistory() if cpID == "" { t.Fatal("no checkpoint id found in history") } sessionPath := ShardedCheckpointPath(cpID) + "/0/" manifest, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.AssetsManifestFile) if !ok { t.Fatal("PRECONDITION: condensation should have stored assets/manifest.json") }
// Toggle the flag OFF before the stop finalize. if err := os.Remove(localSettings); err != nil { t.Fatalf("remove settings.local.json: %v", err) } if err := env.SimulateStop(session.ID, session.TranscriptPath); err != nil { t.Fatalf("Stop: %v", err) }
// The stored assets must survive the finalize rewrite. manifestAfter, ok := env.ReadFileFromBranch(paths.MetadataBranchName, sessionPath+paths.AssetsManifestFile) if !ok { t.Fatal("assets/manifest.json was cleared by a finalize that ran without externalization") } if manifestAfter != manifest { t.Errorf("manifest changed across a flag-off finalize:\nbefore: %s\nafter: %s", manifest, manifestAfter) } }
Acmd/entire/cli/integration\_test/image\_externalize\_test.go+194
1102 unmodified lines
1103 1104 1105 1106 1107 1108 1109 1110 1111
1102 unmodified lines
"ENTIRE_TEST_OPENCODE_PROJECT_DIR="+env.OpenCodeProjectDir, "ENTIRE_TEST_OPENCODE_MOCK_EXPORT=1", ) // Propagate per-test overrides (e.g. agent project/store dirs) to hook // subprocesses. Empty for tests that don't set ExtraEnv. envVars = append(envVars, env.ExtraEnv...) envVars = append(envVars, env.checkpointStoreEnv()...) return append(envVars, extra...) }
Mcmd/entire/cli/integration\_test/testenv.go+3
35 unmodified lines
36 37 38 39 40 41 42 43 44 45 46 47 48
35 unmodified lines
CheckpointFileName = "checkpoint.json" ContentHashFileName = "content_hash.txt" SettingsFileName = "settings.json"
// AssetsDir is the per-session subfolder holding externalized transcript // assets (e.g. images); AssetsManifestFile indexes them. AssetsDirName is the // bare tree-entry name (no trailing slash) used when walking git trees. AssetsDirName = "assets" AssetsDir = "assets/" AssetsManifestFile = "assets/manifest.json" )
// MetadataBranchName is the orphan branch used by manual-commit strategy to store metadata
Mcmd/entire/cli/paths/paths.go+7
30 unmodified lines
31 32 33 34 35 36 37 38 39 40 41 42 43 44 12 unmodified lines
57 58 59 52 60 61 62 63 64 65 66 67 68 69
30 unmodified lines
agentPluginBinaryPrefix = "entire-agent-" )
// selfUpdatePluginName is the plugin that replaces the entire binary on
// disk (entire upgrade → entire-upgrade).
const selfUpdatePluginName = "upgrade"
// postPluginVersionCheck is a test seam for the version-check notice that // fires after a successful plugin run. var postPluginVersionCheck = versioncheck.CheckAndNotify
// MaybeRunPlugin returns (true, exitCode) when an external command was // resolved and run. On launch failure (e.g. missing executable bit) // returns (true, 1) after printing to stderr. On no-match returns 12 unmodified lines
maybeTrackPluginInvocation(ctx, pluginName) // Stderr, matching the built-in PersistentPostRun: the plugin's own // stdout may be machine-readable and piped. versioncheck.CheckAndNotify(ctx, os.Stderr, versioninfo.Version) // // Skipped after a self-update: this process still carries the // pre-upgrade compiled-in version, so the check would see itself as // outdated and prompt to redo the upgrade that just completed. if pluginName != selfUpdatePluginName { postPluginVersionCheck(ctx, os.Stderr, versioninfo.Version) } } return true, exitCode }
Mcmd/entire/cli/plugin.go+15/-1
2 unmodified lines
3 4 5 6 7 8 9 182 unmodified lines
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
2 unmodified lines
import ( "context" "fmt" "io" "os" "path/filepath" "runtime" 182 unmodified lines
} }
// interceptVersionCheck swaps the post-plugin version-check seam for a // counter and restores it on cleanup. func interceptVersionCheck(t *testing.T) *int { t.Helper() calls := 0 orig := postPluginVersionCheck postPluginVersionCheck = func(context.Context, io.Writer, string) { calls++ } t.Cleanup(func() { postPluginVersionCheck = orig }) return &calls }
func TestMaybeRunPlugin_VersionCheckAfterSuccess(t *testing.T) { //nolint:paralleltest // mutates PATH and the version-check seam dir := t.TempDir() writePluginBinary(t, dir, "entire-pgr", filepath.Join(dir, "args.txt"), 0) withPathDir(t, dir) calls := interceptVersionCheck(t)
handled, code := MaybeRunPlugin(context.Background(), newTestRoot(), []string{"pgr"}) if !handled || code != 0 { t.Fatalf("handled=%v code=%d, want handled=true code=0", handled, code) } if *calls != 1 { t.Errorf("version check calls: got %d, want 1", *calls) } }
// After entire upgrade replaces the binary on disk, this process still
// carries the pre-upgrade compiled-in version — a post-run version check
// would see itself as outdated and prompt to redo the finished upgrade.
func TestMaybeRunPlugin_NoVersionCheckAfterSelfUpdate(t *testing.T) { //nolint:paralleltest // mutates PATH and the version-check seam
dir := t.TempDir()
writePluginBinary(t, dir, "entire-upgrade", filepath.Join(dir, "args.txt"), 0)
withPathDir(t, dir)
calls := interceptVersionCheck(t)
handled, code := MaybeRunPlugin(context.Background(), newTestRoot(), []string{"upgrade", "--nightly"}) if !handled || code != 0 { t.Fatalf("handled=%v code=%d, want handled=true code=0", handled, code) } if *calls != 0 { t.Errorf("version check calls: got %d, want 0", *calls) } }
func TestMaybeRunPlugin_NoVersionCheckAfterFailure(t *testing.T) { //nolint:paralleltest // mutates PATH and the version-check seam dir := t.TempDir() writePluginBinary(t, dir, "entire-pgr", filepath.Join(dir, "args.txt"), 3) withPathDir(t, dir) calls := interceptVersionCheck(t)
handled, code := MaybeRunPlugin(context.Background(), newTestRoot(), []string{"pgr"}) if !handled || code != 3 { t.Fatalf("handled=%v code=%d, want handled=true code=3", handled, code) } if *calls != 0 { t.Errorf("version check calls: got %d, want 0", *calls) } }
func equalStrings(a, b []string) bool { if len(a) != len(b) { return false
Mcmd/entire/cli/plugin\_test.go+60
1170 unmodified lines
1171 1172 1173 1174 1175 1176 1177 4 unmodified lines
1182 1183 1184 1184 1185 1186 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 15 unmodified lines
1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224
1170 unmodified lines
checkpointContext = deps.ReviewCheckpointContext(ctx, worktreeRoot, scopeBaseRef) } reviewers := make([]reviewtypes.AgentReviewer, 0, len(launchableEligible)) var excludedWorkers []string for _, choice := range launchableEligible { workerName := choice.Name agentCfg := profile.Agents[workerName] 4 unmodified lines
return fmt.Errorf("resolve agent %s: %w", agentName, agErr) } if err := VerifyConfiguredSkillsInstalled(ctx, ag, agentCfg); err != nil { cmd.SilenceUsage = true fmt.Fprintln(cmd.ErrOrStderr(), err.Error()) return deps.NewSilentError(err) // One worker's stale config must not hold the whole crew // hostage (e.g. codex's legacy auto-preselected "/review", // orphaned when its curated builtin was removed). Exclude // the worker loudly and let the remaining reviewers run; // the all-excluded case fails below. excludedWorkers = append(excludedWorkers, workerName) fmt.Fprintf(cmd.ErrOrStderr(), "skipping reviewer %s: %s\n", workerName, err.Error()) continue } } reviewer := deps.ReviewerFor(agentName) 15 unmodified lines
}) }
if len(reviewers) == 0 {
cmd.SilenceUsage = true
err := fmt.Errorf("no runnable reviewers: every configured worker failed skill validation (%s); run entire review --edit to reconfigure",
strings.Join(excludedWorkers, ", "))
fmt.Fprintln(cmd.ErrOrStderr(), err.Error())
return deps.NewSilentError(err)
}
runCtx, cancelRun := context.WithCancel(ctx) defer cancelRun()
Mcmd/entire/cli/review/cmd.go+17/-3
3 unmodified lines
4 5 6 7 8 9 10 11 752 unmodified lines
764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 771 788 789 790 791 34 unmodified lines
826 827 828 829 830 831 814 815 832 833 834 835 836 837 820 821 838 839 840 841 842 394 unmodified lines
1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
3 unmodified lines
"bytes" "context" "errors" "os" "path/filepath" "strings" "testing" "time" 752 unmodified lines
func TestDispatchFork_MultiAgentPassesPerAgentConfigs(t *testing.T) { setupCmdTestRepo(t)
// Codex has no curated built-ins — its skills are discovered on disk in // $name form, so spawn-time validation needs a real SKILL.md under a // controlled HOME. (Cannot t.Parallel — t.Setenv; setupCmdTestRepo // already precludes parallelism via t.Chdir.) home := t.TempDir() t.Setenv("HOME", home) skillDir := filepath.Join(home, ".codex", "skills", "code-review") if err := os.MkdirAll(skillDir, 0o755); err != nil { t.Fatal(err) } skillMD := "---\nname: code-review\ndescription: Review code changes.\n---\n\nbody\n" if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte(skillMD), 0o644); err != nil { t.Fatal(err) }
if err := seedReviewConfig(context.Background(), map[string]settings.ReviewConfig{ "claude-code": { Skills: []string{"/review"}, Prompt: "Claude saved prompt.", }, testCodexAgent: { Skills: []string{"/review"}, Skills: []string{"$code-review"}, Prompt: "Codex saved prompt.", }, }); err != nil { 34 unmodified lines
for _, tc := range []struct { name string reviewer *captureRunConfigReviewer wantSkill string wantPrompt string }{ {name: "claude-code", reviewer: claudeReviewer, wantPrompt: "Claude saved prompt."}, {name: "codex", reviewer: codexReviewer, wantPrompt: "Codex saved prompt."}, {name: "claude-code", reviewer: claudeReviewer, wantSkill: "/review", wantPrompt: "Claude saved prompt."}, {name: "codex", reviewer: codexReviewer, wantSkill: "$code-review", wantPrompt: "Codex saved prompt."}, } { if !tc.reviewer.called { t.Fatalf("%s reviewer was not started", tc.name) } if got := tc.reviewer.got.Skills; len(got) != 1 || got[0] != "/review" { t.Fatalf("%s Skills = %v, want [/review]", tc.name, got) if got := tc.reviewer.got.Skills; len(got) != 1 || got[0] != tc.wantSkill { t.Fatalf("%s Skills = %v, want [%s]", tc.name, got, tc.wantSkill) } if tc.reviewer.got.AlwaysPrompt != tc.wantPrompt { t.Fatalf("%s AlwaysPrompt = %q, want %q", tc.name, tc.reviewer.got.AlwaysPrompt, tc.wantPrompt) 394 unmodified lines
t.Fatal("auto synthesis should notify the TUI when the final judge starts/completes") } }
// TestDispatchFork_InvalidSkillExcludesWorkerNotWholeCrew pins the blast // radius of spawn-time skill validation in multi-agent runs: a worker whose // configured skill no longer validates (e.g. codex's legacy auto-preselected // "/review", orphaned when the curated builtin was removed) is excluded with // a loud warning, and the remaining reviewers still run. Aborting the whole // crew for one stale entry held every other agent hostage to a codex // reconfigure. func TestDispatchFork_InvalidSkillExcludesWorkerNotWholeCrew(t *testing.T) { setupCmdTestRepo(t) // Controlled empty HOME: codex discovery finds nothing, so its "/review" // (no longer a curated builtin) fails validation. Cannot t.Parallel — // t.Setenv (setupCmdTestRepo already precludes it via t.Chdir). t.Setenv("HOME", t.TempDir())
if err := seedReviewConfig(context.Background(), map[string]settings.ReviewConfig{ testAgentName: { Skills: []string{"/review"}, }, testCodexAgent: { Skills: []string{"/review"}, // stale legacy entry }, }); err != nil { t.Fatal(err) }
claudeReviewer := &captureRunConfigReviewer{name: testAgentName} codexReviewer := &captureRunConfigReviewer{name: testCodexAgent} deps := review.Deps{ GetAgentsWithHooksInstalled: func(_ context.Context) []types.AgentName { return []types.AgentName{testAgentName, testCodexAgent} }, NewSilentError: func(err error) error { return err }, HeadHasReviewCheckpoint: func(_ context.Context) (bool, string) { return false, "" }, ReviewerFor: func(agentName string) reviewtypes.AgentReviewer { switch agentName { case testAgentName: return claudeReviewer case testCodexAgent: return codexReviewer default: return nil } }, }
cmd := review.NewCommand(deps) cmd.SetOut(&bytes.Buffer{}) errBuf := &bytes.Buffer{} cmd.SetErr(errBuf) cmd.SetArgs([]string{"general"})
if err := cmd.Execute(); err != nil { t.Fatalf("run should proceed with the valid reviewer, got error: %v", err) } if !claudeReviewer.called { t.Error("claude-code reviewer was not started — valid worker excluded with the invalid one") } if codexReviewer.called { t.Error("codex reviewer started despite failing skill validation") } stderr := errBuf.String() if !strings.Contains(stderr, "/review") || !strings.Contains(stderr, "skipping") { t.Errorf("stderr should warn about the excluded worker and its skill; got:\n%s", stderr) } }
// TestDispatchFork_AllWorkersInvalidStillFails pins the floor: when skill // validation excludes every worker, the run fails loudly instead of silently // reviewing with nobody. func TestDispatchFork_AllWorkersInvalidStillFails(t *testing.T) { setupCmdTestRepo(t) t.Setenv("HOME", t.TempDir())
if err := seedReviewConfig(context.Background(), map[string]settings.ReviewConfig{ testCodexAgent: {Skills: []string{"/review"}}, "gemini": {Skills: []string{"$also-missing"}}, }); err != nil { t.Fatal(err) }
deps := review.Deps{ GetAgentsWithHooksInstalled: func(_ context.Context) []types.AgentName { return []types.AgentName{testCodexAgent, "gemini"} }, NewSilentError: func(err error) error { return err }, HeadHasReviewCheckpoint: func(_ context.Context) (bool, string) { return false, "" }, ReviewerFor: func(agentName string) reviewtypes.AgentReviewer { return &captureRunConfigReviewer{name: agentName} }, }
cmd := review.NewCommand(deps) cmd.SetOut(&bytes.Buffer{}) cmd.SetErr(&bytes.Buffer{}) cmd.SetArgs([]string{"general"})
if err := cmd.Execute(); err == nil { t.Fatal("expected an error when every worker fails skill validation") } }
Mcmd/entire/cli/review/cmd\_test.go+128/-5
250 unmodified lines
251 252 253 254 255 256 257 258 259 260 261 882 unmodified lines
1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 188 unmodified lines
1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365
250 unmodified lines
// OpenAIPrivacyFilter is the optional 8th redaction layer (opt-in).
// See docs/security-and-privacy.md.
OpenAIPrivacyFilter *OPFSettings json:"openai_privacy_filter,omitempty"
// ExternalizeImages opts into lifting inline base64 images out of transcripts
// into the checkpoint's assets/ store (off by default). Restore re-injects
// them regardless of this flag.
ExternalizeImages bool json:"externalize_images,omitempty"
}
// PIISettings configures PII detection categories. 882 unmodified lines
return err } } if extRaw, ok := raw["externalize_images"]; ok { var v bool if err := json.Unmarshal(extRaw, &v); err != nil { return fmt.Errorf("parsing redaction.externalize_images: %w", err) } dst.ExternalizeImages = v } return nil }
188 unmodified lines
return settings.IsSummarizeEnabled() }
// IsImageExternalizationEnabled reports whether inline base64 images should be // lifted out of transcripts into the checkpoint asset store. Opt-in via // redaction.externalize_images, or the ENTIRE_EXTERNALIZE_IMAGES=1 env override // (handy for testing/rollout). Off by default. func IsImageExternalizationEnabled(ctx context.Context) bool { if v := os.Getenv("ENTIRE_EXTERNALIZE_IMAGES"); v == "1" || v == "true" { return true } s, err := Load(ctx) if err != nil { return false } return s.Redaction != nil && s.Redaction.ExternalizeImages }
// IsSummarizeEnabled checks if auto-summarize is enabled in this settings instance. func (s *EntireSettings) IsSummarizeEnabled() bool { if s.StrategyOptions == nil {
Mcmd/entire/cli/settings/settings.go+27
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
package settings
import ( "context" "testing" )
// These tests use setupSettingsDir (t.Chdir) and t.Setenv, both process-global, // so they cannot run in parallel.
func TestIsImageExternalizationEnabled_DefaultsFalse(t *testing.T) {
setupSettingsDir(t, {"enabled": true}, "")
if IsImageExternalizationEnabled(context.Background()) {
t.Error("image externalization should be off by default")
}
}
func TestIsImageExternalizationEnabled_FileEnabled(t *testing.T) {
setupSettingsDir(t, {"enabled": true, "redaction": {"externalize_images": true}}, "")
if !IsImageExternalizationEnabled(context.Background()) {
t.Error("redaction.externalize_images: true should enable externalization")
}
}
func TestIsImageExternalizationEnabled_EnvOverride(t *testing.T) {
setupSettingsDir(t, {"enabled": true}, "")
t.Setenv("ENTIRE_EXTERNALIZE_IMAGES", "1")
if !IsImageExternalizationEnabled(context.Background()) {
t.Error("ENTIRE_EXTERNALIZE_IMAGES=1 should enable externalization regardless of settings")
}
}
func TestIsImageExternalizationEnabled_LocalFileEnables(t *testing.T) {
// The gitignored settings.local.json is the natural place to opt into a
// rollout feature; the merge path must honor it.
setupSettingsDir(t, {"enabled": true}, {"redaction": {"externalize_images": true}})
if !IsImageExternalizationEnabled(context.Background()) {
t.Error("externalize_images in settings.local.json must enable externalization")
}
}
func TestIsImageExternalizationEnabled_LocalFileDisablesBaseEnable(t *testing.T) {
// A per-machine kill switch: local:false must override base:true.
setupSettingsDir(t,
{"enabled": true, "redaction": {"externalize_images": true}},
{"redaction": {"externalize_images": false}})
if IsImageExternalizationEnabled(context.Background()) {
t.Error("local externalize_images:false must override a base value of true")
}
}
// TestRedactionSettings_ExternalizeImagesJSONTag guards the JSON field name.
// LoadFromBytes uses DisallowUnknownFields, so a wrong tag fails to parse.
func TestRedactionSettings_ExternalizeImagesJSONTag(t *testing.T) {
t.Parallel()
s, err := LoadFromBytes([]byte({"enabled": true, "redaction": {"externalize_images": true}}))
if err != nil {
t.Fatalf("LoadFromBytes() error = %v", err)
}
if s.Redaction == nil || !s.Redaction.ExternalizeImages {
t.Errorf("externalize_images did not parse into RedactionSettings.ExternalizeImages")
}
}
Acmd/entire/cli/settings/settings\_images\_test.go+63
934 unmodified lines
935 936 937 938 938 939 940 941 3 unmodified lines
945 946 947 948 949 950 951 952 953 954 955 956 957
934 unmodified lines
cmd.Flags().BoolVar(&opts.AbsoluteGitHookPath, flagAbsoluteGitHookPath, false, "Embed full binary path in git hooks (for GUI git clients that don't source shell profiles)")
cmd.Flags().BoolVar(&opts.SearchSkill, flagSearchSkill, false, "Install the optional Entire search skill for selected agent(s)")
cmd.Flags().BoolVar(&opts.AgentHelpSkill, flagAgentHelpSkill, false, "Install the stable Entire agent-help skill (points agents at entire agent-help) for selected agent(s)")
cmd.Flags().BoolVarP(&opts.Yes, "yes", "y", false, "Accept all defaults without prompting (in a non-repo directory: init git, create private GitHub repo, commit; then enable all agents and accept telemetry)")
cmd.Flags().BoolVarP(&opts.Yes, "yes", "y", false, "Accept all defaults without prompting (in a non-repo directory: init git, create private GitHub repo, commit, and push; then enable all agents and accept telemetry)")
addInsecureHTTPAuthFlag(cmd, &insecureHTTPAuth)
// Bootstrap flags for non-git-repo folders. 3 unmodified lines
cmd.Flags().StringVar(&bootstrapOpts.RepoOwner, "repo-owner", "", "GitHub user or organization login for the new repo") cmd.Flags().StringVar(&bootstrapOpts.RepoVisibility, "repo-visibility", "", "GitHub repository visibility: public, private, or internal") cmd.Flags().BoolVar(&bootstrapOpts.NoGitHub, "no-github", false, "Initialize local git repo only; skip creating a GitHub remote") cmd.Flags().BoolVar(&bootstrapOpts.Push, "push", false, "When bootstrapping a new repo, push the initial commit to the created GitHub remote (implies creating the remote; without it the repo is created but not pushed)") cmd.Flags().StringVar(&bootstrapOpts.InitialCommitMessage, "initial-commit-message", "", "Commit message for the initial commit when bootstrapping a new repo") cmd.Flags().BoolVar(&bootstrapOpts.SkipInitialCommit, "skip-initial-commit", false, "Don't create the initial commit when bootstrapping a new repo") cmd.MarkFlagsMutuallyExclusive("init-repo", "no-init-repo") cmd.MarkFlagsMutuallyExclusive("initial-commit-message", "skip-initial-commit") cmd.MarkFlagsMutuallyExclusive("push", "no-github") cmd.MarkFlagsMutuallyExclusive("push", "skip-initial-commit")
// Provide a helpful error when --agent is used without a value defaultFlagErr := cmd.FlagErrorFunc()
Mcmd/entire/cli/setup.go+4/-1
41 unmodified lines
42 43 44 45 46 45 46 47 48 49 50 51 52 53 54 67 unmodified lines
122 123 124 125 126 127 128 36 unmodified lines
165 166 167 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 26 unmodified lines
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 2 unmodified lines
253 254 255 256 257 258 259 22 unmodified lines
282 283 284 258 285 286 287 288 15 unmodified lines
304 305 306 307 308 309 281 310 311 312 313 314 286 315 316 317 318 319 320 321 322 323 324 325 16 unmodified lines
342 343 344 345 346 347 348 349 350 351 352 353 354 312 313 314 355 356 357 358 359 360 361 362 363 364 365 318 366 367 368 369 6 unmodified lines
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 10 unmodified lines
418 419 420 347 348 421 422 423 424 425 426 427 428 429 352 430 431 432 433 508 unmodified lines
942 943 944 867 868 869 945 946 947 948 949 950 951 12 unmodified lines
964 965 966 888 967 968 969 970
41 unmodified lines
// still created, but nothing is pushed. SkipInitialCommit bool // Yes accepts all defaults without prompting: init repo, create GitHub // repo under the user's account (private), default commit message. // Explicit flags (--no-github, --repo-owner, etc.) take precedence. // repo under the user's account (private), default commit message, and // push. Explicit flags (--no-github, --repo-owner, etc.) take precedence. Yes bool // Push opts into pushing the initial commit to the created GitHub remote // without prompting. Pushing is otherwise an explicit, separate opt-in // (interactive "yes" or --yes). Implies creating the remote. Push bool }
// bootstrapRunner executes external commands. Tests override this to avoid 67 unmodified lines
visibility string // public/private/internal, if useGitHub commit bool // false means the user opted out of the initial commit message string // resolved initial commit message (empty when !commit) push bool // false means create the GitHub repo but don't push to it }
// runGitHubBootstrapInit handles the pre-setup half of "enable on a non-git 36 unmodified lines
paths.ClearWorktreeRootCache() fmt.Fprintln(w, " ✓ Initialized empty git repository")
// Step 3: decide whether to create a GitHub repo. If gh is missing or the
// user passed --no-github, we skip that branch but still bootstrap the
// local repo.
useGitHub := !opts.NoGitHub
if useGitHub {
if !ghAvailable(ctx, runner) {
fmt.Fprintln(errW, "gh CLI not found. Install it from https://cli.github.com/ and run gh auth login to add a GitHub remote.")
fmt.Fprintln(errW, "Continuing with local initialization only.")
useGitHub = false
} else if !ghAuthenticated(ctx, runner) {
fmt.Fprintln(errW, "gh CLI is not authenticated. Run gh auth login to add a GitHub remote.")
fmt.Fprintln(errW, "Continuing with local initialization only.")
useGitHub = false
}
}
// Step 3b: ask a simple yes/no before diving into owner/name/visibility
// prompts. Skip the confirm when any gh-specific flag is set (the flag
// implies intent) or when we're non-interactive (keep the documented
// happy path: default to yes).
if useGitHub && !opts.Yes && !ghFlagsProvided(opts) && interactive.CanPromptInteractively() {
confirmed, err := confirmCreateGitHubRepo()
if err != nil {
return nil, err
}
if !confirmed {
useGitHub = false
// Step 3: decide whether to create a GitHub repo. Creating a remote is an
// explicit opt-in: it happens only on an explicit signal (repo flags,
// --push, or --yes) or an interactive "yes". A non-interactive run with no
// such signal stays local-only — we never create a repo on the user's
// behalf. --no-github always wins.
useGitHub := false
if !opts.NoGitHub {
explicit := ghCreateRequested(opts)
// Only probe gh (and warn about a missing/unauthenticated CLI) when the
// user actually wants a GitHub repo — explicitly, or via the confirm
// prompt we're about to show interactively.
if explicit || interactive.CanPromptInteractively() {
switch {
case !ghAvailable(ctx, runner):
fmt.Fprintln(errW, "gh CLI not found. Install it from https://cli.github.com/ and run gh auth login to add a GitHub remote.")
fmt.Fprintln(errW, "Continuing with local initialization only.")
case !ghAuthenticated(ctx, runner):
fmt.Fprintln(errW, "gh CLI is not authenticated. Run gh auth login to add a GitHub remote.")
fmt.Fprintln(errW, "Continuing with local initialization only.")
case explicit:
useGitHub = true
default:
// Interactive with no explicit signal: prompt, defaulting to No.
confirmed, err := confirmCreateGitHubRepo(cwd)
if err != nil {
return nil, err
}
useGitHub = confirmed
}
}
}
26 unmodified lines
} }
// Step 6: pushing is also an explicit opt-in, separate from creating the // repo. Publishing the directory's contents is a distinct outward-facing // action, so it happens only on an explicit signal (--push or --yes) or an // interactive "yes". Otherwise the repo is created but left unpushed. Only // relevant when we'll create a GitHub repo and have a commit to push. push := false if useGitHub && commit { switch { case opts.Yes || opts.Push: push = true case interactive.CanPromptInteractively(): confirmed, err := confirmPushToRemote(fullName) if err != nil { return nil, err } push = confirmed } }
return &bootstrapState{ runner: runner, cwd: cwd, 2 unmodified lines
visibility: visibility, commit: commit, message: message, push: push, }, nil }
22 unmodified lines
// Pick a single section title for this phase based on what we'll do. if s.useGitHub || s.commit { switch { case s.useGitHub && s.commit: case s.useGitHub && s.commit && s.push: printBootstrapSection(w, "Publishing to GitHub") case s.useGitHub: printBootstrapSection(w, "Creating GitHub repository") 15 unmodified lines
fmt.Fprintln(w, " ✓ Nothing to commit — the folder has no files yet") } } // Push only when there's a commit AND the user opted into pushing. pushed := committed && s.push if s.useGitHub { if err := ghRepoCreate(ctx, s.runner, s.cwd, s.fullName, s.visibility, committed); err != nil { if err := ghRepoCreate(ctx, s.runner, s.cwd, s.fullName, s.visibility, pushed); err != nil { return fmt.Errorf("gh repo create: %w", err) } fmt.Fprintf(w, " ✓ Created %s (%s)\n", s.fullName, s.visibility) fmt.Fprintf(w, " https://github.com/%s\n", s.fullName) if committed { if pushed { fmt.Fprintln(w, " ✓ Pushed initial commit to origin") } else if committed { // Repo created and origin configured, but the user declined the // push. Tell them how to publish when ready. fmt.Fprintln(w) fmt.Fprintln(w, " Skipped push — nothing was published. When you're ready:") fmt.Fprintln(w, " git push -u origin HEAD") } } if !s.commit { 16 unmodified lines
return opts.RepoName != "" || opts.RepoOwner != "" || opts.RepoVisibility != "" }
// ghCreateRequested reports whether the caller has explicitly opted into // creating a GitHub repo without an interactive prompt: --yes, --push (which // needs a remote to push to), or any repo-targeting flag. When false and the // session is non-interactive, the bootstrap stays local-only. func ghCreateRequested(opts GitHubBootstrapOptions) bool { return opts.Yes || opts.Push || ghFlagsProvided(opts) }
// confirmCreateGitHubRepo asks the user whether they want to also create // a matching GitHub repository. Interactive-only; callers gate on // interactive.CanPromptInteractively. func confirmCreateGitHubRepo() (bool, error) { confirmed := true // interactive.CanPromptInteractively. Pushing to the repo is confirmed // separately (see confirmPushToRemote). // // Defaults to No: creating a remote repository on the user's behalf must // never happen just because the user pressed Enter. The absolute path is in // the title so it's clear which directory is the source. func confirmCreateGitHubRepo(cwd string) (bool, error) { confirmed := false form := NewAccessibleForm( huh.NewGroup( huh.NewConfirm(). Title("Create a matching repository on GitHub?"). Title(fmt.Sprintf("Create a GitHub repository for %q?", cwd)). Value(&confirmed), ), ) 6 unmodified lines
return confirmed, nil }
// confirmPushToRemote asks the user whether to push the initial commit to // the newly-created GitHub repository. Interactive-only; callers gate on // interactive.CanPromptInteractively. // // Defaults to No: pushing publishes the directory's contents to the remote, // a distinct outward-facing action from creating the repo, so it must never // happen just because the user pressed Enter. Declining leaves the repo // created with origin configured but nothing pushed. func confirmPushToRemote(fullName string) (bool, error) { confirmed := false form := NewAccessibleForm( huh.NewGroup( huh.NewConfirm(). Title(fmt.Sprintf("Push the initial commit to %q?", fullName)). Value(&confirmed), ), ) if err := form.Run(); err != nil { if errors.Is(err, huh.ErrUserAborted) { return false, errBootstrapInterrupted } return false, fmt.Errorf("push confirm prompt: %w", err) } return confirmed, nil }
// confirmInitRepo returns true if we should proceed with git init. It
// respects --init-repo / --no-init-repo; otherwise prompts. In
// non-interactive mode we return false without printing anything so
10 unmodified lines
return false, nil }
folder := filepath.Base(cwd)
confirmed := true
// Default to No: entire enable is often run reflexively inside an
// existing project, so a stray run in the wrong (non-repo) directory
// must not initialize a repo just because the user pressed Enter. The
// absolute path is in the title so a wrong-directory mistake is obvious
// in both interactive and accessible modes.
confirmed := false
form := NewAccessibleForm(
huh.NewGroup(
huh.NewConfirm().
Title(fmt.Sprintf("No git repository in %q. Initialize one here?", folder)).
Title(fmt.Sprintf("Warning: Not a git repository. Initialize a new one in %q?", cwd)).
Value(&confirmed),
),
)
508 unmodified lines
return false, fmt.Errorf("gh repo view: %w", err) }
// ghRepoCreate creates a GitHub repo from the local source directory, adds // origin as its remote, and pushes if there's anything to push. func ghRepoCreate(ctx context.Context, runner bootstrapRunner, dir, fullName, visibility string, hasCommits bool) error { // ghRepoCreate creates a GitHub repo from the local source directory and // adds origin as its remote. It pushes only when push is true; callers gate // this on both having a commit and the user opting into the push. func ghRepoCreate(ctx context.Context, runner bootstrapRunner, dir, fullName, visibility string, push bool) error { // Create the remote repo and add origin, but don't push yet. We push // separately below with --no-verify so the pre-push hook doesn't run // on this first push: the entire/checkpoints/v1 branch has nothing to 12 unmodified lines
if _, err := runner.RunInDir(ctx, dir, "gh", args...); err != nil { return fmt.Errorf("gh repo create: %w", ghRunnerErr(err)) } if hasCommits { if push { // -q silences "Enumerating objects..." etc. --no-verify bypasses // the pre-push hook so entire/checkpoints/v1 isn't pushed // alongside the default branch.
Mcmd/entire/cli/setup\_github.go+122/-43
384 unmodified lines
385 386 387 388 388 389 390 391 392 393 33 unmodified lines
427 428 429 430 431 432 433 157 unmodified lines
591 592 593 591 592 593 594 594 595 596 597 598 599 600 601 602 603 600 601 602 603 604 605 606 604 605 606 607 608 609 613 614 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 30 unmodified lines
700 701 702 703 704 705 706 322 unmodified lines
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 28 unmodified lines
1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 89 unmodified lines
1310 1311 1312 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1313 1314 1315 1316 1317 1318
384 unmodified lines
r.set("git", []string{"add", "-A"}, "", nil) r.set("git", []string{"status", "--porcelain"}, "", nil)
opts := GitHubBootstrapOptions{InitRepo: true} // A repo flag is an explicit GitHub request, so gh is probed; since it's // missing we warn and fall back to local-only. opts := GitHubBootstrapOptions{InitRepo: true, RepoName: "wanted"} var errBuf bytes.Buffer err := runGitHubBootstrapWith(context.Background(), io.Discard, &errBuf, opts, r) if err != nil { 33 unmodified lines
RepoName: "my-new", RepoVisibility: "private", InitialCommitMessage: "Seed", Push: true, } err := runGitHubBootstrapWith(context.Background(), io.Discard, io.Discard, opts, r) if err != nil { 157 unmodified lines
} }
// TestRunGitHubBootstrap_NonInteractive_NoFlagsDefaultsToGitHub confirms the // non-interactive happy path still creates a GitHub repo when the user // didn't set any explicit flag (the confirm prompt is only interactive). func TestRunGitHubBootstrap_NonInteractive_NoFlagsDefaultsToGitHub(t testing.T) { // TestRunGitHubBootstrap_NonInteractive_NoFlagsStaysLocal confirms that a // non-interactive bootstrap with no explicit GitHub signal stays local-only: // it does not probe gh, create a repo, or push. Creating and pushing are // explicit opt-ins (--repo-, --push, --yes, or an interactive "yes"). func TestRunGitHubBootstrap_NonInteractive_NoFlagsStaysLocal(t *testing.T) { dir := t.TempDir() restoreCwd(t, dir)
r := newFakeRunner() r.setIdentityConfigured() r.set("gh", []string{"--version"}, "gh", nil) r.set("gh", []string{"auth", "status"}, "ok", nil) r.set("gh", []string{"api", "user", "--jq", ".login"}, "octocat\n", nil) r.set("gh", []string{"api", "user/orgs", "--jq", ".[].login"}, "", nil) // Default folder slug derived from t.TempDir(). suggested := slugifyRepoName(filepath.Base(dir)) r.set("gh", []string{"repo", "view", "octocat/" + suggested, "--json", "name"}, "", errors.New("not found")) r.set("git", []string{"init"}, "", nil)
state, err := runGitHubBootstrapInitWith(context.Background(), io.Discard, io.Discard, GitHubBootstrapOptions{InitRepo: true}, r) if err != nil { t.Fatalf("init failed: %v", err) } if !state.useGitHub { t.Fatal("non-interactive bootstrap should default to using GitHub") if state.useGitHub { t.Fatal("non-interactive bootstrap with no explicit signal must stay local-only") } if state.push { t.Fatal("push must be false when staying local-only") } // gh must never be probed when no GitHub repo was requested. if r.hasCall(func(c fakeCall) bool { return c.name == "gh" }) { t.Fatal("must not invoke gh when no GitHub repo was requested") } }
// TestRunGitHubBootstrap_RepoFlagsCreateButDoNotPush confirms that repo flags // opt into creating the GitHub repo but NOT into pushing. Non-interactively, // the repo is created and origin configured, but nothing is pushed unless // --push or --yes is also given; the user is told how to publish manually. func TestRunGitHubBootstrap_RepoFlagsCreateButDoNotPush(t *testing.T) { dir := t.TempDir() restoreCwd(t, dir)
r := newFakeRunner() r.setIdentityConfigured() r.set("gh", []string{"--version"}, "gh 2.81.0", nil) r.set("gh", []string{"auth", "status"}, "Logged in", nil) r.set("gh", []string{"api", "user", "--jq", ".login"}, "octocat\n", nil) r.set("gh", []string{"api", "user/orgs", "--jq", ".[].login"}, "", nil) r.set("gh", []string{"repo", "view", "octocat/create-only", "--json", "name"}, "", errors.New("not found")) r.set("git", []string{"init"}, "", nil) r.set("git", []string{"add", "-A"}, "", nil) r.set("git", []string{"status", "--porcelain"}, " M f\n", nil) r.set("git", []string{"-c", "commit.gpgsign=false", "commit", "-m", "Seed"}, "", nil) r.set("gh", []string{ "repo", "create", "octocat/create-only", "--private", "--source=.", "--remote=origin", }, "", nil)
opts := GitHubBootstrapOptions{ InitRepo: true, RepoName: "create-only", RepoVisibility: "private", InitialCommitMessage: "Seed", } var out bytes.Buffer if err := runGitHubBootstrapWith(context.Background(), &out, io.Discard, opts, r); err != nil { t.Fatalf("bootstrap failed: %v", err) }
if !r.hasCall(argsMatch("gh", []string{"repo", "create"})) { t.Fatal("expected gh repo create when repo flags are given") } if r.hasCall(argsMatch("git", []string{"push"})) { t.Fatal("must not push without --push or --yes") } if !strings.Contains(out.String(), "Skipped push") { t.Fatalf("expected 'Skipped push' guidance, got: %s", out.String()) } }
30 unmodified lines
RepoName: "phased", RepoVisibility: "private", InitialCommitMessage: "First", Push: true, }
// Phase 1: init. This must NOT call git add/commit/ gh repo create. 322 unmodified lines
} }
func TestEnableCmd_PushNoGitHubMutuallyExclusive(t *testing.T) { setupTestRepo(t)
cmd := newEnableCmd() var stderr bytes.Buffer cmd.SetErr(&stderr) cmd.SetOut(&bytes.Buffer{}) cmd.SetArgs([]string{"--push", "--no-github"}) err := cmd.Execute() if err == nil { t.Fatal("expected error when both --push and --no-github are set") } if !strings.Contains(err.Error(), "push") || !strings.Contains(err.Error(), "no-github") { t.Fatalf("expected error to mention both flags, got: %v", err) } }
func TestEnableCmd_InitCommitMessageFlagsMutuallyExclusive(t *testing.T) { setupTestRepo(t)
28 unmodified lines
} }
// withInteractivePromptStdin forces interactive, accessible (text-based) // prompt mode and feeds input to os.Stdin for the duration of the test, so a // huh prompt reads a scripted answer instead of opening /dev/tty or blocking // on a real terminal. ENTIRE_TEST_TTY makes CanPromptInteractively report // true; ACCESSIBLE makes the form read os.Stdin rather than dial the terminal. func withInteractivePromptStdin(t *testing.T, input string) { t.Helper() t.Setenv("ENTIRE_TEST_TTY", "1") t.Setenv("ACCESSIBLE", "1") pr, pw, err := os.Pipe() if err != nil { t.Fatal(err) } t.Cleanup(func() { pr.Close() }) go func() { pw.WriteString(input) //nolint:errcheck // test helper pw.Close() }() old := os.Stdin os.Stdin = pr t.Cleanup(func() { os.Stdin = old }) }
// TestConfirmInitRepo_DefaultsToNo verifies that pressing Enter (empty
// input) at the init-repo prompt declines. entire enable is often run
// reflexively, so a stray run in a non-repo directory must not initialize
// a repo on the user's behalf. Regression guard for issue #1717.
func TestConfirmInitRepo_DefaultsToNo(t *testing.T) {
withInteractivePromptStdin(t, "\n")
proceed, err := confirmInitRepo(io.Discard, t.TempDir(), GitHubBootstrapOptions{}) if err != nil { t.Fatalf("unexpected error: %v", err) } if proceed { t.Fatal("confirmInitRepo should default to No (decline) on empty input") } }
// TestConfirmInitRepo_ExplicitYesProceeds verifies an explicit "y" still // opts in, so the safer default doesn't block intentional use. func TestConfirmInitRepo_ExplicitYesProceeds(t *testing.T) { withInteractivePromptStdin(t, "y\n")
proceed, err := confirmInitRepo(io.Discard, t.TempDir(), GitHubBootstrapOptions{}) if err != nil { t.Fatalf("unexpected error: %v", err) } if !proceed { t.Fatal("confirmInitRepo should proceed when the user explicitly answers yes") } }
// TestConfirmCreateGitHubRepo_DefaultsToNo verifies that pressing Enter at // the GitHub-repo prompt declines. Creating and pushing a remote repository // publishes the directory's contents, so it must never happen just because // the user pressed Enter. Regression guard for issue #1717. func TestConfirmCreateGitHubRepo_DefaultsToNo(t *testing.T) { withInteractivePromptStdin(t, "\n")
confirmed, err := confirmCreateGitHubRepo(t.TempDir()) if err != nil { t.Fatalf("unexpected error: %v", err) } if confirmed { t.Fatal("confirmCreateGitHubRepo should default to No on empty input") } }
// TestConfirmPushToRemote_DefaultsToNo verifies that pressing Enter at the // push prompt declines. Pushing publishes the directory's contents, so it // must never happen just because the user pressed Enter, even after they // opted into creating the repo. Regression guard for issue #1717. func TestConfirmPushToRemote_DefaultsToNo(t *testing.T) { withInteractivePromptStdin(t, "\n")
confirmed, err := confirmPushToRemote("octocat/example") if err != nil { t.Fatalf("unexpected error: %v", err) } if confirmed { t.Fatal("confirmPushToRemote should default to No on empty input") } }
// TestRunGitHubBootstrapFinalize_HonorsPushFalse verifies that finalize // respects state.push == false: the GitHub repo is still created and origin // configured, but nothing is pushed and the user is told how to publish // manually. The push decision (default No on Enter) is covered separately // by TestConfirmPushToRemote_DefaultsToNo; this test covers finalize honoring // that decision. func TestRunGitHubBootstrapFinalize_HonorsPushFalse(t *testing.T) { t.Parallel() dir := t.TempDir()
r := newFakeRunner() r.set("git", []string{"add", "-A"}, "", nil) r.set("git", []string{"status", "--porcelain"}, " M f\n", nil) r.set("git", []string{"-c", "commit.gpgsign=false", "commit", "-m", "Seed"}, "", nil) r.set("gh", []string{ "repo", "create", "octocat/no-push", "--private", "--source=.", "--remote=origin", }, "", nil)
s := &bootstrapState{ runner: r, cwd: dir, useGitHub: true, fullName: "octocat/no-push", visibility: "private", commit: true, message: "Seed", push: false, }
var out bytes.Buffer if err := runGitHubBootstrapFinalize(context.Background(), &out, s); err != nil { t.Fatalf("finalize failed: %v", err) }
// The repo is still created (create guard was accepted)... if !r.hasCall(argsMatch("gh", []string{"repo", "create"})) { t.Fatal("expected gh repo create to run") } // ...but the push guard was declined, so nothing is pushed. if r.hasCall(argsMatch("git", []string{"push"})) { t.Fatal("git push must not run when the push guard was declined") } if !strings.Contains(out.String(), "Skipped push") { t.Fatalf("expected 'Skipped push' guidance in output, got: %s", out.String()) } }
// restoreCwd chdirs into dir for the duration of the test. func restoreCwd(t *testing.T, dir string) { t.Helper() 89 unmodified lines
// When --yes is set, the name is taken, and a TTY is available, // resolveRepoName should print a conflict message and fall through // to the interactive prompt. We verify the conflict message was // printed (proving the fallback path was taken). t.Setenv("ENTIRE_TEST_TTY", "1")
// Force accessible (text-based) mode so the huh form reads from // os.Stdin instead of trying to open /dev/tty via bubbletea. // Pipe a unique name so the form completes instead of blocking. t.Setenv("ACCESSIBLE", "1") pr, pw, err := os.Pipe() if err != nil { t.Fatal(err) } t.Cleanup(func() { pr.Close() }) go func() { // The form reads one line; provide a unique name so it exits the loop. pw.WriteString("unique-test-repo\n") //nolint:errcheck // test helper pw.Close() }() oldStdin := os.Stdin os.Stdin = pr t.Cleanup(func() { os.Stdin = oldStdin }) // printed (proving the fallback path was taken). Pipe a unique name so // the form completes with it instead of blocking. withInteractivePromptStdin(t, "unique-test-repo\n")
dir := t.TempDir() restoreCwd(t, dir)
Mcmd/entire/cli/setup\_github\_test.go+222/-34
package strategy
import (
"context"
"encoding/base64"
"strings"
"testing"
"github.com/entireio/cli/cmd/entire/cli/agent"
"github.com/entireio/cli/cmd/entire/cli/agent/types"
"github.com/entireio/cli/cmd/entire/cli/transcript/imageextract"
)
// These tests exercise the opt-in image-externalization step in the condensation
// pipeline. They use t.Chdir / t.Setenv (process-global) to control the settings
// flag, so they cannot run in parallel.
// claudeImageLine returns a Claude Code user line embedding one inline base64
// image, plus the base64 string for assertions.
func claudeImageLine(t *testing.T, payload string) (line, b64 string) {
t.Helper()
b64 = base64.StdEncoding.EncodeToString([]byte(payload))
line = `{"type":"user","message":{"role":"user","content":[` +\
`{"type":"text","text":"look"},` +\
`{"type":"image","source":{"type":"base64","media_type":"image/png","data":"` + b64 + `"}}` +\
`]}}`
return line, b64
}
func TestExternalizeSessionImages_DisabledIsNoOp(t *testing.T) {
t.Chdir(t.TempDir()) // isolate settings; externalization defaults off
line, b64 := claudeImageLine(t, "disabled-noop-bytes-padded-long-enough-to-externalize")
raw := []byte(line + "\n")
state := &SessionState{SessionID: "s1", AgentType: agent.AgentTypeClaudeCode}
rewritten, assets := externalizeSessionImages(context.Background(), context.Background(), state, raw)
if assets != nil {
t.Errorf("expected no assets when flag off, got %d", len(assets))
}
if string(rewritten) != string(raw) {
t.Error("transcript must be unchanged when externalization is off")
}
if !strings.Contains(string(rewritten), b64) {
t.Error("base64 image should still be inline when externalization is off")
}
}
func TestExternalizeSessionImages_EnabledExtracts(t *testing.T) {
t.Chdir(t.TempDir())
t.Setenv("ENTIRE_EXTERNALIZE_IMAGES", "1")
line, b64 := claudeImageLine(t, "enabled-extract-bytes-padded-long-enough-to-externalize")
raw := []byte(line + "\n")
state := &SessionState{SessionID: "s2", AgentType: agent.AgentTypeClaudeCode}
rewritten, assets := externalizeSessionImages(context.Background(), context.Background(), state, raw)
if len(assets) != 1 {
t.Fatalf("expected 1 asset when flag on, got %d", len(assets))
}
if strings.Contains(string(rewritten), b64) {
t.Error("base64 image should be externalized out of the transcript")
}
if !strings.Contains(string(rewritten), "entire-asset:assets/") {
t.Error("transcript should carry a placeholder after externalization")
}
// The caller's raw transcript must be left untouched (growth-baseline / result).
if !strings.Contains(string(raw), b64) {
t.Error("the input transcript must not be mutated by externalization")
}
}
func TestExternalizeSessionImages_NonImageAgentIsNoOp(t *testing.T) {
t.Chdir(t.TempDir())
t.Setenv("ENTIRE_EXTERNALIZE_IMAGES", "1") // on, but agent has no codec
line, b64 := claudeImageLine(t, "codex-noop-bytes-padded-long-enough-to-externalize")
raw := []byte(line + "\n")
state := &SessionState{SessionID: "s3", AgentType: types.AgentType("Codex")}
rewritten, assets := externalizeSessionImages(context.Background(), context.Background(), state, raw)
if assets != nil {
t.Errorf("agent with no image codec should extract nothing, got %d assets", len(assets))
}
if string(rewritten) != string(raw) || !strings.Contains(string(rewritten), b64) {
t.Error("transcript must pass through unchanged for a no-codec agent")
}
}
// TestExtractThenRedact_ImageExternalizedSecretRedacted proves the mandatory
// ordering: on a line carrying BOTH a base64 image and a high-entropy secret,
// extracting first lifts the image into an asset (placeholder left behind), the
// redaction pass then strips the secret while leaving the low-entropy
// placeholder intact, and reinjection restores the exact image bytes. The stored
// (post-extract, post-redact) transcript therefore contains neither the raw
// image blob nor the secret.
func TestExtractThenRedact_ImageExternalizedSecretRedacted(t *testing.T) {
t.Parallel()
secret := "aB3xK9mQ7pL2wR8tY4vN6cF1gH5jD0sZeW7uI2oP"
b64 := base64.StdEncoding.EncodeToString([]byte("\x89PNG\r\n\x1a\nordering-fixture-bytes-padded-long-enough-to-externalize\x00\x01\x02"))
raw := []byte(`{"type":"user","message":{"role":"user","content":[` +\
`{"type":"text","text":"my token ` + secret + ` ok"},` +\
`{"type":"image","source":{"type":"base64","media_type":"image/png","data":"` + b64 + `"}}` +\
`]}}` + "\n")
codec := imageextract.CodecFor(agent.AgentTypeClaudeCode)
if codec == nil {
t.Fatal("expected a Claude Code image codec")
}
// Step 1: extract images (before redaction).
rewritten, assets, err := codec.ExtractImages(raw)
if err != nil {
t.Fatalf("ExtractImages() error = %v", err)
}
if len(assets) != 1 {
t.Fatalf("expected 1 asset, got %d", len(assets))
}
if strings.Contains(string(rewritten), b64) {
t.Error("image base64 should be gone after extraction")
}
if !strings.Contains(string(rewritten), secret) {
t.Error("secret must still be present pre-redaction")
}
// Step 2: redact the placeholder-bearing transcript.
redacted, err := redactSessionJSONLBytes(context.Background(), rewritten)
if err != nil {
t.Fatalf("redactSessionJSONLBytes() error = %v", err)
}
stored := string(redacted.Bytes())
if strings.Contains(stored, secret) {
t.Error("secret must be redacted out of the stored transcript")
}
if !strings.Contains(stored, "REDACTED") {
t.Error("expected a REDACTED marker where the secret was")
}
if !strings.Contains(stored, "entire-asset:assets/") {
t.Error("placeholder must survive redaction (low entropy)")
}
if strings.Contains(stored, b64) {
t.Error("stored transcript must not contain the raw image blob")
}
// Step 3: reinject restores the exact image bytes.
lookup := func(name string) (imageextract.Asset, bool) {
for _, a := range assets {
if a.Name == name {
return a, true
}
}
return imageextract.Asset{}, false
}
restored, err := codec.ReinjectImages(redacted.Bytes(), lookup)
if err != nil {
t.Fatalf("ReinjectImages() error = %v", err)
}
final := string(restored)
if !strings.Contains(final, b64) {
t.Error("image should be reinjected on restore")
}
if strings.Contains(final, "entire-asset:assets/") {
t.Error("no placeholder should remain after reinjection")
}
if strings.Contains(final, secret) {
t.Error("secret must stay redacted after reinjection")
}
}
Acmd/entire/cli/strategy/condense_images_test.go+165
6 unmodified lines
7
8
9
10
11
12
13
448 unmodified lines
462
463
464
464
465
466
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
6 unmodified lines
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"sync"
448 unmodified lines
if err != nil {
return "", fmt.Errorf("--absolute-git-hook-path: failed to resolve binary path: %w", err)
}
resolved, err := filepath.EvalSymlinks(exe)
resolved, err := resolveHookExePath(exe, filepath.EvalSymlinks, runtime.GOOS)
if err != nil {
return "", fmt.Errorf("--absolute-git-hook-path: failed to resolve symlinks for %s: %w", exe, err)
return "", err
}
return shellQuote(resolved), nil
}
return "entire", nil
}
// resolveHookExePath resolves exe through symlinks for embedding as an absolute
// path in a git hook. On Windows, filepath.EvalSymlinks can fail when a path
// component is an NTFS directory junction rather than a plain symlink — notably
// Scoop's `…\scoop\apps\<app>\current\` junction, which yields "The system
// cannot find the path specified" (issue #1424). The unresolved os.Executable()
// path is itself a valid, launchable absolute path (and on Scoop the stable
// `current\` junction path is actually preferable, since it survives version
// updates that repoint the junction), so on Windows we fall back to it rather
// than failing the hook install outright. Off Windows, an EvalSymlinks failure
// is unexpected and still surfaced as an error.
func resolveHookExePath(exe string, evalSymlinks func(string) (string, error), goos string) (string, error) {
resolved, err := evalSymlinks(exe)
if err != nil {
if goos == "windows" {
return exe, nil
}
return "", fmt.Errorf("--absolute-git-hook-path: failed to resolve symlinks for %s: %w", exe, err)
}
return resolved, nil
}
// shellQuote wraps a string in single quotes for safe use in #!/bin/sh scripts.
// Handles paths containing spaces, apostrophes, or other shell metacharacters
// (e.g., /Users/John O'Brien/bin/entire).
Mcmd/entire/cli/strategy/hooks.go+24/-2
1 unmodified line
2
3
4
5
6
7
8
1691 unmodified lines
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1 unmodified line
import (
"context"
"errors"
"os"
"os/exec"
"path/filepath"
1691 unmodified lines
t.Errorf("error should mention 'failed to remove hooks', got: %v", err)
}
}
// TestResolveHookExePath covers the absolute-git-hook-path symlink resolution,
// including the Windows fallback for NTFS junctions that EvalSymlinks cannot
// resolve (e.g. Scoop's `…\current\` junction — issue #1424). GOOS and the
// symlink resolver are injected so every branch runs on any host.
func TestResolveHookExePath(t *testing.T) {
t.Parallel()
const exe = `C:\Users\admin\scoop\apps\cli\current\entire.exe`
// Stand-in for the Windows junction error ("The system cannot find the path
// specified") that filepath.EvalSymlinks returns on Scoop's `current\`.
junctionErr := errors.New("cannot find the path specified")
t.Run("resolves normally when EvalSymlinks succeeds", func(t *testing.T) {
t.Parallel()
got, err := resolveHookExePath("/tmp/linkto", func(string) (string, error) {
return "/opt/entire/entire", nil
}, "linux")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got != "/opt/entire/entire" {
t.Errorf("got %q, want resolved target", got)
}
})
t.Run("windows falls back to unresolved path on EvalSymlinks failure", func(t *testing.T) {
t.Parallel()
got, err := resolveHookExePath(exe, func(string) (string, error) {
return "", junctionErr
}, "windows")
if err != nil {
t.Fatalf("windows should fall back, got error: %v", err)
}
if got != exe {
t.Errorf("got %q, want unresolved exe %q", got, exe)
}
})
t.Run("non-windows surfaces EvalSymlinks failure", func(t *testing.T) {
t.Parallel()
_, err := resolveHookExePath("/usr/local/bin/entire", func(string) (string, error) {
return "", junctionErr
}, "linux")
if err == nil {
t.Fatal("expected error on non-windows EvalSymlinks failure")
}
if !strings.Contains(err.Error(), "failed to resolve symlinks") {
t.Errorf("error should mention symlink resolution, got: %v", err)
}
})
}
Mcmd/entire/cli/strategy/hooks_test.go+53
25 unmodified lines
26
27
28
29
30
31
32
85 unmodified lines
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
98 unmodified lines
303
304
305
224
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
40 unmodified lines
365
366
367
368
369
370
371
25 unmodified lines
"github.com/entireio/cli/cmd/entire/cli/settings"
"github.com/entireio/cli/cmd/entire/cli/summarize"
"github.com/entireio/cli/cmd/entire/cli/transcript"
"github.com/entireio/cli/cmd/entire/cli/transcript/imageextract"
"github.com/entireio/cli/perf"
"github.com/entireio/cli/redact"
85 unmodified lines
return redact.JSONLBytes(b)
}
// extractSessionImages lifts inline base64 images out of a transcript into
// externalized assets via the per-agent image codec, returning the rewritten
// (placeholder-bearing) transcript. Agents with no codec, or transcripts with no
// externalizable images, pass through unchanged. Injectable for tests.
var extractSessionImages = func(agentType types.AgentType, transcript []byte) ([]byte, []cpkg.TranscriptAsset, error) {
codec := imageextract.CodecFor(agentType)
if codec == nil {
return transcript, nil, nil
}
rewritten, assets, err := codec.ExtractImages(transcript)
if err != nil {
return transcript, nil, fmt.Errorf("extract images: %w", err)
}
if len(assets) == 0 {
return transcript, nil, nil
}
out := make([]cpkg.TranscriptAsset, len(assets))
for i, a := range assets {
out[i] = cpkg.TranscriptAsset{Name: a.Name, MediaType: a.MediaType, Data: a.Data}
}
return rewritten, out, nil
}
// externalizeSessionImages runs the opt-in image-externalization step over a
// session transcript before redaction. When enabled it returns the rewritten
// (placeholder-bearing) transcript plus the extracted assets; when disabled,
// unsupported for the agent, or on error it returns the transcript unchanged with
// nil assets (the checkpoint then stores the inline transcript).
//
// It deliberately does NOT mutate the caller's transcript: the raw transcript is
// still needed for CondenseResult.Transcript (trail titles) and, critically, as
// the CheckpointTranscriptSize growth baseline, which is compared against the raw
// inline shadow-branch blob — feeding it the shrunken externalized size would
// report spurious growth on every subsequent commit.
func externalizeSessionImages(ctx, logCtx context.Context, state *SessionState, transcript []byte) ([]byte, []cpkg.TranscriptAsset) {
if !settings.IsImageExternalizationEnabled(ctx) {
return transcript, nil
}
rewritten, assets, err := extractSessionImages(state.AgentType, transcript)
if err != nil {
logging.Warn(logCtx, "image externalization failed; leaving transcript inline",
slog.String("session_id", state.SessionID),
slog.String("error", err.Error()))
return transcript, nil
}
return rewritten, assets
}
// sidecarSessionImages captures images an agent stores OUTSIDE the transcript
// (e.g. Cursor's per-session SQLite blob store) as checkpoint assets, so they are
// preserved with the session even though they never appear in full.jsonl. Unlike
// externalizeSessionImages there is no transcript placeholder and no round trip:
// these assets are preserve/view-only (the agent reads its own store on restore).
//
// Gated on the same opt-in flag. Best-effort: agents without the capability, or
// any capture error, yield no assets (the checkpoint is written without them).
func sidecarSessionImages(ctx, logCtx context.Context, ag agent.Agent, state *SessionState) []cpkg.TranscriptAsset {
if !settings.IsImageExternalizationEnabled(ctx) {
return nil
}
provider, ok := agent.AsSidecarImageProvider(ag)
if !ok {
return nil
}
assets, err := provider.SidecarImages(ctx, state.TranscriptPath)
if err != nil {
logging.Warn(logCtx, "sidecar image capture failed; checkpoint stored without them",
slog.String("session_id", state.SessionID),
slog.String("error", err.Error()))
return nil
}
if len(assets) == 0 {
return nil
}
out := make([]cpkg.TranscriptAsset, len(assets))
for i, a := range assets {
out[i] = cpkg.TranscriptAsset{Name: a.Name, MediaType: a.MediaType, Data: a.Data}
}
return out
}
// checkpointStepCount returns the number of user prompts attributed to the
// checkpoint being written: the turns counted since the current window's base.
// The base is re-anchored (deferred) the next time a turn is counted after a
98 unmodified lines
filterFilesTouched(sessionData, committedFiles, state)
redactedTranscript, redactDuration := redactOrDrop(logCtx, sessionData.Transcript, state.SessionID, checkpointID)
// Externalize inline images BEFORE redaction: base64 is high-entropy and
// redaction would otherwise flag/destroy it. Opt-in; a no-codec agent or a
// transcript with no externalizable images is a no-op. sessionData.Transcript
// is left as the raw transcript (used for the result / growth baseline); only
// the redacted, externalized copy is stored.
externalizedTranscript, extractedAssets := externalizeSessionImages(ctx, logCtx, state, sessionData.Transcript)
redactedTranscript, redactDuration := redactOrDrop(logCtx, externalizedTranscript, state.SessionID, checkpointID)
if skipped := skipIfPostRedactionEmpty(logCtx, redactedTranscript, sessionData, state, checkpointID); skipped != nil {
return skipped, nil
}
// Capture agent sidecar images (e.g. Cursor's SQLite store) after the skip
// check, so the sqlite3 shell-out is avoided when the checkpoint is discarded.
extractedAssets = append(extractedAssets, sidecarSessionImages(ctx, logCtx, ag, state)...)
store, err := s.getPersistentStore(ctx, repo)
if err != nil {
return nil, err
40 unmodified lines
Strategy: StrategyNameManualCommit,
Branch: branchName,
Transcript: redactedTranscript,
Assets: extractedAssets,
Prompts: sessionData.Prompts,
FilesTouched: sessionData.FilesTouched,
CheckpointsCount: checkpointStepCount(state),
Mcmd/entire/cli/strategy/manual_commit_condensation.go+95/-1
2844 unmodified lines
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
32 unmodified lines
2922
2923
2924
2886
2887
2888
2889
2890
2891
2892
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2844 unmodified lines
// Run the 7-layer pipeline over the transcript — OPF runs later in
// the pre-push rewrite path, which re-redacts these 7-layer blobs
// and produces 8-layer commits before the push goes out.
// Externalize inline images BEFORE redaction, mirroring CondenseSession, so the
// finalized (authoritative, full-session) transcript keeps its placeholders and
// matching assets instead of re-inlining what condensation lifted out. Opt-in;
// a no-codec agent or a transcript with no images is a no-op.
var finalizeAssets []checkpoint.TranscriptAsset
// Whether externalization actually RAN at finalize. When it did not (flag
// off here even though it may have been on at condensation — e.g. an
// ENTIRE_EXTERNALIZE_IMAGES env override not inherited by the hook
// process, or settings toggled mid-session), an empty finalizeAssets means
// "extraction didn't run", NOT "the transcript has no images" — clearing
// the previously-stored assets would permanently lose them (the re-inlined
// base64 is destroyed by redaction below).
externalizationRan := false
if settings.IsImageExternalizationEnabled(ctx) {
rewritten, assets, exErr := extractSessionImages(state.AgentType, fullTranscript)
if exErr != nil {
logging.Warn(logCtx, "finalize: image externalization failed; leaving transcript inline",
slog.String("session_id", state.SessionID),
slog.String("error", exErr.Error()),
)
} else {
fullTranscript = rewritten
finalizeAssets = assets
externalizationRan = true
}
}
// Re-capture sidecar images (e.g. Cursor's SQLite store) so a finalize that
// rewrites the transcript re-writes them too. When the full transcript differs
// from the stored (mid-turn) one, writeAssets clears the whole assets/ folder
// and re-writes only these assets — omitting the sidecar images here would drop
// what CondenseSession captured. Content-hash names make this idempotent with
// condensation's write; when the transcript is unchanged, writeAssets is not
// called and condensation's assets are left intact.
finalizeAssets = append(finalizeAssets, sidecarSessionImages(ctx, logCtx, ag, state)...)
// Sidecar capture is best-effort: a transient miss here (sqlite3 locked/timed
// out) yields no assets. For a sidecar-capable agent, preserve the assets a
// prior condensation stored rather than letting an empty set clear them.
_, sidecarCapable := agent.AsSidecarImageProvider(ag)
_, redactSpan := perf.Start(logCtx, "redact_transcript")
redactedTranscript, redactErr := redact.JSONLBytes(fullTranscript)
redactSpan.End()
32 unmodified lines
}
updateOpts := checkpoint.UpdateOptions{
CheckpointID: cpID,
SessionID: state.SessionID,
Transcript: redactedTranscript,
Prompts: prompts,
Agent: state.AgentType,
SkillEvents: skillEvents,
PrecomputedBlobs: precomputed,
CheckpointID: cpID,
SessionID: state.SessionID,
Transcript: redactedTranscript,
Assets: finalizeAssets,
PreserveAssetsWhenEmpty: sidecarCapable || !externalizationRan,
Prompts: prompts,
Agent: state.AgentType,
SkillEvents: skillEvents,
PrecomputedBlobs: precomputed,
}
updateErr := store.Write(ctx, checkpoint.SessionTranscript(updateOpts))
Mcmd/entire/cli/strategy/manual_commit_hooks.go+48/-7
30 unmodified lines
31
32
33
34
35
36
37
38
39
40
41
524 unmodified lines
566
567
568
569
570
571
572
573
574
575
576
577
578
63 unmodified lines
642
643
644
645
646
647
648
649
650
651
652
653
654
30 unmodified lines
"github.com/go-git/go-git/v6/storage"
)
// assetsDirName mirrors paths.AssetsDirName, captured at package scope so the
// OPF tree walkers can reference it even where a local variable named `paths`
// shadows the paths package (collectTreeBlobs).
const assetsDirName = paths.AssetsDirName
// V1DivergedError: local entire/checkpoints/v1 has commits that aren't
// ancestors of the remote tip (force-push or another machine pushed).
// Rewriting under divergence would silently rebase rejected work, so
524 unmodified lines
for _, e := range tree.Entries {
switch e.Mode {
case filemode.Dir:
// Externalized image assets are opaque binary lifted out of the
// (already redaction-skipped) transcript; byte-redacting them would
// only corrupt the images. Skip the whole subtree — symmetrically with
// rebuildTreeWithCachedRedaction, which preserves it verbatim.
if e.Name == assetsDirName {
continue
}
subPath := e.Name
if pathPrefix != "" {
subPath = pathPrefix + "/" + e.Name
63 unmodified lines
for _, e := range tree.Entries {
switch e.Mode {
case filemode.Dir:
// Preserve externalized image assets verbatim (see collectTreeBlobs):
// they are opaque binary and carry no redactable text. Copying the
// subtree hash keeps blobs byte-identical so restore still round-trips.
if e.Name == assetsDirName {
entries = append(entries, e)
continue
}
subPath := e.Name
if pathPrefix != "" {
subPath = pathPrefix + "/" + e.Name
Mcmd/entire/cli/strategy/manual_commit_opf_rewrite.go+19
566 unmodified lines
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
566 unmodified lines
require.False(t, collectedNames[paths.ContentHashFileName], "content_hash.txt must be excluded from collection (deferred path)")
}
// The OPF rewrite must not touch externalized image assets: byte-redacting the
// raw image blobs would corrupt them (breaking restore), and the fail-closed
// rebuild would abort the push if they were collected but not redacted. Both
// passes skip the assets/ subtree, preserving it verbatim.
func TestOPFRewrite_PreservesAssetsSubtreeVerbatim(t *testing.T) {
configureFakeOPF(t, &fakeOPFForRewrite{})
tempDir := t.TempDir()
testutil.InitRepo(t, tempDir)
repo, err := git.PlainOpen(tempDir)
require.NoError(t, err)
writeBlob := func(content []byte) plumbing.Hash {
obj := repo.Storer.NewEncodedObject()
obj.SetType(plumbing.BlobObject)
w, err := obj.Writer()
require.NoError(t, err)
_, err = w.Write(content)
require.NoError(t, err)
require.NoError(t, w.Close())
hash, err := repo.Storer.SetEncodedObject(obj)
require.NoError(t, err)
return hash
}
writeTree := func(entries []object.TreeEntry) plumbing.Hash {
tree := &object.Tree{Entries: entries}
obj := repo.Storer.NewEncodedObject()
require.NoError(t, tree.Encode(obj))
hash, err := repo.Storer.SetEncodedObject(obj)
require.NoError(t, err)
return hash
}
// Raw binary image (high-entropy: byte redaction would mangle it) + manifest.
imgBytes := []byte("\x89PNG\r\n\x1a\nPERSONABC-binary-image-bytes\x00\x01\x02\x03\xff\xfe")
imgHash := writeBlob(imgBytes)
manifestBytes := []byte(`{"version":1,"assets":[{"name":"img-abc.png"}]}` + "\n")
// Entries within a tree must be lexicographically ordered.
assetsTreeHash := writeTree([]object.TreeEntry{
{Name: "img-abc.png", Mode: filemode.Regular, Hash: imgHash},
{Name: "manifest.json", Mode: filemode.Regular, Hash: writeBlob(manifestBytes)},
})
fullHash := writeBlob([]byte(`{"type":"text","text":"hi"}` + "\n"))
tree := &object.Tree{Entries: []object.TreeEntry{
{Name: paths.AssetsDirName, Mode: filemode.Dir, Hash: assetsTreeHash},
{Name: paths.ContentHashFileName, Mode: filemode.Regular, Hash: writeBlob([]byte("sha256:abcd"))},
{Name: paths.TranscriptFileName, Mode: filemode.Regular, Hash: fullHash},
}}
// Collect pass: assets/ contents are excluded; full.jsonl is collected.
var blobs []redact.NamedBlob
var blobPaths []string
require.NoError(t, collectTreeBlobs(repo, tree, "", &blobs, &blobPaths))
collected := make(map[string]bool, len(blobs))
for _, b := range blobs {
collected[b.Name] = true
}
require.True(t, collected[paths.TranscriptFileName], "full.jsonl must be collected for redaction")
require.False(t, collected["img-abc.png"], "image asset must NOT be collected (would corrupt binary)")
require.False(t, collected["manifest.json"], "asset manifest must NOT be collected")
// Rebuild pass: must not fail-closed, and must preserve the assets subtree
// hash byte-for-byte (only full.jsonl gets redacted bytes from the map).
redactedByPath := map[string][]byte{paths.TranscriptFileName: []byte(`{"type":"text","text":"redacted"}` + "\n")}
newTreeHash, err := rebuildTreeWithCachedRedaction(repo, tree, "", redactedByPath)
require.NoError(t, err, "rebuild must not abort on the assets subtree")
newTree, err := repo.TreeObject(newTreeHash)
require.NoError(t, err)
var gotAssets plumbing.Hash
for _, e := range newTree.Entries {
if e.Name == paths.AssetsDirName {
gotAssets = e.Hash
}
}
require.Equal(t, assetsTreeHash, gotAssets, "assets subtree must be preserved verbatim")
// And the image blob inside is byte-identical.
rebuiltAssets, err := repo.TreeObject(gotAssets)
require.NoError(t, err)
imgFile, err := rebuiltAssets.File("img-abc.png")
require.NoError(t, err)
gotImg, err := imgFile.Contents()
require.NoError(t, err)
require.Equal(t, string(imgBytes), gotImg, "image bytes must survive the OPF rewrite unchanged")
}
// Fail-closed regression: when the OPF runtime fails and the breaker
// trips, the rewrite must NOT CAS the ref. Otherwise the new commits
// would carry Entire-OPF-Applied: true while their content is 7-layer
Mcmd/entire/cli/strategy/manual_commit_opf_rewrite_test.go+87
package imageextract
import ( "encoding/base64" "strings" "testing"
"github.com/entireio/cli/cmd/entire/cli/agent" )
// codexImageLine returns a real-format Codex user message embedding one inline
// image as a data-URI in an input_image content block (compact serialization,
// matching how the Codex rollout JSONL is written).
func codexImageLine(b64 string) string {
return {"type":"response_item","payload":{"type":"message","role":"user","content":[ +
{"type":"input_text","text":"<image name=[Image #1]>"}, +
{"type":"input_image","image_url":"data:image/png;base64, + b64 + "}, +
{"type":"input_text","text":"</image>"} +
]}}
}
// codexFunctionOutputLine embeds a data-URI inside function_call_output text,
// the way a screenshot/generated-image tool result appears in the rollout.
func codexFunctionOutputLine(b64 string) string {
return {"type":"response_item","payload":{"type":"function_call_output","call_id":"call_1", +
"output":"here is the render: data:image/png;base64, + b64 + done"}}
}
func codexPNG(payload string) string { return base64.StdEncoding.EncodeToString([]byte("\x89PNG\r\n\x1a\n" + payload + strings.Repeat("-codex-image-bytes", 3))) }
func codexJPEG(payload string) string { return base64.StdEncoding.EncodeToString([]byte("\xFF\xD8\xFF" + payload + strings.Repeat("-codex-image-bytes", 3))) }
// The core contract for Codex: extract then reinject reproduces the bytes exactly.
func TestCodexCodec_RoundTripByteExact(t *testing.T) {
t.Parallel()
c := CodecFor(agent.AgentTypeCodex)
if c == nil {
t.Fatal("expected a codec for Codex")
}
b64 := codexPNG("round-trip")
orig := codexImageLine(b64) + "\n" +
{"type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"ok"}]}} + "\n"
rewritten, assets, err := c.ExtractImages([]byte(orig)) if err != nil { t.Fatalf("ExtractImages: %v", err) } if len(assets) != 1 { t.Fatalf("expected 1 asset, got %d", len(assets)) } if assets[0].MediaType != mediaTypePNG { t.Errorf("media type = %q, want image/png", assets[0].MediaType) } if strings.Contains(string(rewritten), b64) { t.Error("base64 must be gone from the rewritten transcript") } // The data-URI prefix stays inline; only the base64 value became a placeholder. if !strings.Contains(string(rewritten), "data:image/png;base64,"+placeholderPrefix) { t.Error("expected the placeholder to sit inside the data-URI, prefix preserved") }
restored, err := c.ReinjectImages(rewritten, lookupFrom(assets)) if err != nil { t.Fatalf("ReinjectImages: %v", err) } if string(restored) != orig { t.Fatalf("round trip not byte-exact:\n got: %s\nwant: %s", restored, orig) } }
// A data-URI embedded in function_call_output text round-trips too. func TestCodexCodec_FunctionOutputDataURIRoundTrips(t *testing.T) { t.Parallel() c := CodecFor(agent.AgentTypeCodex) b64 := codexPNG("tool-output") orig := codexFunctionOutputLine(b64) + "\n"
rewritten, assets, err := c.ExtractImages([]byte(orig)) if err != nil { t.Fatalf("ExtractImages: %v", err) } if len(assets) != 1 { t.Fatalf("expected 1 asset, got %d", len(assets)) } if strings.Contains(string(rewritten), b64) { t.Error("base64 in tool output should be externalized") } restored, err := c.ReinjectImages(rewritten, lookupFrom(assets)) if err != nil { t.Fatalf("ReinjectImages: %v", err) } if string(restored) != orig { t.Fatal("function_call_output round trip not byte-exact") } }
// A single message with many images (the real Codex case) round-trips, each a
// distinct asset.
func TestCodexCodec_MultipleImagesOneMessage(t *testing.T) {
t.Parallel()
c := CodecFor(agent.AgentTypeCodex)
b1, b2, b3 := codexPNG("one"), codexJPEG("two"), codexPNG("three")
orig := {"type":"response_item","payload":{"type":"message","role":"user","content":[ +
{"type":"input_image","image_url":"data:image/png;base64, + b1 + "}, +
{"type":"input_image","image_url":"data:image/jpeg;base64, + b2 + "}, +
{"type":"input_image","image_url":"data:image/png;base64, + b3 + "} +
]}} + "\n"
rewritten, assets, err := c.ExtractImages([]byte(orig)) if err != nil { t.Fatalf("ExtractImages: %v", err) } if len(assets) != 3 { t.Fatalf("expected 3 assets, got %d", len(assets)) } // jpeg maps to .jpg extension. var sawJPG bool for _, a := range assets { if strings.HasSuffix(a.Name, ".jpg") { sawJPG = true } } if !sawJPG { t.Error("expected a .jpg asset from the image/jpeg data-URI") } restored, err := c.ReinjectImages(rewritten, lookupFrom(assets)) if err != nil { t.Fatalf("ReinjectImages: %v", err) } if string(restored) != orig { t.Fatal("multi-image round trip not byte-exact") } }
// Identical images dedupe to one asset but round-trip both occurrences. func TestCodexCodec_DedupesIdenticalImages(t *testing.T) { t.Parallel() c := CodecFor(agent.AgentTypeCodex) b64 := codexPNG("same") orig := codexImageLine(b64) + "\n" + codexImageLine(b64) + "\n" rewritten, assets, err := c.ExtractImages([]byte(orig)) if err != nil { t.Fatalf("ExtractImages: %v", err) } if len(assets) != 1 { t.Fatalf("identical images should dedupe to 1 asset, got %d", len(assets)) } restored, err := c.ReinjectImages(rewritten, lookupFrom(assets)) if err != nil { t.Fatalf("ReinjectImages: %v", err) } if string(restored) != orig { t.Fatal("dedup round trip not byte-exact") } }
// A text-only Codex transcript is a no-op.
func TestCodexCodec_NoImagesIsNoOp(t *testing.T) {
t.Parallel()
c := CodecFor(agent.AgentTypeCodex)
orig := {"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"hi"}]}} + "\n"
rewritten, assets, err := c.ExtractImages([]byte(orig))
if err != nil {
t.Fatalf("ExtractImages: %v", err)
}
if assets != nil {
t.Errorf("expected no assets, got %d", len(assets))
}
if string(rewritten) != orig {
t.Error("text-only transcript should be unchanged")
}
}
// A tiny data-URI (below the externalize threshold) is left inline. func TestCodexCodec_LeavesTinyDataURIInline(t *testing.T) { t.Parallel() c := CodecFor(agent.AgentTypeCodex) tiny := base64.StdEncoding.EncodeToString([]byte("tiny")) if len(tiny) >= minExternalizedBase64Len { t.Fatalf("fixture too long: %d", len(tiny)) } orig := codexImageLine(tiny) + "\n" rewritten, assets, err := c.ExtractImages([]byte(orig)) if err != nil { t.Fatalf("ExtractImages: %v", err) } if len(assets) != 0 || string(rewritten) != orig { t.Errorf("tiny data-URI must be left inline; assets=%d changed=%v", len(assets), string(rewritten) != orig) } }
// Ordering: a Codex line carrying both a secret and an image data-URI —
// extraction lifts the image, leaving the secret for the redaction pass, and the
// image reinjects cleanly.
func TestCodexCodec_ExtractLeavesSecretForRedaction(t *testing.T) {
t.Parallel()
c := CodecFor(agent.AgentTypeCodex)
secret := "aB3xK9mQ7pL2wR8tY4vN6cF1gH5jD0sZeW7uI2oP"
b64 := codexPNG("secret-plus-image")
orig := {"type":"response_item","payload":{"type":"message","role":"user","content":[ +
{"type":"input_text","text":"token + secret + "}, +
{"type":"input_image","image_url":"data:image/png;base64, + b64 + "} +
]}} + "\n"
rewritten, assets, err := c.ExtractImages([]byte(orig)) if err != nil { t.Fatalf("ExtractImages: %v", err) } if len(assets) != 1 { t.Fatalf("expected 1 asset, got %d", len(assets)) } if strings.Contains(string(rewritten), b64) { t.Error("image should be externalized") } if !strings.Contains(string(rewritten), secret) { t.Error("the secret must remain for the downstream redaction pass") } restored, err := c.ReinjectImages(rewritten, lookupFrom(assets)) if err != nil { t.Fatalf("ReinjectImages: %v", err) } if string(restored) != orig { t.Fatal("round trip not byte-exact") } }
Acmd/entire/cli/transcript/imageextract/codex\_test.go+229
// Package imageextract externalizes inline base64 images from an agent's session
// transcript into a checkpoint asset store, replacing each with a compact,
// path-bearing placeholder, and re-injects them byte-exactly on restore.
//
// The transform is per-agent because transcript formats differ: only agents that
// inline base64 images (Claude Code and Codex today) register a codec; every
// other agent resolves to nil and its transcript flows through untouched (a
// graceful no-op). All codecs share one extraction engine and reinjection routine
// and differ only in how they *find* images (their collector).
//
// Correctness contract: for any transcript x from a supported agent,
// ReinjectImages(ExtractImages(x)) == x, byte-for-byte. This is achieved by only
// ever swapping the base64 image *value* in place (never re-marshalling the JSON)
// and by refusing to externalize any image whose raw bytes don't re-encode to the
// exact original base64 string.
package imageextract
import (
"bytes"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"regexp"
"sort"
"strconv"
"github.com/entireio/cli/cmd/entire/cli/agent"
"github.com/entireio/cli/cmd/entire/cli/agent/types"
)
// Asset is one externalized image. It reuses the canonical agent asset model;
// Name is the stable asset filename (also the id used in the placeholder).
type Asset = agent.CompactedTranscriptAsset
// ImageCodec extracts/reinjects inline images for one agent's transcript format.
type ImageCodec interface {
// ExtractImages lifts inline base64 images out, returning the rewritten
// (placeholder-bearing) transcript plus the extracted assets. A transcript
// with no externalizable images is returned unchanged with nil assets.
ExtractImages(transcript []byte) (rewritten []byte, assets []Asset, err error)
// ReinjectImages restores the original transcript by looking each placeholder's
// asset up by Name. Placeholders whose asset is missing are left in place.
ReinjectImages(transcript []byte, lookup func(name string) (Asset, bool)) ([]byte, error)
}
// placeholderPrefix leads every externalized-image reference. It is deliberately
// low-entropy so the (later) redaction pass never flags it, and it carries the
// asset's path so an agent summarizing the stored log still understands an image
// was here and where it lives.
const placeholderPrefix = "entire-asset:assets/"
// placeholderRe matches a full placeholder and captures the asset name.
var placeholderRe = regexp.MustCompile(`entire-asset:assets/(img-[0-9a-f]+\.[a-z0-9]+)`)
// newAssetID returns a random hex id (16 bytes → 32 hex chars). Hex is ~4
// bits/char, below the redaction entropy threshold, so placeholders survive
// redaction. The rand error is surfaced (never swallowed) so a broken entropy
// source can't silently yield an all-zero, collision-prone id. Injectable for
// deterministic tests.
var newAssetID = func() (string, error) {
b := make([]byte, 16)
if _, err := rand.Read(b); err != nil {
return "", fmt.Errorf("generate asset id: %w", err)
}
return hex.EncodeToString(b), nil
}
// uniqueImageName returns an asset name not already in used, recording it. It
// guarantees the "distinct image data -> distinct asset name" invariant the
// byte-exact round trip relies on: two assets sharing a name would make
// ReinjectImages restore both placeholders to whichever the lookup returns first.
// With crypto/rand collisions never happen; the hex-counter suffix guarantees
// termination even if a caller injects a degenerate id source.
func uniqueImageName(used map[string]bool, mediaType string) (string, error) {
ext := extForMedia(mediaType)
id, err := newAssetID()
if err != nil {
return "", err
}
name := "img-" + id + "." + ext
for suffix := 0; used[name]; suffix++ {
name = "img-" + id + strconv.FormatInt(int64(suffix), 16) + "." + ext
}
used[name] = true
return name, nil
}
// minExternalizedBase64Len is the shortest base64 image value we externalize.
// It must exceed the 32-char random-hex run in a placeholder so that an
// externalized value can never be a substring of any placeholder — that is the
// single condition under which the in-place value swap could corrupt a
// placeholder and break the byte-exact round trip. As a bonus it leaves tiny
// blobs (which are never real images) inline, where they cost nothing.
const minExternalizedBase64Len = 64
// maxExternalizedImageBytes is the largest decoded image we externalize. Above
// it the image stays inline: writeAssets stores each asset as a single git blob,
// so one over agent.MaxChunkSize would become an unpushable object — the same
// guard the Cursor sidecar path applies. The transcript itself is chunked under
// MaxChunkSize, so an oversized image left inline still pushes fine. It is a var
// (not a const) only so tests can lower it without allocating a 50MB fixture.
var maxExternalizedImageBytes = agent.MaxChunkSize
var codecs = map[types.AgentType]ImageCodec{
agent.AgentTypeClaudeCode: claudeCodec{},
agent.AgentTypeCodex: codexCodec{},
}
// CodecFor returns the image codec for an agent type, or nil if the agent's
// transcript is not known to inline images (a no-op).
func CodecFor(t types.AgentType) ImageCodec { return codecs[t] }
// HasPlaceholders reports whether a transcript carries any externalized-image
// placeholders — so restore knows to reinject regardless of the current config.
func HasPlaceholders(transcript []byte) bool {
return bytes.Contains(transcript, []byte(placeholderPrefix))
}
// imgHit is one image found by a collector: the bare base64 value to swap out and
// its declared media type (used only to pick the asset filename extension).
type imgHit struct{ data, mediaType string }
// extractImagesWith is the shared extraction engine. It parses each JSONL line,
// gathers image hits via the per-agent collector, dedupes, decodes and re-encodes
// to confirm the base64 is byte-exactly reversible, then swaps each value out for
// a placeholder — longest value first (so a value that is a substring of another
// can't orphan it) and only recording assets whose swap actually replaced bytes.
func extractImagesWith(transcript []byte, collect func(v any, out *[]imgHit)) ([]byte, []Asset, error) {
if len(transcript) == 0 {
return transcript, nil, nil
}
// Map each unique base64 image value → its asset (dedupes repeats within the
// transcript; git dedupes identical blobs across checkpoints by content).
seen := map[string]Asset{}
var order []string // unique base64 values, later sorted longest-first
usedNames := map[string]bool{} // guards distinct-data -> distinct-name
for _, line := range bytes.Split(transcript, []byte("\n")) {
trimmed := bytes.TrimSpace(line)
// Accept object- and array-rooted JSON lines; the collectors walk both.
if len(trimmed) == 0 || (trimmed[0] != '{' && trimmed[0] != '[') {\
continue\
}\
var v any\
if err := json.Unmarshal(trimmed, &v); err != nil {\
continue // non-JSON line; leave untouched\
}\
var hits []imgHit\
collect(v, &hits)\
for _, h := range hits {\
if len(h.data) < minExternalizedBase64Len {\
continue // too small to be a real image; also keeps it out of placeholders\
}\
if _, ok := seen[h.data]; ok {\
continue\
}\
raw, err := base64.StdEncoding.DecodeString(h.data)\
if err != nil {\
continue // not standard base64; leave inline\
}\
// Only externalize if re-encoding reproduces the exact original string;\
// otherwise the restore round-trip could not be byte-exact.\
if base64.StdEncoding.EncodeToString(raw) != h.data {\
continue\
}\
// Leave oversized images inline. Each asset is stored as one git blob,\
// and a blob over MaxChunkSize would be unpushable; the transcript, by\
// contrast, is chunked under that limit, so keeping the image inline\
// stays pushable (mirrors the Cursor sidecar guard).\
if len(raw) > maxExternalizedImageBytes {\
continue\
}\
// Prefer the media type detected from the actual bytes over the declared\
// one: agents mislabel it (real Codex data-URIs declare image/jpeg for\
// PNG bytes), and the asset filename/manifest should reflect the content.\
// This is metadata only — the transcript's declared type is untouched, so\
// the round trip stays byte-exact.\
mediaType := detectMediaType(raw)\
if mediaType == "" {\
mediaType = h.mediaType\
}\
name, err := uniqueImageName(usedNames, mediaType)\
if err != nil {\
return transcript, nil, err\
}\
seen[h.data] = Asset{Name: name, MediaType: mediaType, Data: raw}\
order = append(order, h.data)\
}\
}\
\
if len(order) == 0 {\
return transcript, nil, nil\
}\
\
// Replace longest values first so that if one image's base64 is a substring of\
// another's, the containing (longer) value is swapped out before the shorter\
// one, keeping every asset's placeholder present. Ties broken by value for\
// determinism.\
sort.SliceStable(order, func(i, j int) bool {\
if len(order[i]) != len(order[j]) {\
return len(order[i]) > len(order[j])\
}\
return order[i] < order[j]\
})\
\
rewritten := transcript\
assets := make([]Asset, 0, len(order))\
for _, data := range order {\
a := seen[data]\
// Swap the base64 value itself, not a field wrapper. Agents serialize the\
// enclosing JSON differently and across versions (compact vs spaced, string\
// vs object image_url, data-URI vs bare), so the bare value is the only\
// format-agnostic anchor. This can also swap a copy of the same base64 in a\
// text field, but the round trip stays byte-exact because ReinjectImages\
// restores every occurrence to the identical value.\
swapped := bytes.ReplaceAll(rewritten, []byte(data), []byte(placeholderPrefix+a.Name))\
if bytes.Equal(swapped, rewritten) {\
// Exact bytes weren't present (e.g. JSON-escaped in the raw transcript);\
// leave the image inline rather than record an asset no placeholder\
// references.\
continue\
}\
rewritten = swapped\
assets = append(assets, a)\
}\
if len(assets) == 0 {\
return transcript, nil, nil\
}\
return rewritten, assets, nil\
}\
\
// reinjectImages restores every placeholder to its asset's base64. It is shared\
// by all codecs: the placeholder token is agent-independent, so restore only\
// needs the asset lookup.\
func reinjectImages(transcript []byte, lookup func(name string) (Asset, bool)) ([]byte, error) {\
if !HasPlaceholders(transcript) {\
return transcript, nil\
}\
result := transcript\
done := map[string]bool{}\
for _, m := range placeholderRe.FindAllSubmatch(transcript, -1) {\
full, name := m[0], string(m[1])\
if done[name] {\
continue\
}\
done[name] = true\
a, ok := lookup(name)\
if !ok {\
continue // asset unavailable; leave the placeholder (best-effort)\
}\
result = bytes.ReplaceAll(result, full, []byte(base64.StdEncoding.EncodeToString(a.Data)))\
}\
return result, nil\
}\
\
// claudeCodec handles Claude Code (and, structurally, Cursor) JSONL transcripts,\
// which embed images as {"type":"image","source":{"type":"base64","media_type":…,"data":…}}.\
type claudeCodec struct{}\
\
func (claudeCodec) ExtractImages(transcript []byte) ([]byte, []Asset, error) {\
return extractImagesWith(transcript, collectClaudeImages)\
}\
\
func (claudeCodec) ReinjectImages(transcript []byte, lookup func(name string) (Asset, bool)) ([]byte, error) {\
return reinjectImages(transcript, lookup)\
}\
\
// codexCodec handles OpenAI Codex rollout JSONL transcripts, which embed images\
// as base64 data-URIs (data:image/<type>;base64,<data>) — in input_image\
// image_url values, user messages, and function_call_output content alike.\
type codexCodec struct{}\
\
func (codexCodec) ExtractImages(transcript []byte) ([]byte, []Asset, error) {\
return extractImagesWith(transcript, collectCodexImages)\
}\
\
func (codexCodec) ReinjectImages(transcript []byte, lookup func(name string) (Asset, bool)) ([]byte, error) {\
return reinjectImages(transcript, lookup)\
}\
\
// collectClaudeImages walks any decoded JSON value and gathers every inline\
// base64 image block, at any nesting depth (top-level content, tool_result\
// content, etc.).\
func collectClaudeImages(v any, out *[]imgHit) {\
switch t := v.(type) {\
case map[string]any:\
if t["type"] == "image" {\
if src, ok := t["source"].(map[string]any); ok && src["type"] == "base64" {\
if data, ok := src["data"].(string); ok && data != "" {\
var mediaType string\
if mt, mok := src["media_type"].(string); mok {\
mediaType = mt\
}\
*out = append(*out, imgHit{data: data, mediaType: mediaType})\
}\
}\
}\
for _, vv := range t {\
collectClaudeImages(vv, out)\
}\
case []any:\
for _, vv := range t {\
collectClaudeImages(vv, out)\
}\
}\
}\
\
// codexDataURIRe matches an image data-URI and captures (media subtype, base64).\
// Codex serializes every inline image this way — input_image image_url values,\
// user-message content, and function_call_output payloads — so keying on the\
// data-URI (rather than a specific field) covers input and generated/tool images\
// uniformly. The captured group is the bare base64, which is what gets swapped.\
var codexDataURIRe = regexp.MustCompile(`data:image/([a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/]+={0,2})`)\
\
// collectCodexImages walks any decoded JSON value and, for each string leaf,\
// gathers every image data-URI it contains (a leaf may be the URI itself, e.g.\
// input_image.image_url, or embed one inside larger tool output).\
func collectCodexImages(v any, out *[]imgHit) {\
switch t := v.(type) {\
case map[string]any:\
for _, vv := range t {\
collectCodexImages(vv, out)\
}\
case []any:\
for _, vv := range t {\
collectCodexImages(vv, out)\
}\
case string:\
for _, m := range codexDataURIRe.FindAllStringSubmatch(t, -1) {\
*out = append(*out, imgHit{data: m[2], mediaType: "image/" + m[1]})\
}\
}\
}\
\
const (\
mediaTypePNG = "image/png"\
mediaTypeJPEG = "image/jpeg"\
mediaTypeGIF = "image/gif"\
mediaTypeWEBP = "image/webp"\
)\
\
// detectMediaType returns the image media type implied by the leading magic\
// bytes, or "" if unrecognized (caller falls back to the declared type).\
func detectMediaType(raw []byte) string {\
switch {\
case bytes.HasPrefix(raw, []byte("\x89PNG\r\n\x1a\n")):\
return mediaTypePNG\
case bytes.HasPrefix(raw, []byte{0xFF, 0xD8, 0xFF}):\
return mediaTypeJPEG\
case bytes.HasPrefix(raw, []byte("GIF87a")), bytes.HasPrefix(raw, []byte("GIF89a")):\
return mediaTypeGIF\
case len(raw) >= 12 && bytes.HasPrefix(raw, []byte("RIFF")) && bytes.Equal(raw[8:12], []byte("WEBP")):\
return mediaTypeWEBP\
default:\
return ""\
}\
}\
\
func extForMedia(mediaType string) string {\
switch mediaType {\
case mediaTypePNG:\
return "png"\
case mediaTypeJPEG:\
return "jpg"\
case mediaTypeGIF:\
return "gif"\
case mediaTypeWEBP:\
return "webp"\
default:\
return "bin"\
}\
}\
```\
\
Acmd/entire/cli/transcript/imageextract/imageextract.go+375\
\
```\
1\
2\
3\
4\
5\
6\
7\
8\
9\
10\
11\
12\
13\
14\
15\
16\
17\
18\
19\
20\
21\
22\
23\
24\
25\
26\
27\
28\
29\
30\
31\
32\
33\
34\
35\
36\
37\
38\
39\
40\
41\
42\
43\
44\
45\
46\
47\
48\
49\
50\
51\
52\
53\
54\
55\
56\
57\
58\
59\
60\
61\
62\
63\
64\
65\
66\
67\
68\
69\
70\
71\
72\
73\
74\
75\
76\
77\
78\
79\
80\
81\
82\
83\
84\
85\
86\
87\
88\
89\
90\
91\
92\
93\
94\
95\
96\
97\
98\
99\
100\
101\
102\
103\
104\
105\
106\
107\
108\
109\
110\
111\
112\
113\
114\
115\
116\
117\
118\
119\
120\
121\
122\
123\
124\
125\
126\
127\
128\
129\
130\
131\
132\
133\
134\
135\
136\
137\
138\
139\
140\
141\
142\
143\
144\
145\
146\
147\
148\
149\
150\
151\
152\
153\
154\
155\
156\
157\
158\
159\
160\
161\
162\
163\
164\
165\
166\
167\
168\
169\
170\
171\
172\
173\
174\
175\
176\
177\
178\
179\
180\
181\
182\
183\
184\
185\
186\
187\
188\
189\
190\
191\
192\
193\
194\
195\
196\
197\
198\
199\
200\
201\
202\
203\
204\
205\
206\
207\
208\
209\
210\
211\
212\
213\
214\
215\
216\
217\
218\
219\
220\
221\
222\
223\
224\
225\
226\
227\
228\
229\
230\
231\
232\
233\
234\
235\
236\
237\
238\
239\
240\
241\
242\
243\
244\
245\
246\
247\
248\
249\
250\
251\
252\
253\
254\
255\
256\
257\
258\
259\
260\
261\
262\
263\
264\
265\
266\
267\
268\
269\
270\
271\
272\
273\
274\
275\
276\
277\
278\
279\
280\
281\
282\
283\
284\
285\
286\
287\
288\
289\
290\
291\
292\
293\
294\
295\
296\
297\
298\
299\
300\
301\
302\
303\
304\
305\
306\
307\
308\
309\
310\
311\
312\
313\
314\
315\
316\
317\
318\
319\
320\
321\
322\
323\
324\
325\
326\
327\
328\
329\
330\
331\
332\
333\
334\
335\
336\
337\
338\
339\
340\
341\
342\
343\
344\
345\
346\
347\
348\
349\
350\
351\
352\
353\
354\
355\
356\
357\
358\
359\
360\
361\
362\
363\
364\
365\
366\
367\
368\
369\
370\
371\
372\
373\
374\
375\
376\
377\
378\
379\
380\
381\
382\
383\
384\
385\
386\
387\
388\
389\
390\
391\
392\
393\
394\
395\
396\
397\
398\
399\
400\
401\
\
package imageextract\
\
import (\
"encoding/base64"\
"errors"\
"math"\
"regexp"\
"strings"\
"testing"\
\
"github.com/entireio/cli/cmd/entire/cli/agent"\
"github.com/entireio/cli/cmd/entire/cli/agent/types"\
)\
\
var errTestRand = errors.New("simulated rand failure")\
\
func lookupFrom(assets []Asset) func(string) (Asset, bool) {\
return func(name string) (Asset, bool) {\
for _, a := range assets {\
if a.Name == name {\
return a, true\
}\
}\
return Asset{}, false\
}\
}\
\
func claudeLine(b64 string) string {\
return `{"type":"user","message":{"role":"user","content":[` +\
`{"type":"text","text":"look at this"},` +\
`{"type":"image","source":{"type":"base64","media_type":"image/png","data":"` + b64 + `"}}` +\
`]}}`\
}\
\
// The core contract: extract then reinject reproduces the original bytes exactly.\
func TestClaudeCodec_RoundTripByteExact(t *testing.T) {\
t.Parallel()\
c := CodecFor(agent.AgentTypeClaudeCode)\
if c == nil {\
t.Fatal("expected a codec for Claude Code")\
}\
b64 := base64.StdEncoding.EncodeToString([]byte("\x89PNG\r\n\x1a\nfake-png-bytes-with-enough-length-to-be-a-real-image\x00\x01\x02"))\
orig := claudeLine(b64) + "\n{\"type\":\"assistant\",\"message\":{\"content\":[{\"type\":\"text\",\"text\":\"ok\"}]}}\n"\
\
rewritten, assets, err := c.ExtractImages([]byte(orig))\
if err != nil {\
t.Fatalf("ExtractImages: %v", err)\
}\
if len(assets) != 1 {\
t.Fatalf("expected 1 asset, got %d", len(assets))\
}\
if strings.Contains(string(rewritten), b64) {\
t.Error("base64 must be gone from the rewritten transcript")\
}\
if !strings.Contains(string(rewritten), placeholderPrefix) {\
t.Error("rewritten transcript should carry a placeholder")\
}\
if assets[0].MediaType != mediaTypePNG {\
t.Errorf("asset media type = %q, want image/png", assets[0].MediaType)\
}\
\
restored, err := c.ReinjectImages(rewritten, lookupFrom(assets))\
if err != nil {\
t.Fatalf("ReinjectImages: %v", err)\
}\
if string(restored) != orig {\
t.Fatalf("round-trip not byte-exact:\n got: %s\nwant: %s", restored, orig)\
}\
}\
\
// A transcript with no images is returned unchanged with no assets.\
func TestClaudeCodec_NoImagesIsNoOp(t *testing.T) {\
t.Parallel()\
c := CodecFor(agent.AgentTypeClaudeCode)\
orig := `{"type":"user","message":{"role":"user","content":[{"type":"text","text":"hi"}]}}` + "\n"\
rewritten, assets, err := c.ExtractImages([]byte(orig))\
if err != nil {\
t.Fatalf("ExtractImages: %v", err)\
}\
if assets != nil {\
t.Errorf("expected no assets, got %d", len(assets))\
}\
if string(rewritten) != orig {\
t.Errorf("no-image transcript should be unchanged")\
}\
}\
\
// Identical images dedupe to one asset but round-trip both occurrences.\
func TestClaudeCodec_DedupesIdenticalImages(t *testing.T) {\
t.Parallel()\
c := CodecFor(agent.AgentTypeClaudeCode)\
b64 := base64.StdEncoding.EncodeToString([]byte("same-image-bytes-repeated-with-enough-length-to-externalize"))\
orig := claudeLine(b64) + "\n" + claudeLine(b64) + "\n"\
rewritten, assets, err := c.ExtractImages([]byte(orig))\
if err != nil {\
t.Fatalf("ExtractImages: %v", err)\
}\
if len(assets) != 1 {\
t.Fatalf("identical images should dedupe to 1 asset, got %d", len(assets))\
}\
restored, err := c.ReinjectImages(rewritten, lookupFrom(assets))\
if err != nil {\
t.Fatalf("ReinjectImages: %v", err)\
}\
if string(restored) != orig {\
t.Fatalf("round-trip mismatch for duplicated image")\
}\
}\
\
// When one image's base64 is a substring of another's, the round trip must still\
// be byte-exact (longest-first replacement guarantees this).\
func TestClaudeCodec_SubstringImagesRoundTrip(t *testing.T) {\
t.Parallel()\
c := CodecFor(agent.AgentTypeClaudeCode)\
long := base64.StdEncoding.EncodeToString([]byte(\
"prefix-bytes-AAAABBBBCCCCDDDD-and-a-considerably-longer-image-tail-payload-so-a-64-char-substring-fits-xyz"))\
// A canonical base64 substring of long (>= threshold) that decodes/re-encodes\
// cleanly, taken from a non-zero offset so it is genuinely embedded.\
var short string\
for i := 4; i+64 <= len(long); i += 4 {\
cand := long[i : i+64]\
if raw, err := base64.StdEncoding.DecodeString(cand); err == nil && base64.StdEncoding.EncodeToString(raw) == cand {\
short = cand\
break\
}\
}\
if short == "" {\
t.Fatal("could not construct a canonical base64 substring")\
}\
// Shorter block first, so first-seen order would (without the sort) replace it\
// before the containing longer value.\
orig := claudeLine(short) + "\n" + claudeLine(long) + "\n"\
rewritten, assets, err := c.ExtractImages([]byte(orig))\
if err != nil {\
t.Fatalf("ExtractImages: %v", err)\
}\
if len(assets) != 2 {\
t.Fatalf("expected 2 assets, got %d", len(assets))\
}\
// Longest-first replacement means both assets have a live placeholder (neither\
// is orphaned by the other's swap).\
for _, a := range assets {\
if !strings.Contains(string(rewritten), placeholderPrefix+a.Name) {\
t.Errorf("asset %s has no placeholder in the rewritten transcript (orphaned)", a.Name)\
}\
}\
restored, err := c.ReinjectImages(rewritten, lookupFrom(assets))\
if err != nil {\
t.Fatalf("ReinjectImages: %v", err)\
}\
if string(restored) != orig {\
t.Fatalf("substring round-trip not byte-exact:\n got: %s\nwant: %s", restored, orig)\
}\
}\
\
// Even if the id source degenerates to a constant, distinct images must still get\
// distinct names so the round trip stays byte-exact (no asset shadows another).\
func TestClaudeCodec_DistinctNamesUnderCollidingIDSource(t *testing.T) {\
c := CodecFor(agent.AgentTypeClaudeCode)\
orig := newAssetID\
newAssetID = func() (string, error) { return "deadbeefdeadbeefdeadbeefdeadbeef", nil } // constant\
defer func() { newAssetID = orig }()\
\
img1 := base64.StdEncoding.EncodeToString([]byte("first-distinct-image-payload-long-enough-to-externalize"))\
img2 := base64.StdEncoding.EncodeToString([]byte("second-distinct-image-payload-long-enough-to-externalize"))\
in := claudeLine(img1) + "\n" + claudeLine(img2) + "\n"\
\
rewritten, assets, err := c.ExtractImages([]byte(in))\
if err != nil {\
t.Fatalf("ExtractImages: %v", err)\
}\
if len(assets) != 2 {\
t.Fatalf("want 2 assets, got %d", len(assets))\
}\
if assets[0].Name == assets[1].Name {\
t.Fatalf("distinct images got the same name %q", assets[0].Name)\
}\
restored, err := c.ReinjectImages(rewritten, lookupFrom(assets))\
if err != nil {\
t.Fatalf("ReinjectImages: %v", err)\
}\
if string(restored) != in {\
t.Fatalf("round trip broke under colliding id source:\n got: %s\nwant: %s", restored, in)\
}\
}\
\
// The same base64 appearing in both an image and a text field round-trips\
// byte-exactly: the value swap is value-preserving and reversible, so every\
// occurrence is restored to the identical bytes on reinject.\
func TestClaudeCodec_Base64InTextRoundTrips(t *testing.T) {\
t.Parallel()\
c := CodecFor(agent.AgentTypeClaudeCode)\
b64 := base64.StdEncoding.EncodeToString([]byte("shared-image-and-text-payload-long-enough-to-externalize"))\
textLine := `{"type":"user","message":{"role":"user","content":[{"type":"text","text":"raw was ` + b64 + `"}]}}`\
in := textLine + "\n" + claudeLine(b64) + "\n"\
\
rewritten, assets, err := c.ExtractImages([]byte(in))\
if err != nil {\
t.Fatalf("ExtractImages: %v", err)\
}\
if len(assets) != 1 {\
t.Fatalf("want 1 asset, got %d", len(assets))\
}\
restored, err := c.ReinjectImages(rewritten, lookupFrom(assets))\
if err != nil {\
t.Fatalf("ReinjectImages: %v", err)\
}\
if string(restored) != in {\
t.Fatalf("round trip not byte-exact:\n got: %s\nwant: %s", restored, in)\
}\
}\
\
// Regression: Claude Code serializes image content blocks with a space after the\
// colon ("data": "<b64>") as well as compactly ("data":"<b64>"). Both forms must\
// externalize and round-trip. (A data-field-scoped swap missed the spaced form.)\
func TestClaudeCodec_SpacedAndCompactDataFields(t *testing.T) {\
t.Parallel()\
c := CodecFor(agent.AgentTypeClaudeCode)\
for _, tc := range []struct {\
name, line string\
}{\
{"compact", `{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data":"%s"}}`},\
{"spaced", `{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "%s"}}`},\
} {\
b64 := base64.StdEncoding.EncodeToString([]byte("spaced-vs-compact-payload-long-enough-to-externalize-" + tc.name))\
in := strings.Replace(tc.line, "%s", b64, 1) + "\n"\
rewritten, assets, err := c.ExtractImages([]byte(in))\
if err != nil {\
t.Fatalf("[%s] ExtractImages: %v", tc.name, err)\
}\
if len(assets) != 1 {\
t.Fatalf("[%s] want 1 asset, got %d", tc.name, len(assets))\
}\
if strings.Contains(string(rewritten), b64) {\
t.Errorf("[%s] base64 not externalized", tc.name)\
}\
restored, err := c.ReinjectImages(rewritten, lookupFrom(assets))\
if err != nil {\
t.Fatalf("[%s] ReinjectImages: %v", tc.name, err)\
}\
if string(restored) != in {\
t.Fatalf("[%s] round trip not byte-exact", tc.name)\
}\
}\
}\
\
// A crypto/rand failure surfaces as an error instead of a silent all-zero id.\
func TestClaudeCodec_IDGenerationErrorSurfaces(t *testing.T) {\
c := CodecFor(agent.AgentTypeClaudeCode)\
orig := newAssetID\
newAssetID = func() (string, error) { return "", errTestRand }\
defer func() { newAssetID = orig }()\
\
b64 := base64.StdEncoding.EncodeToString([]byte("payload-long-enough-to-externalize-and-trigger-id-gen"))\
_, _, err := c.ExtractImages([]byte(claudeLine(b64) + "\n"))\
if err == nil {\
t.Fatal("expected an error when id generation fails, got nil")\
}\
}\
\
// An array-rooted JSONL line carrying an image is walked like an object line.\
func TestClaudeCodec_ArrayRootedLine(t *testing.T) {\
t.Parallel()\
c := CodecFor(agent.AgentTypeClaudeCode)\
b64 := base64.StdEncoding.EncodeToString([]byte("array-rooted-line-image-payload-long-enough-to-externalize"))\
in := `[{"type":"image","source":{"type":"base64","media_type":"image/png","data":"` + b64 + `"}}]` + "\n"\
rewritten, assets, err := c.ExtractImages([]byte(in))\
if err != nil {\
t.Fatalf("ExtractImages: %v", err)\
}\
if len(assets) != 1 {\
t.Fatalf("array-rooted line: want 1 asset, got %d", len(assets))\
}\
restored, err := c.ReinjectImages(rewritten, lookupFrom(assets))\
if err != nil {\
t.Fatalf("ReinjectImages: %v", err)\
}\
if string(restored) != in {\
t.Fatalf("array-rooted round trip not byte-exact")\
}\
}\
\
// Base64 values too short to be a real image are left inline (and can therefore\
// never collide with a placeholder's hex id).\
func TestClaudeCodec_LeavesTinyBase64Inline(t *testing.T) {\
t.Parallel()\
c := CodecFor(agent.AgentTypeClaudeCode)\
tiny := base64.StdEncoding.EncodeToString([]byte("tiny-blob")) // < minExternalizedBase64Len\
if len(tiny) >= minExternalizedBase64Len {\
t.Fatalf("test fixture too long: %d", len(tiny))\
}\
orig := claudeLine(tiny) + "\n"\
rewritten, assets, err := c.ExtractImages([]byte(orig))\
if err != nil {\
t.Fatalf("ExtractImages: %v", err)\
}\
if len(assets) != 0 || string(rewritten) != orig {\
t.Errorf("tiny base64 must be left inline; assets=%d changed=%v", len(assets), string(rewritten) != orig)\
}\
}\
\
// Images whose decoded bytes exceed maxExternalizedImageBytes are left inline: as\
// a single asset blob they could become an unpushable git object, so (like the\
// Cursor sidecar path) they stay in the transcript, which is chunked to stay\
// pushable. Not parallel: it lowers the shared cap to avoid a 50MB fixture.\
func TestClaudeCodec_LeavesOversizedImageInline(t *testing.T) {\
c := CodecFor(agent.AgentTypeClaudeCode)\
\
restore := maxExternalizedImageBytes\
maxExternalizedImageBytes = 8\
t.Cleanup(func() { maxExternalizedImageBytes = restore })\
\
// 72 bytes: over the lowered cap, and its base64 clears minExternalizedBase64Len\
// so only the size guard (not the min-length filter) can keep it inline.\
raw := append([]byte("\x89PNG\r\n\x1a\n"), make([]byte, 64)...)\
b64 := base64.StdEncoding.EncodeToString(raw)\
if len(b64) < minExternalizedBase64Len {\
t.Fatalf("fixture too short to exercise the max guard: %d", len(b64))\
}\
orig := claudeLine(b64) + "\n"\
rewritten, assets, err := c.ExtractImages([]byte(orig))\
if err != nil {\
t.Fatalf("ExtractImages: %v", err)\
}\
if len(assets) != 0 || string(rewritten) != orig {\
t.Errorf("oversized image must be left inline; assets=%d changed=%v", len(assets), string(rewritten) != orig)\
}\
}\
\
// Non-base64 image sources (e.g. url) and non-decodable data are left inline.\
func TestClaudeCodec_LeavesNonBase64Inline(t *testing.T) {\
t.Parallel()\
c := CodecFor(agent.AgentTypeClaudeCode)\
orig := `{"type":"user","message":{"content":[{"type":"image","source":{"type":"url","url":"https://x/y.png"}}]}}` + "\n"\
rewritten, assets, err := c.ExtractImages([]byte(orig))\
if err != nil {\
t.Fatalf("ExtractImages: %v", err)\
}\
if len(assets) != 0 || string(rewritten) != orig {\
t.Errorf("url image source must be left inline; assets=%d changed=%v", len(assets), string(rewritten) != orig)\
}\
}\
\
// Agents that don't inline images in the transcript have no codec (graceful\
// no-op upstream). Cursor is included deliberately: its images live in a separate\
// SQLite store, captured via the SidecarImageProvider path, not a transcript codec.\
func TestCodecFor_NonImageAgentsAreNil(t *testing.T) {\
t.Parallel()\
for _, at := range []string{"Cursor", "Gemini CLI", "OpenCode", "Pi", "Factory AI Droid", "Copilot CLI"} {\
if CodecFor(types.AgentType(at)) != nil {\
t.Errorf("agent %q should not have an image codec yet", at)\
}\
}\
}\
\
// The placeholder must stay low-entropy so the downstream redaction pass never\
// flags it. Redaction's entropy detector runs over each [A-Za-z0-9+_=-]{10,}\
// RUN (threshold 4.5 bits/char), not the whole string, so mirror that here.\
func TestPlaceholder_RunsAreLowEntropy(t *testing.T) {\
t.Parallel()\
c := CodecFor(agent.AgentTypeClaudeCode)\
b64 := base64.StdEncoding.EncodeToString([]byte("entropy-check-bytes-xyz-padded-to-exceed-the-externalize-threshold"))\
rewritten, _, err := c.ExtractImages([]byte(claudeLine(b64) + "\n"))\
if err != nil {\
t.Fatalf("ExtractImages: %v", err)\
}\
ph := placeholderRe.Find(rewritten)\
if ph == nil {\
t.Fatal("no placeholder produced")\
}\
runRe := regexp.MustCompile(`[A-Za-z0-9+_=-]{10,}`)\
runs := runRe.FindAll(ph, -1)\
if len(runs) == 0 {\
t.Fatalf("expected at least one detector-sized run in %s", ph)\
}\
for _, run := range runs {\
if e := shannonBitsPerChar(run); e >= 4.5 {\
t.Errorf("placeholder run %q entropy %.2f >= 4.5 — redaction could flag it", run, e)\
}\
}\
}\
\
func shannonBitsPerChar(b []byte) float64 {\
if len(b) == 0 {\
return 0\
}\
var counts [256]int\
for _, c := range b {\
counts[c]++\
}\
var e float64\
n := float64(len(b))\
for _, c := range counts {\
if c == 0 {\
continue\
}\
p := float64(c) / n\
e -= p * math.Log2(p)\
}\
return e\
}\
```\
\
Acmd/entire/cli/transcript/imageextract/imageextract\_test.go+401\
\
```\
15 unmodified lines\
\
16\
17\
18\
19\
20\
21\
19\
20\
21\
22\
23\
24\
57 unmodified lines\
\
82\
83\
84\
85\
86\
85\
86\
87\
88\
89\
\
15 unmodified lines\
\
github.com/go-faster/errors v0.7.1\
github.com/go-faster/jx v1.2.0\
github.com/go-git/go-billy/v6 v6.0.0-alpha.1.0.20260519112248-0095b064a6c6\
github.com/go-git/go-git/v6 v6.0.0-alpha.4.0.20260521161150-3af8745c291b\
github.com/go-git/x/plugin/objectsigner/auto v0.1.0\
github.com/go-git/x/plugin/objectsigner/program v0.0.0-20260506121155-e7fc238fcab6\
github.com/go-git/go-git/v6 v6.0.0-alpha.4.0.20260713100844-d5e9b9c7895b\
github.com/go-git/x/plugin/objectsigner/auto v0.1.1-0.20260624122410-382b2905c041\
github.com/go-git/x/plugin/objectsigner/program v0.0.0-20260624122410-382b2905c041\
github.com/gofrs/flock v0.13.0\
github.com/google/uuid v1.6.0\
github.com/mattn/go-isatty v0.0.22\
57 unmodified lines\
\
github.com/gitleaks/go-gitdiff v0.9.1 // indirect\
github.com/go-faster/yaml v0.4.6 // indirect\
github.com/go-git/gcfg/v2 v2.0.2 // indirect\
github.com/go-git/x/plugin/objectsigner/gpg v0.1.0 // indirect\
github.com/go-git/x/plugin/objectsigner/ssh v0.1.0 // indirect\
github.com/go-git/x/plugin/objectsigner/gpg v0.2.1-0.20260624122410-382b2905c041 // indirect\
github.com/go-git/x/plugin/objectsigner/ssh v0.2.1-0.20260624122410-382b2905c041 // indirect\
github.com/go-sprout/sprout v1.0.3 // indirect\
github.com/goccy/go-json v0.10.5 // indirect\
github.com/godbus/dbus/v5 v5.2.2 // indirect\
```\
\
Mgo.mod+5/-5\
\
```\
134 unmodified lines\
\
135\
136\
137\
138\
139\
140\
141\
142\
143\
144\
145\
146\
147\
138\
139\
140\
141\
142\
143\
144\
145\
146\
147\
148\
149\
150\
\
134 unmodified lines\
\
github.com/go-git/go-billy/v6 v6.0.0-alpha.1.0.20260519112248-0095b064a6c6/go.mod h1:eaCUpHbedW7//EwcYmUDfJe2N6sJC9O12AT0OTqJR1E=\
github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1 h1:gmqi2jvsreu0s8JMLylYDFq4sbjHwwlhktMw0DUg3mA=\
github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1/go.mod h1:ECf1MqJlBdYpKggBrOXjo/0EnvRZx6D++I86UYjPgAQ=\
github.com/go-git/go-git/v6 v6.0.0-alpha.4.0.20260521161150-3af8745c291b h1:99k+na4J/y/rKvB21GFeFhIf/Rqskfc4a6mvTf4wbJA=\
github.com/go-git/go-git/v6 v6.0.0-alpha.4.0.20260521161150-3af8745c291b/go.mod h1:OTUSi3RzPFoC0j/+uxHdVG1X/xXz84QCxLzYvXRvyXk=\
github.com/go-git/x/plugin/objectsigner/auto v0.1.0 h1:RcLW29RgwSCmqrNSs7QOxvWkRbM1vPu0Vp9TCECZjMs=\
github.com/go-git/x/plugin/objectsigner/auto v0.1.0/go.mod h1:iP2cXPyXc//9v9THS3y/MLi0jnt7vEqwUDj11qQfFPg=\
github.com/go-git/x/plugin/objectsigner/gpg v0.1.0 h1:NEGVSOD+LPnus6j4iNkAZaHVTc4DNY223y1/I2Jq2yI=\
github.com/go-git/x/plugin/objectsigner/gpg v0.1.0/go.mod h1:1iosWq3OOqZxtNrwDHtcjicswuaOT45J5GMFyCk80wc=\
github.com/go-git/x/plugin/objectsigner/program v0.0.0-20260506121155-e7fc238fcab6 h1:ZRy5GVQf/EisYhLj3zwU+eGVhMDWhYxCfaq3wBusGsM=\
github.com/go-git/x/plugin/objectsigner/program v0.0.0-20260506121155-e7fc238fcab6/go.mod h1:qqkRcAeBDQLDJTBiN/s4k4Xj6eFBP+2cdoZDzsld0b0=\
github.com/go-git/x/plugin/objectsigner/ssh v0.1.0 h1:lAeeDgc1oxsMMvVUed6ssrqJnD97UR1K/dXIDdeg1Yc=\
github.com/go-git/x/plugin/objectsigner/ssh v0.1.0/go.mod h1:6BvpZj9Yry1ZFNw4N5OZDc+7M1T8oyrZilLNFg2aTsM=\
github.com/go-git/go-git/v6 v6.0.0-alpha.4.0.20260713100844-d5e9b9c7895b h1:RP3bg2PI8ZMeyGr31iJIqLTqLMEEXWGE2pSbAn/oRLk=\
github.com/go-git/go-git/v6 v6.0.0-alpha.4.0.20260713100844-d5e9b9c7895b/go.mod h1:6B0m9RQWmjdn10aHogqSpI7z7kwEUtELzzA8wW8gogg=\
github.com/go-git/x/plugin/objectsigner/auto v0.1.1-0.20260624122410-382b2905c041 h1:ATVPaVKC1wbuQdvGKfKXotuwXYeGfigyHERl7lmNG+I=\
github.com/go-git/x/plugin/objectsigner/auto v0.1.1-0.20260624122410-382b2905c041/go.mod h1:Cpmdf+1Pmw6nPWTpfBMsPmWju2Tb+qjwccWR5AvOBC4=\
github.com/go-git/x/plugin/objectsigner/gpg v0.2.1-0.20260624122410-382b2905c041 h1:Tni6GTpv/Nx4HAub64YmnxGWe99za33jfzy3GesditQ=\
github.com/go-git/x/plugin/objectsigner/gpg v0.2.1-0.20260624122410-382b2905c041/go.mod h1:1iosWq3OOqZxtNrwDHtcjicswuaOT45J5GMFyCk80wc=\
github.com/go-git/x/plugin/objectsigner/program v0.0.0-20260624122410-382b2905c041 h1:3SNIy+i6ou6OX1ekdFKpuTg+BGPO3Q4Jj6by0KX/2lY=\
github.com/go-git/x/plugin/objectsigner/program v0.0.0-20260624122410-382b2905c041/go.mod h1:qqkRcAeBDQLDJTBiN/s4k4Xj6eFBP+2cdoZDzsld0b0=\
github.com/go-git/x/plugin/objectsigner/ssh v0.2.1-0.20260624122410-382b2905c041 h1:mmJ/LFr0c7ij9UYQorU66989ge06vf1H07ud533UQ/I=\
github.com/go-git/x/plugin/objectsigner/ssh v0.2.1-0.20260624122410-382b2905c041/go.mod h1:6BvpZj9Yry1ZFNw4N5OZDc+7M1T8oyrZilLNFg2aTsM=\
github.com/go-sprout/sprout v1.0.3 h1:LLuz0D3aYazgbVTOwCVuMor3LOUVYinipXRIdjA/D+I=\
github.com/go-sprout/sprout v1.0.3/go.mod h1:cFFzpnyGGry3cmN0UNCAM1f7AGok6vPVabeYQzBMBZY=\
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=\
```\
\
Mgo.sum+10/-10\
\
```\
322 unmodified lines\
\
323\
324\
325\
326\
326\
327\
328\
329\
2 unmodified lines\
\
332\
333\
334\
335\
336\
337\
338\
339\
340\
341\
342\
343\
344\
345\
346\
347\
348\
349\
350\
351\
352\
353\
354\
355\
356\
357\
358\
359\
360\
361\
362\
363\
364\
365\
366\
367\
368\
369\
370\
371\
372\
373\
374\
375\
376\
377\
378\
379\
380\
381\
382\
383\
384\
385\
386\
387\
388\
389\
390\
391\
392\
393\
394\
\
322 unmodified lines\
\
stdin := bufio.NewReader(strings.NewReader("\n"))\
\
var stdout bytes.Buffer\
err := handlePush(context.Background(), ft, firstLine, &Options{}, stdin, &stdout)\
err := handlePush(context.Background(), ft, &refAdvCache{}, firstLine, &Options{}, stdin, &stdout)\
if err == nil {\
t.Fatal("expected error from send-pack exit 1")\
}\
2 unmodified lines\
\
stdout.String(), helperStatusLine)\
}\
}\
\
// TestInvariant_PushReusesListForPushAdvertisement pins the fix for ENCLI-267.\
// Within one helper session the "push" command MUST reuse the ref\
// advertisement fetched during "list for-push" rather than re-fetching\
// info/refs. Git snapshots the remote refs from "list for-push" into its\
// remote_refs list *before* running the pre-push hook, and the hook pushes\
// per-checkpoint refs to the same remote. A fresh info/refs at push time then\
// hands send-pack a ref (the freshly-pushed checkpoint) Git never asked to\
// push; send-pack emits `error <ref> no match`, and Git — not finding it in\
// remote_refs — warns `helper reported unexpected status of <ref>`. Reusing\
// the list-for-push snapshot mirrors remote-curl.c's discovery cache and keeps\
// that phantom ref out of send-pack's view.\
func TestInvariant_PushReusesListForPushAdvertisement(t *testing.T) {\
// No t.Parallel(): t.Setenv("PATH", ...) mutates process-global state.\
if runtime.GOOS == "windows" {\
t.Skip("shell-script PATH stub is POSIX-only")\
}\
\
ref := testRefMain\
oldSHA := strings.Repeat("a", 40)\
\
// Stub git send-pack: emit the empty-request terminator, drain stdin,\
// then the trailing flush + a plain "ok" helper-status, exit 0.\
stubDir := t.TempDir()\
stub := "#!/bin/sh\nprintf '0000'\ncat > /dev/null\nprintf '0000ok " + ref + "\\n'\nexit 0\n"\
if err := os.WriteFile(filepath.Join(stubDir, "git"), []byte(stub), 0o755); err != nil {\
t.Fatalf("writing stub git: %v", err)\
}\
t.Setenv("PATH", stubDir+string(os.PathListSeparator)+os.Getenv("PATH"))\
\
// The checkpoint ref the pre-push hook would push between list-for-push\
// and push: absent from the first advertisement, present in the second, so\
// a re-fetch (the bug) would expose it to send-pack.\
checkpointRef := "refs/entire/checkpoints/9H/01KX2ATMJ3FAZZFZ8CP1CA279H"\
receivePackCalls := 0\
ft := &fakeTransport{\
infoRefsResp: func() (io.ReadCloser, error) {\
receivePackCalls++\
refLine := oldSHA + " " + ref + "\x00report-status object-format=sha1\n"\
if receivePackCalls == 1 {\
return stringRC(serviceAnnouncement(serviceReceivePack, refLine)), nil\
}\
return stringRC(serviceAnnouncement(serviceReceivePack, refLine,\
oldSHA+" "+checkpointRef+"\n")), nil\
},\
serviceRPCResp: func(string, []byte) (io.ReadCloser, error) {\
return stringRC(""), nil\
},\
}\
\
stdin := strings.NewReader("list for-push\npush " + oldSHA + ":" + ref + "\n\n")\
var stdout bytes.Buffer\
if err := Run(context.Background(), ft, 2, stdin, &stdout); err != nil {\
t.Fatalf("Run: %v", err)\
}\
\
if receivePackCalls != 1 {\
t.Fatalf("receive-pack info/refs fetched %d times; want 1 (push must reuse the list-for-push advertisement)", receivePackCalls)\
}\
}\
```\
\
Minternal/remotehelper/githelper/invariants\_test.go+61/-1\
\
```\
17 unmodified lines\
\
18\
19\
20\
21\
21\
22\
23\
24\
25\
26\
26\
27\
28\
29\
\
17 unmodified lines\
\
// writes one "<value> <name>" line per ref followed by a blank-line\
// terminator. HEAD is emitted as "@<target> HEAD" when the symref\
// capability resolves; detached HEAD falls back to "<sha> HEAD".\
func handleList(ctx context.Context, t Transport, forPush bool, stdout io.Writer) error {\
func handleList(ctx context.Context, t Transport, adv *refAdvCache, forPush bool, stdout io.Writer) error {\
service := serviceUploadPack\
if forPush {\
service = serviceReceivePack\
}\
refs, err := t.InfoRefs(ctx, service)\
refs, err := adv.infoRefs(ctx, t, service)\
if err != nil {\
return fmt.Errorf("list %s info/refs: %w", service, err)\
}\
```\
\
Minternal/remotehelper/githelper/list.go+2/-2\
\
```\
106 unmodified lines\
\
107\
108\
109\
110\
110\
111\
112\
113\
\
106 unmodified lines\
\
defer server.Close()\
\
var out bytes.Buffer\
if err := handleList(context.Background(), testTransport(server), tt.forPush, &out); err != nil {\
if err := handleList(context.Background(), testTransport(server), &refAdvCache{}, tt.forPush, &out); err != nil {\
t.Fatalf("handleList: %v", err)\
}\
if out.String() != tt.want {\
```\
\
Minternal/remotehelper/githelper/list\_test.go+1/-1\
\
```\
41 unmodified lines\
\
42\
43\
44\
45\
45\
46\
47\
48\
49\
50\
51\
51\
52\
53\
54\
55\
56\
57\
58\
\
41 unmodified lines\
\
// 6. Send-pack writes a trailing flush + helper-status lines to\
// stdout; we discard the flush and relay helper-status to git,\
// then append the blank line that terminates the status batch.\
func handlePush(ctx context.Context, t Transport, firstLine string, opts *Options, stdin *bufio.Reader, stdout io.Writer) error {\
func handlePush(ctx context.Context, t Transport, adv *refAdvCache, firstLine string, opts *Options, stdin *bufio.Reader, stdout io.Writer) error {\
refspecs, err := readPushBatch(firstLine, stdin)\
if err != nil {\
return err\
}\
\
refsResp, err := t.InfoRefs(ctx, serviceReceivePack)\
// Reuse the advertisement "list for-push" already fetched. Re-fetching\
// here would observe refs the pre-push hook pushed after Git's ref\
// snapshot, which send-pack reports and Git flags as "unexpected status"\
// (see refAdvCache / ENCLI-267).\
refsResp, err := adv.infoRefs(ctx, t, serviceReceivePack)\
if err != nil {\
return fmt.Errorf("fetching receive-pack info/refs: %w", err)\
}\
```\
\
Minternal/remotehelper/githelper/push.go+6/-2\
\
```\
1\
2\
3\
4\
5\
6\
7\
8\
9\
10\
11\
12\
13\
14\
15\
16\
17\
18\
19\
20\
21\
22\
23\
24\
25\
26\
27\
28\
29\
30\
31\
32\
33\
34\
35\
36\
37\
38\
39\
40\
41\
42\
43\
44\
45\
46\
47\
48\
49\
50\
51\
52\
53\
54\
55\
\
package githelper\
\
import (\
"bytes"\
"context"\
"fmt"\
"io"\
)\
\
// refAdvCache memoizes the receive-pack ref advertisement across a single\
// helper session so the "push" command reuses the exact ref snapshot that\
// "list for-push" fetched. This mirrors remote-curl.c's discovery cache\
// (get_refs/last_refs): Git builds its remote_refs list from the\
// "list for-push" advertisement, then runs the pre-push hook, then issues\
// "push". The Entire pre-push hook pushes per-checkpoint refs to the same\
// remote in that window, so a fresh info/refs at push time would hand\
// send-pack a ref Git never asked to push. send-pack then reports\
// `error <ref> no match` and Git — not finding it in remote_refs — warns\
// `helper reported unexpected status of <ref>` (ENCLI-267). Reusing the\
// snapshot keeps that phantom ref out of send-pack's view.\
//\
// Only the receive-pack (for-push) advertisement is cached; upload-pack and\
// v2 fetches pass straight through, matching remote-curl's per-for_push cache.\
type refAdvCache struct {\
receivePack []byte\
cached bool\
}\
\
// infoRefs returns the ref advertisement for service. The receive-pack\
// advertisement is fetched from the Transport once and replayed from an\
// in-memory buffer on subsequent calls; every other service is fetched fresh.\
func (c *refAdvCache) infoRefs(ctx context.Context, t Transport, service string) (io.ReadCloser, error) {\
if service != serviceReceivePack {\
rc, err := t.InfoRefs(ctx, service)\
if err != nil {\
return nil, fmt.Errorf("fetch %s advertisement: %w", service, err)\
}\
return rc, nil\
}\
if c.cached {\
return io.NopCloser(bytes.NewReader(c.receivePack)), nil\
}\
rc, err := t.InfoRefs(ctx, service)\
if err != nil {\
return nil, fmt.Errorf("fetch %s advertisement: %w", service, err)\
}\
defer rc.Close()\
buf, err := io.ReadAll(rc)\
if err != nil {\
return nil, fmt.Errorf("buffer %s advertisement: %w", service, err)\
}\
c.receivePack = buf\
c.cached = true\
return io.NopCloser(bytes.NewReader(buf)), nil\
}\
```\
\
Ainternal/remotehelper/githelper/refadv\_cache.go+55\
\
```\
30 unmodified lines\
\
31\
32\
33\
34\
35\
36\
37\
38\
39\
21 unmodified lines\
\
61\
62\
63\
61\
64\
65\
66\
67\
20 unmodified lines\
\
88\
89\
90\
88\
91\
92\
93\
94\
\
30 unmodified lines\
\
func Run(ctx context.Context, t Transport, protocolVersion int, stdin io.Reader, stdout io.Writer) error {\
commandReader := bufio.NewReader(stdin)\
opts := &Options{}\
// One advertisement snapshot per session: "push" reuses what\
// "list for-push" fetched. See refAdvCache / ENCLI-267.\
adv := &refAdvCache{}\
\
for {\
line, err := commandReader.ReadString('\n')\
21 unmodified lines\
\
fmt.Fprintln(stdout)\
\
case line == "list" || line == "list for-push":\
if err := handleList(ctx, t, line == "list for-push", stdout); err != nil {\
if err := handleList(ctx, t, adv, line == "list for-push", stdout); err != nil {\
return err\
}\
\
20 unmodified lines\
\
return nil\
\
case strings.HasPrefix(line, "push "):\
if err := handlePush(ctx, t, line, opts, commandReader, stdout); err != nil {\
if err := handlePush(ctx, t, adv, line, opts, commandReader, stdout); err != nil {\
return err\
}\
```\
\
Minternal/remotehelper/githelper/run.go+5/-2