checkpoint: store full compact transcript per checkpoint with boundary marker

7e9c63f→main·

Soph·2w ago·3 files·+157 added/-46 removed

writeCompactTranscript now uses compact.FullWithBoundary to write the entire compaction session into transcript.jsonl on every checkpoint, and returns the checkpoint's boundary so it can be recorded in session metadata as CompactTranscriptStart. Both write paths are covered:

• normal write: writeTranscript returns the boundary, threaded into the per-session metadata.json.
• pre-push OPF rewrite: replaceTranscript re-records the boundary via setCompactTranscriptStart after regenerating the compact transcript.

This makes each checkpoint self-contained: the session is reconstructable from any single surviving checkpoint, robust to a mid-history checkpoint being lost, reverted, or dropped during a rebase. Readers segment a checkpoint's slice as fullCompactLines[CompactTranscriptStart:]:

fsstore writes no compact transcript, so its metadata leaves the marker nil.

Update the existing scoped-transcript tests to assert the new full-content + marker contract instead of the old delta semantics.

Co-Authored-By: Claude Opus 4.8 (1M context) noreply@anthropic.com

Sessions

27a4b7b0bdebView transcript

Changes

3

• cmd/entire/cli/checkpoint

• Mcheckpoint_test.go+6/-6

  • Mpersistent.go+77/-26
  • Mpersistent_compact_transcript_test.go+74/-14

4795 unmodified lines

4796
4797
4798
4799
4800
4801
4799
4800
4801
4802
4803
4804
11 unmodified lines

4816
4817
4818
4819
4819
4820
4821
4821
4822
4823
4824
122 unmodified lines

4947
4948
4949
4950
4950
4951
4952
4953

4795 unmodified lines

return summary
}

// readSessionMetadataAtIndex reads the per-session Metadata for
// session at numbered subfolder `index` (0-based) under the checkpoint.
func readSessionMetadataAtIndex(t *testing.T, repo *git.Repository, checkpointID id.CheckpointID, index int) Metadata {
// readSessionMetadata reads the per-session Metadata for the first session
// (numbered subfolder "0") under the checkpoint.
func readSessionMetadata(t *testing.T, repo *git.Repository, checkpointID id.CheckpointID) Metadata {
    t.Helper()
    ref, err := repo.Reference(plumbing.NewBranchReferenceName(paths.MetadataBranchName), true)
    if err != nil {
11 unmodified lines

"get checkpoint subtree: %v", err)
    }
    sessionTree, err := checkpointTree.Tree(strconv.Itoa(index))
sessionTree, err := checkpointTree.Tree("0")
    if err != nil {
        "get session subtree %d: %v", index, err)
        "get session subtree 0: %v", err)
    }
sessionFile, err := sessionTree.File(paths.MetadataFileName)
    if err != nil {
122 unmodified lines

"WriteCommitted: %v", err)
    }

meta := readSessionMetadataAtIndex(t, repo, checkpointID, 0)
meta := readSessionMetadata(t, repo, checkpointID)
if meta.Kind != "agent_investigate" {
    t.Errorf("Kind: got %q, want agent_investigate", meta.Kind)
}

Mcmd/entire/cli/checkpoint/checkpoint_test.go+6/-6

406 unmodified lines

407
408
409
410
410
411
412
413
41 unmodified lines

455
456
457
458
459
460
461
285 unmodified lines

747
748
749
749
750
751
752
753
9 unmodified lines

763
764
765
765
766
767
768
769
770
771
771
772
773
774
775
7 unmodified lines

783
784
785
785
786
787
788
789
7 unmodified lines

797
798
799
799
800
801
802
803
12 unmodified lines

816
817
818
818
819
820
821
822
2 unmodified lines

825
826
827
827
828
829
830
831
828
829
830
831
832
833
834
835
836
5 unmodified lines

842
843
844
843
845
846
847
848
7 unmodified lines

856
857
858
857
858
859
860
861
862
863
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
867
875
876
877
878
4 unmodified lines

883
884
885
878
886
887
888
889
890
891
884
892
893
894
895
896
897
898
891
899
900
901
902
2 unmodified lines

905
906
907
900
908
909
910
911
1 unmodified line

913
914
915
916
917
918
919
883 unmodified lines

1803
1804
1805
1797
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852

406 unmodified lines

// Write transcript. Transcript points at full.jsonl (CLI
    // rewind/resume/explain read it by filename); the compact transcript.jsonl,
    // when written, is also pushed and pointed at by CompactTranscript.
    wroteTranscript, err := s.writeTranscript(ctx, opts, sessionPath, entries)
wroteTranscript, compactTranscriptStart, err := s.writeTranscript(ctx, opts, sessionPath, entries)
    if err != nil {
        return filePaths, err
    }
    41 unmodified lines

TranscriptIdentifierAtStart: opts.TranscriptIdentifierAtStart,
        CheckpointTranscriptStart:   opts.CheckpointTranscriptStart,
        TranscriptLinesAtStart:      opts.CheckpointTranscriptStart, // Deprecated: kept for backward compat
        CompactTranscriptStart:      compactTranscriptStart,
        TokenUsage:                  opts.TokenUsage,
        SkillEventsVersion:          skillEventsVersion(opts.SkillEvents),
        SkillEvents:                 opts.SkillEvents,
    285 unmodified lines

// tree (so it is pushed alongside full.jsonl) but is not yet referenced by
// metadata. Returns true when a transcript was written, false when it was
// empty and nothing was written.
func (s *GitStore) writeTranscript(ctx context.Context, opts WriteOptions, basePath string, entries map[string]object.TreeEntry) (bool, error) {
func (s *GitStore) writeTranscript(ctx context.Context, opts WriteOptions, basePath string, entries map[string]object.TreeEntry) (bool, *int, error) {
    logCtx := logging.WithComponent(ctx, "checkpoint")
    transcriptBytes := opts.Transcript.Bytes()

9 unmodified lines

if len(rawData) > 0 {
            redacted, redactErr := redact.JSONLBytes(rawData)
            if redactErr != nil {
                return false, fmt.Errorf("failed to redact transcript from file: %w", redactErr)
            }
            transcriptBytes = redacted.Bytes()
        }
    }
    if len(transcriptBytes) == 0 {
        return false, nil
    }

if opts.Agent == agent.AgentTypeCodex {
7 unmodified lines

if err != nil {
        chunkTranscriptSpan.RecordError(err)
        chunkTranscriptSpan.End()
        return false, fmt.Errorf("failed to chunk transcript: %w", err)
    }
    chunkTranscriptSpan.End()
    chunkDuration := time.Since(chunkStart)
7 unmodified lines

if err != nil {
        writeTranscriptBlobsSpan.RecordError(err)
        writeTranscriptBlobsSpan.End()
        return false, err
    }
    entries[chunkPath] = object.TreeEntry{
        Name: chunkPath,
12 unmodified lines

if err != nil {
        contentHashSpan.RecordError(err)
        contentHashSpan.End()
        return false, err
    }
    entries[basePath+paths.ContentHashFileName] = object.TreeEntry{
        Name: basePath + paths.ContentHashFileName,
2 unmodified lines

}
    contentHashSpan.End()

// Write the compact transcript (transcript.jsonl) into the tree so it is
    // pushed alongside full.jsonl. The metadata pointer intentionally stays on
    // full.jsonl for now — pointing it at the compact transcript is deferred to
    // a later change.
    s.writeCompactTranscript(logCtx, opts.Agent, opts.CheckpointTranscriptStart, transcriptBytes, basePath, entries)
    // Write the full compact transcript (transcript.jsonl) into the tree so it
    // is pushed alongside full.jsonl. The metadata pointer (filePaths) stays on
    // full.jsonl, which the CLI read paths resolve by filename. compactStart is
    // the line offset of this checkpoint's slice within the full compact output,
    // recorded into session metadata so downstream readers can segment it.
    compactStart := s.writeCompactTranscript(logCtx, opts.Agent, opts.CheckpointTranscriptStart, transcriptBytes, basePath, entries)

logging.Debug(logCtx, "write transcript timings",
        slog.String("session_id", opts.SessionID),
5 unmodified lines

slog.Int("transcript_bytes", len(transcriptBytes)),
        slog.Int("chunk_count", len(chunks)),
    )
    return true, nil
    return true, compactStart, nil
}

// compactAgentName resolves the agent slug used in compact transcript lines
7 unmodified lines

// writeCompactTranscript converts the pre-redacted full transcript into the
// compact transcript.jsonl format, scoped to this checkpoint via startLine,
// and records it at sessionPath in the tree. Best-effort: the compact
// transcript is derived data, so failures are logged and never fail the
// checkpoint write. transcriptBytes must already be sanitized for the agent
// (e.g. Codex portable-transcript sanitization); callers sanitize before
// calling so the expensive pass runs exactly once.
func (s *GitStore) writeCompactTranscript(ctx context.Context, agentType types.AgentType, startLine int, transcriptBytes []byte, sessionPath string, entries map[string]object.TreeEntry) {
// compact transcript.jsonl format and records it at sessionPath in the tree.
// The whole session is compacted (so each checkpoint is self-contained); the
// returned offset is the line in the compact output at which this checkpoint's
// data begins (derived from startLine), to be stored as
// Metadata.CompactTranscriptStart so readers can segment the slice.

// Best-effort: the compact transcript is derived data, so failures are logged
// and never fail the checkpoint write, in which case a nil offset is returned
// (no transcript.jsonl written, no marker to record). transcriptBytes must
// already be sanitized for the agent (e.g. Codex portable-transcript
// sanitization); callers sanitize before calling so the expensive pass runs
// exactly once.
func (s *GitStore) writeCompactTranscript(ctx context.Context, agentType types.AgentType, startLine int, transcriptBytes []byte, sessionPath string, entries map[string]object.TreeEntry) *int {
    compactCtx, compactSpan := perf.Start(ctx, "write_compact_transcript")
    defer compactSpan.End()

compacted, err := transcriptcompact.Compact(redact.AlreadyRedacted(transcriptBytes), transcriptcompact.MetadataFields{
    compacted, boundary, err := transcriptcompact.FullWithBoundary(redact.AlreadyRedacted(transcriptBytes), transcriptcompact.MetadataFields{
        Agent:      compactAgentName(agentType),
        CLIVersion: versioninfo.Version,
        StartLine:  startLine,
4 unmodified lines

return
                return nil
        }
        if len(bytes.TrimSpace(compacted)) == 0 {
            logging.Debug(compactCtx, "compact transcript empty, skipping transcript.jsonl",
                slog.String("agent", string(agentType)),
            )
            return
            return nil
        }
        if len(compacted) > agent.MaxChunkSize {
            logging.Warn(compactCtx, "compact transcript exceeds max blob size, skipping transcript.jsonl",
                slog.String("agent", string(agentType)),
                slog.Int("compact_bytes", len(compacted)),
            )
            return
            return nil
        }

blobHash, err := CreateBlobFromContent(s.repo, compacted)
2 unmodified lines

logging.Warn(compactCtx, "failed to create compact transcript blob, skipping transcript.jsonl",
                                slog.String("error", err.Error()),
                        )
            return
            return nil
        }
        compactPath := sessionPath + paths.CompactTranscriptFileName
        entries[compactPath] = object.TreeEntry{
1 unmodified line

Mode: filemode.Regular,
            Hash: blobHash,
        }
    return &boundary
}

// mergeFilesTouched combines two file lists, removing duplicates.
883 unmodified lines

if agentType == agent.AgentTypeCodex {
        compactBytes = codex.SanitizePortableTranscript(compactBytes)
    }
s.writeCompactTranscript(ctx, agentType, startLine, compactBytes, sessionPath, entries)
    compactStart := s.writeCompactTranscript(ctx, agentType, startLine, compactBytes, sessionPath, entries)

// Keep the session metadata's compact-transcript marker consistent with the
    // regenerated transcript.jsonl. The OPF re-redaction can write a compact
    // transcript the initial write lacked, or (in principle) shift line counts,
    // so re-record the boundary whenever one was produced.
    if compactStart != nil {
        if err := s.setCompactTranscriptStart(sessionPath, *compactStart, entries); err != nil {
            return fmt.Errorf("failed to update compact transcript start: %w", err)
        }
    }

return nil
}

// setCompactTranscriptStart reads the session metadata blob from entries, sets
// CompactTranscriptStart, and rewrites it. Used by the OPF rewrite path so the
// finalized session metadata reflects the regenerated compact transcript.
func (s *GitStore) setCompactTranscriptStart(sessionPath string, start int, entries map[string]object.TreeEntry) error {
    metadataPath := sessionPath + paths.MetadataFileName
    entry, exists := entries[metadataPath]
    if !exists {
        return fmt.Errorf("session metadata not found at %s", metadataPath)
    }

metadata, err := s.readMetadataFromBlob(entry.Hash)
    if err != nil {
        return fmt.Errorf("read session metadata: %w", err)
    }
    metadata.CompactTranscriptStart = &start

metadataJSON, err := jsonutil.MarshalIndentWithNewline(metadata, "", "  ")
    if err != nil {
        return fmt.Errorf("marshal session metadata: %w", err)
    }
    metadataHash, err := CreateBlobFromContent(s.repo, metadataJSON)
    if err != nil {
        return err
    }
    entries[metadataPath] = object.TreeEntry{
        Name: metadataPath,
        Mode: filemode.Regular,
        Hash: metadataHash,
    }
    return nil
}