Add `entire doctor rewrite-checkpoints` (re-materialize branch checkpoints as refs) · Entire
Add entire doctor rewrite-checkpoints (re-materialize branch checkpoints as refs)
06b3c99·
Soph·2w ago·4 files·+535 added/-0 removed
A second, data-copying migration alongside the byte-identical migrate-checkpoints. RewriteBranchToRefs re-drives the git-refs store's write path for every checkpoint on the v1 branch:
- Per session it replays the transcript (regenerating the compact transcript.jsonl), prompts, and metadata through Write, so the tree is rooted at the checkpoint (no shard folders) and SessionFilePaths/CompactTranscriptStart are correct by construction. Per-session summaries and combined attribution are replayed too; a tasks/ subtree is grafted in unchanged.
- The checkpoint id is preserved, and each commit keeps the original author — read from the v1-branch commit that wrote the session (metadata carries no author), falling back to the repo git author.
- Fresh commits copy the data; the branch's own commits are not reused.
Idempotent: existing refs are skipped unless --force re-materializes them. --dry-run reports without writing. Push follows the same policy as migrate-checkpoints: interactive asks, non-interactive never pushes.
Co-Authored-By: Claude Opus 4.8 (1M context) noreply@anthropic.com
Sessions
75925b9fdc24View transcript
Changes
4
cmd/entire/cli
checkpoint
Arewrite.go+287
Arewrite_test.go+138
Mdoctor.go+1
Adoctor_rewrite.go+109
package checkpoint
import (
"context"
"errors"
"fmt"
"log/slog"
git "github.com/go-git/go-git/v6"
"github.com/go-git/go-git/v6/plumbing"
"github.com/go-git/go-git/v6/plumbing/filemode"
"github.com/go-git/go-git/v6/plumbing/object"
"github.com/entireio/cli/cmd/entire/cli/checkpoint/id"
"github.com/entireio/cli/cmd/entire/cli/logging"
"github.com/entireio/cli/cmd/entire/cli/trailers"
"github.com/entireio/cli/redact"
)
// RewriteResult summarizes a git-branch → git-refs checkpoint rewrite.
type RewriteResult struct {
// Total is the number of checkpoints found on the v1 branch.
Total int
// Rewritten lists the checkpoints re-materialized as refs.
Rewritten []id.CheckpointID
// Skipped counts checkpoints left alone because their ref already exists.
Skipped int
}
// RewriteBranchToRefs re-materializes every checkpoint on the git-branch v1
// branch as a per-checkpoint ref by re-driving the git-refs store's write path.
//
// Unlike a byte-identical copy, this reads each checkpoint's data and writes it
// fresh: for every session it replays the transcript (so the compact
// transcript.jsonl is regenerated), prompts, and metadata at the ref's tree root
// (no shard folders), then replays the per-session summaries and the combined
// attribution. Any tasks/ subtree is grafted in unchanged. The checkpoint id is
// preserved, and each commit keeps the original author — read from the v1-branch
// commit that wrote the session (metadata carries no author), falling back to
// the repo's git author.
//
// Idempotent: a checkpoint whose ref already exists is skipped unless force is
// set (which re-materializes it from scratch). When dryRun is true it reports
// what would be rewritten without writing anything.
func RewriteBranchToRefs(ctx context.Context, repo *git.Repository, dryRun, force bool) (RewriteResult, error) {
var result RewriteResult
branch := NewGitStore(repo, DefaultV1Refs())
tree, err := branch.getSessionsBranchTree()
if err != nil {
if errors.Is(err, plumbing.ErrReferenceNotFound) {
return result, nil // no v1 branch → nothing to rewrite
}
return result, fmt.Errorf("read v1 checkpoint branch: %w", err)
}
authors := branchSessionAuthors(ctx, repo)
fallbackName, fallbackEmail := GetGitAuthorFromRepo(repo)
refsStore := newGitRefsStore(repo)
walkErr := WalkCheckpointShards(ctx, repo, tree, func(cid id.CheckpointID, cpTreeHash plumbing.Hash) error {
if err := ctx.Err(); err != nil {
return err //nolint:wrapcheck // propagate context cancellation
}
result.Total++
refName, err := RefName(cid)
if err != nil {
logging.Warn(ctx, "rewrite: skipping checkpoint with unmappable id",
slog.String("id", cid.String()), slog.String("error", err.Error()))
return nil
}
if _, err := repo.Reference(refName, true); err == nil && !force {
result.Skipped++
return nil
}
if dryRun {
result.Rewritten = append(result.Rewritten, cid)
return nil
}
if err := rewriteCheckpoint(ctx, branch, refsStore, repo, cid, cpTreeHash, authors, fallbackName, fallbackEmail); err != nil {
return fmt.Errorf("rewrite checkpoint %s: %w", cid, err)
}
result.Rewritten = append(result.Rewritten, cid)
return nil
})
if walkErr != nil {
return result, fmt.Errorf("walk v1 checkpoints: %w", walkErr)
}
return result, nil
}
// commitAuthor is a git author identity.
type commitAuthor struct {
Name string
Email string
}
// branchSessionAuthors maps each session id to the author of the earliest
// v1-branch commit that wrote it, so a rewrite can preserve the original author.
// The map is best-effort: a session with no resolvable commit falls back to the
// repo author at the call site.
func branchSessionAuthors(ctx context.Context, repo *git.Repository) map[string]commitAuthor {
authors := make(map[string]commitAuthor)
ref, err := repo.Reference(DefaultV1Refs().Read, true)
if err != nil {
return authors
}
iter, err := repo.Log(&git.LogOptions{From: ref.Hash()})
if err != nil {
return authors
}
defer iter.Close()
_ = iter.ForEach(func(c *object.Commit) error { //nolint:errcheck // best-effort; partial map is acceptable
if err := ctx.Err(); err != nil {
return err //nolint:wrapcheck // stop the walk on cancellation
}
if sessionID, ok := trailers.ParseSession(c.Message); ok {
authors[sessionID] = commitAuthor{Name: c.Author.Name, Email: c.Author.Email}
}
return nil
})
return authors
}
// rewriteCheckpoint re-materializes one checkpoint as a ref by replaying its
// sessions, summaries, attribution, and tasks through the git-refs write path.
func rewriteCheckpoint(
ctx context.Context,
branch *GitStore,
refsStore *gitRefsStore,
repo *git.Repository,
cid id.CheckpointID,
cpTreeHash plumbing.Hash,
authors map[string]commitAuthor,
fallbackName, fallbackEmail string,
) error {
summary, err := branch.Read(ctx, cid)
if err != nil {
return fmt.Errorf("read checkpoint summary: %w", err)
}
if summary == nil {
return errors.New("checkpoint summary not found on branch")
}
// Start clean: drop any existing ref so the replay builds a fresh history
// rooted at an orphan commit rather than parenting on stale content.
refName, err := RefName(cid)
if err != nil {
return err
}
if _, err := repo.Reference(refName, true); err == nil {
if err := repo.Storer.RemoveReference(refName); err != nil {
return fmt.Errorf("reset existing ref: %w", err)
}
}
// Replay each session in order. Writing session N then its summary keeps
// SessionSummary (which targets the latest session) pointed at the right one.
firstAuthor := commitAuthor{Name: fallbackName, Email: fallbackEmail}
for idx := range summary.Sessions {
content, err := branch.ReadSessionContent(ctx, cid, idx)
if err != nil {
return fmt.Errorf("read session %d: %w", idx, err)
}
m := content.Metadata
name, email := fallbackName, fallbackEmail
if a, ok := authors[m.SessionID]; ok && a.Name != "" {
name, email = a.Name, a.Email
}
if idx == 0 {
firstAuthor = commitAuthor{Name: name, Email: email}
}
if err := refsStore.Write(ctx, Session(WriteOptions{
CheckpointID: cid,
SessionID: m.SessionID,
CreatedAt: m.CreatedAt,
Strategy: m.Strategy,
Branch: m.Branch,
Transcript: redact.AlreadyRedacted(content.Transcript),
Prompts: []string{content.Prompts},
FilesTouched: m.FilesTouched,
CheckpointsCount: m.CheckpointsCount,
SaveStepCount: m.SaveStepCount,
AuthorName: name,
AuthorEmail: email,
Agent: m.Agent,
Model: m.Model,
TurnID: m.TurnID,
TranscriptIdentifierAtStart: m.TranscriptIdentifierAtStart,
CheckpointTranscriptStart: m.CheckpointTranscriptStart,
TokenUsage: m.TokenUsage,
SkillEvents: m.SkillEvents,
})); err != nil {
return fmt.Errorf("write session %d: %w", idx, err)
}
if m.Summary != nil {
if err := refsStore.Write(ctx, SessionSummary{CheckpointID: cid, Summary: m.Summary}); err != nil {
return fmt.Errorf("write session %d summary: %w", idx, err)
}
}
}
if summary.CombinedAttribution != nil {
if err := refsStore.Write(ctx, CheckpointAttribution{CheckpointID: cid, Attribution: summary.CombinedAttribution}); err != nil {
return fmt.Errorf("write combined attribution: %w", err)
}
}
// Graft the tasks/ subtree unchanged if the checkpoint has one — task steps
// aren't reconstructable through the write union, so the existing tree is
// carried over verbatim.
if tasksHash, ok := subtreeEntryHash(repo, cpTreeHash, "tasks"); ok {
if err := graftTasksSubtree(ctx, repo, refsStore, cid, tasksHash, firstAuthor); err != nil {
return fmt.Errorf("graft tasks: %w", err)
}
}
return nil
}
// subtreeEntryHash returns the hash of a named directory entry in a tree.
func subtreeEntryHash(repo *git.Repository, treeHash plumbing.Hash, name string) (plumbing.Hash, bool) {
tree, err := repo.TreeObject(treeHash)
if err != nil {
return plumbing.ZeroHash, false
}
for _, e := range tree.Entries {
if e.Name == name && e.Mode == filemode.Dir {
return e.Hash, true
}
}
return plumbing.ZeroHash, false
}
// graftTasksSubtree adds the tasks/ subtree to the checkpoint's current ref tree
// via one commit on top, reusing the existing task objects unchanged.
func graftTasksSubtree(ctx context.Context, repo *git.Repository, refsStore *gitRefsStore, cid id.CheckpointID, tasksHash plumbing.Hash, author commitAuthor) error {
refName, err := RefName(cid)
if err != nil {
return err
}
ref, err := repo.Reference(refName, true)
if err != nil {
return fmt.Errorf("resolve ref: %w", err)
}
commit, err := repo.CommitObject(ref.Hash())
if err != nil {
return fmt.Errorf("read ref commit: %w", err)
}
rootTree, err := commit.Tree()
if err != nil {
return fmt.Errorf("read ref tree: %w", err)
}
tasksTree, err := repo.TreeObject(tasksHash)
if err != nil {
return fmt.Errorf("read tasks subtree: %w", err)
}
entries := make(map[string]object.TreeEntry)
if err := FlattenTree(repo, rootTree, "", entries); err != nil {
return fmt.Errorf("flatten ref tree: %w", err)
}
if err := FlattenTree(repo, tasksTree, "tasks", entries); err != nil {
return fmt.Errorf("flatten tasks subtree: %w", err)
}
newTree, err := BuildTreeFromEntries(ctx, repo, entries)
if err != nil {
return fmt.Errorf("build grafted tree: %w", err)
}
if newTree == commit.TreeHash {
return nil // tasks already present, nothing to graft
}
msg := fmt.Sprintf("Graft tasks for checkpoint %s (migrated from git-branch)", cid)
graftCommit, err := CreateCommit(ctx, repo, newTree, ref.Hash(), msg, author.Name, author.Email)
if err != nil {
return fmt.Errorf("commit grafted tasks: %w", err)
}
return refsStore.setRef(ctx, cid, graftCommit)
}
// rootTreeEntryNames returns the top-level entry names of a checkpoint ref's tree.
func rootTreeEntryNames(t *testing.T, repo *git.Repository, cid id.CheckpointID) map[string]bool {
t.Helper()
commit, err := repo.CommitObject(refHash(t, repo, cid))
require.NoError(t, err)
tree, err := commit.Tree()
require.NoError(t, err)
names := make(map[string]bool, len(tree.Entries))
for _, e := range tree.Entries {
names[e.Name] = true
}
return names
}
func TestRewriteBranchToRefs(t *testing.T) {
t.Parallel()
repo, _ := setupBranchTestRepo(t)
ctx := context.Background()
branch := NewGitStore(repo, DefaultV1Refs())
cid := id.MustCheckpointID("a1b2c3d4e5f6")
seedBranchCheckpoint(t, branch, cid, "s1") // authored by Test <test@test.com>
result, err := RewriteBranchToRefs(ctx, repo, false, false)
require.NoError(t, err)
assert.Equal(t, 1, result.Total)
assert.Len(t, result.Rewritten, 1)
assert.Equal(t, 0, result.Skipped)
// Reads back through the git-refs store.
refsStore := newGitRefsStore(repo)
summary, err := refsStore.Read(ctx, cid)
require.NoError(t, err)
require.NotNil(t, summary)
assert.Equal(t, cid, summary.CheckpointID)
require.Len(t, summary.Sessions, 1)
// Root-flat: checkpoint files sit at the tree root (no <shard>/<id> nesting).
names := rootTreeEntryNames(t, repo, cid)
assert.True(t, names["metadata.json"], "checkpoint metadata should be at the ref tree root")
assert.True(t, names["0"], "session 0 should be at the ref tree root")
assert.False(t, names["a1"], "there must be no shard folder in the ref tree")
// The commit keeps the original author (read from the branch commit).
commit, err := repo.CommitObject(refHash(t, repo, cid))
require.NoError(t, err)
assert.Equal(t, "Test", commit.Author.Name)
assert.Equal(t, "test@test.com", commit.Author.Email)
}
func TestRewriteBranchToRefs_IdempotentAndForce(t *testing.T) {
t.Parallel()
repo, _ := setupBranchTestRepo(t)
ctx := context.Background()
branch := NewGitStore(repo, DefaultV1Refs())
cid := id.MustCheckpointID("a1b2c3d4e5f6")
seedBranchCheckpoint(t, branch, cid, "s1")
_, err := RewriteBranchToRefs(ctx, repo, false, false)
require.NoError(t, err)
before := refHash(t, repo, cid)
// Re-run without force: the existing ref is skipped and left untouched.
again, err := RewriteBranchToRefs(ctx, repo, false, false)
require.NoError(t, err)
assert.Empty(t, again.Rewritten, "existing ref should be skipped")
assert.Equal(t, 1, again.Skipped)
assert.Equal(t, before, refHash(t, repo, cid), "skipped run must not move the ref")
// Re-run with force: the checkpoint is re-materialized and still reads back.
forced, err := RewriteBranchToRefs(ctx, repo, false, true)
require.NoError(t, err)
assert.Len(t, forced.Rewritten, 1, "force re-materializes the checkpoint")
refsStore := newGitRefsStore(repo)
summary, err := refsStore.Read(ctx, cid)
require.NoError(t, err)
require.NotNil(t, summary)
assert.Equal(t, cid, summary.CheckpointID)
}
func TestRewriteBranchToRefs_MultipleSessions(t *testing.T) {
t.Parallel()
repo, _ := setupBranchTestRepo(t)
ctx := context.Background()
branch := NewGitStore(repo, DefaultV1Refs())
cid := id.MustCheckpointID("a1b2c3d4e5f6")
seedBranchCheckpoint(t, branch, cid, "sess-1")
seedBranchCheckpoint(t, branch, cid, "sess-2")
result, err := RewriteBranchToRefs(ctx, repo, false, false)
require.NoError(t, err)
assert.Len(t, result.Rewritten, 1)
refsStore := newGitRefsStore(repo)
summary, err := refsStore.Read(ctx, cid)
require.NoError(t, err)
require.Len(t, summary.Sessions, 2, "both sessions replayed into the ref checkpoint")
names := rootTreeEntryNames(t, repo, cid)
assert.True(t, names["0"] && names["1"], "both session dirs at root")
}
func TestRewriteBranchToRefs_DryRunAndNoBranch(t *testing.T) {
t.Parallel()
ctx := context.Background()
// No v1 branch → no-op.
emptyRepo, _ := setupBranchTestRepo(t)
res, err := RewriteBranchToRefs(ctx, emptyRepo, false, false)
require.NoError(t, err)
assert.Equal(t, 0, res.Total)
// Dry run reports without writing.
repo, _ := setupBranchTestRepo(t)
branch := NewGitStore(repo, DefaultV1Refs())
cid := id.MustCheckpointID("a1b2c3d4e5f6")
seedBranchCheckpoint(t, branch, cid, "s1")
dry, err := RewriteBranchToRefs(ctx, repo, true, false)
require.NoError(t, err)
assert.Len(t, dry.Rewritten, 1)
refName, err := RefName(cid)
require.NoError(t, err)
_, err = repo.Reference(refName, true)
assert.Error(t, err, "dry-run must not write a ref")
}