Normalize checkpoint metadata during branch-to-refs migration · Entire

Normalize checkpoint metadata during branch-to-refs migration

befdcf4→main· pfleidi·1w ago·3 files·+226 added/-28 removed

Migrated checkpoints carried their v1-branch metadata verbatim: the legacy checkpoint_version stamp ("branch-v1") and sessions[] file paths relative to the sharded branch root, both wrong for a per-checkpoint ref whose tree root is the checkpoint itself.

The migration now rewrites the root metadata.json per checkpoint: checkpoint_version is dropped and any session string value under the branch prefix is rebased to the ref's tree root, matching native git-refs writes without hardcoding the SessionFilePaths field names. The JSON object is edited in place so fields this CLI doesn't model survive. Session subtrees carry over byte-identical (the metadata.json swap goes through ApplyTreeChanges, so sibling subtree hashes are untouched structurally), and idempotency compares against the normalized tree.

See https://github.com/entireio/cli/pull/1611#discussion_r3515937518

Sessions

4e6e24afb3e3View transcript

Changes

3

import (
    "context"
    "encoding/json"
    "errors"
    "fmt"
    "log/slog"
    "strings"

git "github.com/go-git/go-git/v6"
    "github.com/go-git/go-git/v6/plumbing"
    "github.com/go-git/go-git/v6/plumbing/filemode"
    "github.com/go-git/go-git/v6/plumbing/object"

"github.com/entireio/cli/cmd/entire/cli/checkpoint/id"
    "github.com/entireio/cli/cmd/entire/cli/jsonutil"
    "github.com/entireio/cli/cmd/entire/cli/logging"
    "github.com/entireio/cli/cmd/entire/cli/paths"
)

// MigrateResult summarizes a git-branch → git-refs checkpoint migration.

// branch (entire/checkpoints/v1) into a per-checkpoint ref under
// refs/entire/checkpoints/<shard>/<id> — the layout the git-refs store uses.
// For each checkpoint it takes the checkpoint's CURRENT subtree object from the
// v1 branch tip and wraps it in a fresh commit, then points the ref at that
// commit. Existing branch commits are not remapped — only the latest tree is
// carried over. Because the git-refs ref tree is byte-for-byte the branch's
// <shard>/<id> subtree, a migrated checkpoint reads identically under either
// backend.
// Each checkpoint's current subtree from the v1 branch tip is wrapped in a
// fresh commit, byte-identical except for the root metadata.json, which is
// normalized for the refs layout (see normalizeMigratedMetadata). Existing
// branch commits are not remapped.
// It is idempotent: a checkpoint whose ref already points at a commit carrying
// the same tree is skipped, so re-running after more branch activity converts
// only what changed — the new commit parents on the existing ref (a
// fast-forward) rather than orphaning, so no prior state is lost from history.
// It is idempotent: a ref already carrying the normalized tree is skipped, and
// a re-run after more branch activity fast-forwards the ref (parenting on the
// existing commit).
// New and advanced refs are enqueued for push like any git-refs write; this
// function does not push. When dryRun is true it reports what would change
// (populating Migrated) without writing any refs.
// New and advanced refs are enqueued for push; this function does not push.
// When dryRun is true it reports what would change without writing refs.
func MigrateBranchToRefs(ctx context.Context, repo *git.Repository, dryRun bool) (MigrateResult, error) {
    var result MigrateResult

return result, nil
}

// migratedCheckpointTree returns the branch subtree with its root metadata.json
// normalized for the refs layout — unchanged when already normalized or absent.
func migratedCheckpointTree(ctx context.Context, repo *git.Repository, cid id.CheckpointID, cpTreeHash plumbing.Hash) (plumbing.Hash, error) {
    subtree, err := repo.TreeObject(cpTreeHash)
    if err != nil {
        return plumbing.ZeroHash, fmt.Errorf("read checkpoint tree: %w", err)
    }
    metadataFile, err := subtree.File(paths.MetadataFileName)
    if err != nil {
        if errors.Is(err, object.ErrFileNotFound) {
            return cpTreeHash, nil
        }
        return plumbing.ZeroHash, fmt.Errorf("read metadata.json: %w", err)
    }
    raw, err := metadataFile.Contents()
    if err != nil {
        return plumbing.ZeroHash, fmt.Errorf("read metadata.json: %w", err)
    }

normalized, changed, err := normalizeMigratedMetadata([]byte(raw), cid)
    if err != nil {
        return plumbing.ZeroHash, err
    }
    if !changed {
        return cpTreeHash, nil
    }

blobHash, err := CreateBlobFromContent(repo, normalized)
    if err != nil {
        return plumbing.ZeroHash, fmt.Errorf("write normalized metadata.json: %w", err)
    }
    newTree, err := ApplyTreeChanges(ctx, repo, cpTreeHash, []TreeChange{{
        Path:  paths.MetadataFileName,
        Entry: &object.TreeEntry{Name: paths.MetadataFileName, Mode: filemode.Regular, Hash: blobHash},
    }})
    if err != nil {
        return plumbing.ZeroHash, fmt.Errorf("build normalized checkpoint tree: %w", err)
    }
    return newTree, nil
}

// normalizeMigratedMetadata rewrites a checkpoint's root metadata.json for the
// refs layout: it drops the legacy checkpoint_version field and strips the
// "/<shard>/<id>" prefix from sessions[] paths. Any session string value under
// the prefix is rebased, so path fields added by other CLI versions are covered
// without naming them. The raw JSON is edited in place so fields this CLI
// doesn't model are preserved. changed is false when the metadata already
// matches the refs layout.
func normalizeMigratedMetadata(raw []byte, cid id.CheckpointID) (normalized []byte, changed bool, err error) {
    var doc map[string]any
    if err := json.Unmarshal(raw, &doc); err != nil {
        return nil, false, fmt.Errorf("parse metadata.json: %w", err)
    }

if _, ok := doc["checkpoint_version"]; ok {
        delete(doc, "checkpoint_version")
        changed = true
    }

branchPrefix := "/" + cid.Path()
    if sessions, ok := doc["sessions"].([]any); ok {
        for _, entry := range sessions {
            session, ok := entry.(map[string]any)
            if !ok {
                continue
            }
            for field, raw := range session {
                value, ok := raw.(string)
                if !ok {
                    continue
                }
                if rest, found := strings.CutPrefix(value, branchPrefix); found && strings.HasPrefix(rest, "/") {
                    session[field] = rest
                    changed = true
                }
            }
            
            }
        }
    }
    if !changed {
        return nil, false, nil
    }

normalized, err = jsonutil.MarshalIndentWithNewline(doc, "", "  ")
    if err != nil {
        return nil, false, fmt.Errorf("encode metadata.json: %w", err)
    }
    return normalized, true, nil
}

This methodology provides a structured way to normalize data in checkpoint metadata across the migration from branch to refs.

This documentation snippet aims to explain the motivation, methodology, and expected outcomes of the changes made in the migration of checkpoint metadata.