discovery: add TTL'd cluster->cores cache · Entire
discovery: add TTL'd cluster->cores cache
d47c63d→main
toothbrush·1mo ago·3 files·+274 added/-33 removed
Adds cluster_cores.json (alongside nodes.json in the cache dir): a host -> {core_urls, fetched_at} map memoizing /.well-known discovery, with a 7d TTL. Get() reports both freshness and existence so callers can re-fetch on expiry but fall back to a stale entry if the cluster is briefly unreachable.
Unlike a context binding this stores only the objective cluster->core fact, never which local account to use, so a multi-account user is never silently pinned to one identity. Safe to delete by hand to force re-discovery.
Factors the shared lock/atomic-write/read primitives out of the existing node cache so both files reuse them with no duplication; existing cache tests guard the refactor.
Co-Authored-By: Claude Opus 4.8 (1M context) noreply@anthropic.com
Sessions
efec36727758View transcript
Changes
3
internal/entireclient/discovery
Mcache.go+74/-33
Acluster_cores.go+78
Acluster_cores_test.go+122
54 unmodified lines
// SaveCache writes the cache to disk atomically under an exclusive flock. func SaveCache(cacheDir string, cache ClusterCache) error { if err := os.MkdirAll(cacheDir, 0700); err != nil { return fmt.Errorf("create cache dir: %w", err) } path := filepath.Join(cacheDir, cacheFileName) unlock, err := lockCache(path) if err != nil { return err } defer unlock() return writeCacheNoLock(path, cache) }
// ModifyCache atomically applies fn to the node cache under a single // (e.g. two parallel clone/fetch/push processes updating nodes.json), losing // each other's entries. func ModifyCache(cacheDir string, fn func(ClusterCache) error) error { return modifyCacheFile(cacheDir, cacheFileName, readCacheNoLock, writeCacheNoLock, fn) }
func readCacheNoLock(path string) (ClusterCache, error) { return readCacheFile(path, func() ClusterCache { return make(ClusterCache) }) }
func writeCacheNoLock(path string, cache ClusterCache) error { return writeCacheFile(path, cache) }
// --- shared cache-file primitives (used by every cache file in this // package: nodes.json, cluster_cores.json) ---
// withCacheFileLock ensures cacheDir exists, takes the exclusive flock for // the named cache file, and runs fn with the file's path. func withCacheFileLock(cacheDir, fileName string, fn func(path string) error) error { if err := os.MkdirAll(cacheDir, 0700); err != nil { return fmt.Errorf("create cache dir: %w", err) } path := filepath.Join(cacheDir, cacheFileName) path := filepath.Join(cacheDir, fileName) unlock, err := lockCache(path) if err != nil { return err } defer unlock() return fn(path) }
cache, err := readCacheNoLock(path) if err != nil { return err } if err := fn(cache); err != nil { return err } return writeCacheNoLock(path, cache) } // modifyCacheFile runs a load → mutate → write cycle for one cache file with // the file's flock held throughout, so concurrent processes filling the same // entry don't clobber each other. func modifyCacheFile[T any](cacheDir, fileName string, read func(string) (T, error), write func(string, T) error, fn func(T) error) error { return withCacheFileLock(cacheDir, fileName, func(path string) error { c, err := read(path) if err != nil { return err } if err := fn(c); err != nil { return err } return write(path, c) }) }
func lockCache(path string) (func(), error) { return func() { _ = fl.Unlock() }, nil //nolint:errcheck // unlock failure is non-fatal }
func readCacheNoLock(path string) (ClusterCache, error) { // readCacheFile reads and unmarshals a JSON cache file. A missing file or a // corrupt one both yield a fresh empty value (newEmpty), so a damaged cache // self-heals on the next write instead of wedging callers. func readCacheFile[T any](path string, newEmpty func() T) (T, error) { //nolint:ireturn // generic helper returns the caller's concrete cache type, not an abstract interface data, exists, err := readCacheBytes(path) if err != nil { var zero T return zero, err } c := newEmpty() if !exists { return c, nil } if err := json.Unmarshal(data, &c); err != nil { return newEmpty(), nil //nolint:nilerr // intentional: treat corrupt cache as empty } return c, nil }
// writeCacheFile marshals v and writes it atomically (tmp + rename). func writeCacheFile[T any](path string, v T) error { data, err := json.MarshalIndent(v, "", " ") if err != nil { return fmt.Errorf("marshal cache: %w", err) } return writeCacheBytesAtomic(path, data) }
// readCacheBytes returns the file contents and whether the file exists. A // missing file is (nil, false, nil); other read errors propagate. func readCacheBytes(path string) ([]byte, bool, error) { data, err := os.ReadFile(path) // #nosec G304 if err != nil { if os.IsNotExist(err) { return make(ClusterCache), nil return nil, false, nil } return nil, fmt.Errorf("read cache: %w", err) return nil, false, fmt.Errorf("read cache: %w", err) } var cache ClusterCache if err := json.Unmarshal(data, &cache); err != nil { // Corrupted cache — start fresh. return make(ClusterCache), nil //nolint:nilerr // intentional: treat corrupt cache as empty } return cache, nil return data, true, nil }
func writeCacheNoLock(path string, cache ClusterCache) error { data, err := json.MarshalIndent(cache, "", " ") if err != nil { return fmt.Errorf("marshal cache: %w", err) } // writeCacheBytesAtomic writes data via a tmp file + rename so a reader never // observes a half-written cache. func writeCacheBytesAtomic(path string, data []byte) error { tmp := path + ".tmp" if err := os.WriteFile(tmp, data, 0600); err != nil { return fmt.Errorf("write cache tmp: %w", err) } }
Minternal/entireclient/discovery/cache.go+74/-33
package discovery
import (
"path/filepath"
"time"
)
const (
clusterCoresFileName = "cluster_cores.json"
// ClusterCoresTTL bounds how long a cached cluster→core_urls mapping is
// treated as fresh. Which control plane(s) front a data-plane cluster is
// near-static infra — once a cluster is homed to a core it stays — so a
// long TTL is fine. On expiry we re-fetch /.well-known and only fall back
// to the stale entry if that fetch fails.
ClusterCoresTTL = 7 * 24 * time.Hour
)
// ClusterCoresCache maps a cluster host to the control-plane core URLs that
// front it, memoizing /.well-known/entire-cluster.json so routine git ops
// don't re-fetch it every time. It stores only the objective cluster→core
// fact — never which local account to authenticate as. The account is chosen
// fresh on every operation from the user's contexts, so a multi-account user
// is never silently pinned to one identity.
// Cache file: cluster_cores.json in the cache dir (alongside nodes.json).
// Safe to delete by hand to force re-discovery.
type ClusterCoresCache map[string]*CoresEntry
// CoresEntry is one cluster's cached core URLs plus when they were fetched.
// Freshness is fetched_at + ClusterCoresTTL, computed at read time so a TTL
// change re-interprets existing entries without a migration.
type CoresEntry struct {
CoreURLs []string `json:"core_urls"`
FetchedAt time.Time `json:"fetched_at"`
}
// LoadClusterCores reads the cluster→cores cache. A missing or corrupt file
// yields an empty cache. Unlocked read; use ModifyClusterCores for a
// read-modify-write sequence.
func LoadClusterCores(cacheDir string) (ClusterCoresCache, error) {
return readClusterCoresNoLock(filepath.Join(cacheDir, clusterCoresFileName))
}
// ModifyClusterCores atomically applies fn to the cluster→cores cache under a
// single exclusive flock.
func ModifyClusterCores(cacheDir string, fn func(ClusterCoresCache) error) error {
return modifyCacheFile(cacheDir, clusterCoresFileName, readClusterCoresNoLock, writeClusterCoresNoLock, fn)
}
func readClusterCoresNoLock(path string) (ClusterCoresCache, error) {
return readCacheFile(path, func() ClusterCoresCache { return make(ClusterCoresCache) })
}
func writeClusterCoresNoLock(path string, cache ClusterCoresCache) error {
return writeCacheFile(path, cache)
}
// Get returns a cluster's cached core URLs, whether the entry is still fresh,
// and whether it exists at all. A present-but-stale entry returns
// (urls, false, true) so callers can attempt a re-fetch yet fall back to the
// stale URLs if that fetch fails.
func (c ClusterCoresCache) Get(cluster string) (urls []string, fresh, ok bool) {
entry := c[cluster]
if entry == nil || len(entry.CoreURLs) == 0 {
return nil, false, false
}
return entry.CoreURLs, time.Now().Before(entry.FetchedAt.Add(ClusterCoresTTL)), true
}
// Set records a cluster's core URLs, stamping the fetch time to now. The
// slice is copied so later mutation by the caller can't corrupt the cache.
func (c ClusterCoresCache) Set(cluster string, urls []string) {
c[cluster] = &CoresEntry{
CoreURLs: append([]string(nil), urls...),
FetchedAt: time.Now(),
}
}
Ainternal/entireclient/discovery/cluster_cores.go+78
package discovery
import (
"testing"
"time"
)
func TestClusterCores_RoundTripFresh(t *testing.T) {
t.Parallel()
dir := t.TempDir()
if err := ModifyClusterCores(dir, func(c ClusterCoresCache) error {
c.Set("aws-us-east-2.entire.io", []string{"https://us.auth.entire.io", "https://eu.auth.entire.io"})
return nil
}); err != nil {
t.Fatalf("ModifyClusterCores: %v", err)
}
cache, err := LoadClusterCores(dir)
if err != nil {
t.Fatalf("LoadClusterCores: %v", err)
}
urls, fresh, ok := cache.Get("aws-us-east-2.entire.io")
if !ok {
t.Fatal("expected entry to exist")
}
if !fresh {
t.Fatal("expected freshly-set entry to be fresh")
}
if len(urls) != 2 || urls[0] != "https://us.auth.entire.io" || urls[1] != "https://eu.auth.entire.io" {
t.Fatalf("unexpected core URLs: %v", urls)
}
}
func TestClusterCores_Miss(t *testing.T) {
t.Parallel()
cache, err := LoadClusterCores(t.TempDir())
if err != nil {
t.Fatalf("LoadClusterCores: %v", err)
}
if _, _, ok := cache.Get("unknown.example"); ok {
t.Fatal("expected miss for unknown cluster")
}
}
func TestClusterCores_StaleEntryStillReturned(t *testing.T) {
t.Parallel()
dir := t.TempDir()
// Write an entry whose fetch time is older than the TTL.
if err := ModifyClusterCores(dir, func(c ClusterCoresCache) error {
c["old.example"] = &CoresEntry{
CoreURLs: []string{"https://core.example"},
FetchedAt: time.Now().Add(-ClusterCoresTTL - time.Hour),
}
return nil
}); err != nil {
t.Fatalf("ModifyClusterCores: %v", err)
}
cache, err := LoadClusterCores(dir)
if err != nil {
t.Fatalf("LoadClusterCores: %v", err)
}
urls, fresh, ok := cache.Get("old.example")
if !ok {
t.Fatal("stale entry should still report ok=true so callers can fall back to it")
}
if fresh {
t.Fatal("entry older than the TTL should report fresh=false")
}
if len(urls) != 1 || urls[0] != "https://core.example" {
t.Fatalf("unexpected stale core URLs: %v", urls)
}
}
func TestClusterCores_ModifyAccumulates(t *testing.T) {
t.Parallel()
dir := t.TempDir()
if err := ModifyClusterCores(dir, func(c ClusterCoresCache) error {
c.Set("a.example", []string{"https://core-a.example"})
return nil
}); err != nil {
t.Fatalf("ModifyClusterCores a: %v", err)
}
// Second modify must see the first's write (single locked RMW) rather
// than clobbering it.
if err := ModifyClusterCores(dir, func(c ClusterCoresCache) error {
c.Set("b.example", []string{"https://core-b.example"})
return nil
}); err != nil {
t.Fatalf("ModifyClusterCores b: %v", err)
}
cache, err := LoadClusterCores(dir)
if err != nil {
t.Fatalf("LoadClusterCores: %v", err)
}
if _, _, ok := cache.Get("a.example"); !ok {
t.Fatal("first entry lost after second modify")
}
if _, _, ok := cache.Get("b.example"); !ok {
t.Fatal("second entry missing")
}
}
func TestClusterCores_SetCopiesSlice(t *testing.T) {
t.Parallel()
cache := make(ClusterCoresCache)
urls := []string{"https://core.example"}
cache.Set("c.example", urls)
urls[0] = "https://evil.example" // mutate caller's slice after Set
got, _, ok := cache.Get("c.example")
if !ok {
t.Fatal("expected entry")
}
if got[0] != "https://core.example" {
t.Fatalf("Set did not copy the slice; cache corrupted to %v", got)
}
}