data-api: fold discovery cache into the cluster cores cache

6cc6266→main· toothbrush·1mo ago·9 files·+198 added/-351 removed

aud == base URI holds on both data-API environments (entire.io / partial.to), and the token manager already defaults the RFC 8693 audience to the resource origin it's dialing. So the CLI never needs the advertised audience: it derives it from the host. That removes the one field distinguishing the API discovery cache from the git cluster's cores cache.

Fold accordingly: - discovery: api_discovery.json reuses ClusterCoresCache via LoadAPICores/ ModifyAPICores (deleted the bespoke APIDiscoveryCache/Entry); cores cache now serves both clusters and data APIs, two files. - clusterdiscovery: APIResponse slims to {trusted_issuers}; DiscoverAPI requires only trusted_issuers; resolveAPITrustedIssuers mirrors resolveClusterCores; ResolveContextForAPI returns just the context (no doc). - auth: NewRefreshingResourceProvider drops the audience param (token manager defaults aud to the resource origin); ResolveDataAPIToken + seams updated.

Trades away "server changes audience without a CLI release" — fine, aud == base URI is a hard requirement both envs. Doc updated.

Sessions

75020e07c077View transcript

Changes

9

• cmd/entire/cli/auth

• Mdata_api.go+7/-5

• Mdata_api_test.go+15/-17

• Mrefresh.go+11/-10

• docs/architecture

• Mupstream-host-resolution.md+33/-22

• internal/entireclient

• clusterdiscovery

• Mapi_discovery.go+54/-70

• Mapi_discovery_test.go+26/-38

• discovery

• Dapi_discovery.go-86

• Mapi_discovery_test.go+22/-98

• Mcluster_cores.go+30/-5

""" 20 unmodified lines

16 unmodified lines

44 45 46 47 48 47 48 49 50 51 36 unmodified lines

88 89 90 91 91 92 93 94 2 unmodified lines

97 98 99 100 101 102 101 103 104 105 106

20 unmodified lines

// resolveContextForAPIFunc is the shape of the discovery seam: it mirrors // clusterdiscovery.ResolveContextForAPI (ctx, configDir, cacheDir, apiHost, // httpClient, debugf). type resolveContextForAPIFunc func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, *clusterdiscovery.APIResponse, error) type resolveContextForAPIFunc func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, error)

// resolveContextForAPI is the discovery seam, swapped in tests so they don't // reach the network. See SetResolveContextForAPIForTest for cross-package tests.

// DiscoveryUnavailableForTest is a ready-made SetResolveContextForAPIForTest // value that forces the discovery-unavailable fallback (no network), so a // cross-package test exercises the static TokenForResource path deterministically. func DiscoveryUnavailableForTest(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, *clusterdiscovery.APIResponse, error) { return nil, nil, clusterdiscovery.ErrDiscoveryUnavailable

func DiscoveryUnavailableForTest(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, error) { return nil, clusterdiscovery.ErrDiscoveryUnavailable }

// ResolveDataAPIToken returns a bearer for the data API at dataBaseURL.

36 unmodified lines

defer cancel() httpClient := &http.Client{Timeout: dataAPIDiscoveryTimeout}

selected, doc, err := resolveContextForAPI(dctx, contexts.DefaultConfigDir(), discovery.DefaultCacheDir(), host, httpClient, nil) selected, err := resolveContextForAPI(dctx, contexts.DefaultConfigDir(), discovery.DefaultCacheDir(), host, httpClient, nil) if errors.Is(err, clusterdiscovery.ErrDiscoveryUnavailable) { // Old deployment / not rolled out / transient — preserve today's behaviour. return TokenForResource(ctx, dataOrigin) 2 unmodified lines

return "", err }

// Exchange for the data host origin; the token manager derives the RFC 8693 // audience from it, which is the aud the API requires (aud == base URI). allowInsecure := insecureHTTPEnabled() || isLoopbackHTTP(selected.CoreURL) provider, err := NewRefreshingResourceProvider(selected, dataOrigin, doc.Audience, nil, allowInsecure) provider, err := NewRefreshingResourceProvider(selected, dataOrigin, nil, allowInsecure) if err != nil { return "", err } """

Mcmd/entire/cli/auth/data_api.go+7/-5

"""

41 unmodified lines

24 unmodified lines

74 75 76 77 78 77 78 79 80 81 3 unmodified lines

85 86 87 88 89 90 88 89 90 91 92 93 94 95 24 unmodified lines

120 121 122 121 122 123 124 125 126 123 124 125 126 127 8 unmodified lines

136 137 138 141 139 140 141 142 2 unmodified lines

145 146 147 150 148 149 150 153 151 152 153 154 7 unmodified lines

162 163 164 167 165 166 167 168

41 unmodified lines

restore := tokenstore.UseFileBackendForTesting(filepath.Join(t.TempDir(), "tokens.json")) t.Cleanup(restore)

stubResolveContextForAPI(t, func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, *clusterdiscovery.APIResponse, error) { return nil, nil, fmt.Errorf("%w: 404", clusterdiscovery.ErrDiscoveryUnavailable) stubResolveContextForAPI(t, func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, error) { return nil, fmt.Errorf("%w: 404", clusterdiscovery.ErrDiscoveryUnavailable) })

// Pin the singleton manager to an empty store so the static fallback's 24 unmodified lines

t.Cleanup(restore)

sentinel := errors.New("multiple login contexts can authenticate against API host entire.io") stubResolveContextForAPI(t, func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, *clusterdiscovery.APIResponse, error) { return nil, nil, sentinel stubResolveContextForAPI(t, func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, error) { return nil, sentinel })

_, err := ResolveDataAPIToken(context.Background(), "https://entire.io") 3 unmodified lines

// The success path: discovery picks a context, and the provider exchanges that // context's login JWT at its core for the advertised audience, returning the // exchanged token. func TestResolveDataAPIToken_ExchangesForAdvertisedAudience(t *testing.T) { // context's login JWT at its core for an audience equal to the data host // origin (the aud the API requires), returning the exchanged token. The audience is derived from the resource origin by the token manager, not read // from discovery. func TestResolveDataAPIToken_ExchangesForDataHostOrigin(t *testing.T) { t.Setenv("ENTIRE_CONFIG_DIR", t.TempDir()) restore := tokenstore.UseFileBackendForTesting(filepath.Join(t.TempDir(), "tokens.json")) t.Cleanup(restore) 24 unmodified lines

} ctxObj := &contexts.Context{Name: "me@core", CoreURL: srv.URL, Handle: "me", KeychainService: svc};

stubResolveContextForAPI(t, func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, *clusterdiscovery.APIResponse, error) { return ctxObj, &clusterdiscovery.APIResponse{ Issuer: srv.URL, TrustedIssuers: []string{srv.URL}, Audience: wantAudience, }, nil stubResolveContextForAPI(t, func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, error) { return ctxObj, nil })

// allowInsecure flows from the loopback http core (srv.URL) automatically. 8 unmodified lines

t.Fatalf("grant_type = %q, want token-exchange", gotGrant) } if gotAudience != wantAudience { t.Fatalf("audience = %q, want the advertised audience %q", gotAudience, wantAudience) t.Fatalf("audience = %q, want the data host origin %q (derived from the resource)", gotAudience, wantAudience) } if want := mustOrigin(t, dataOrigin); gotResource != want { t.Fatalf("resource = %q, want the data origin %q", gotResource, want) 2 unmodified lines

func TestNewRefreshingResourceProvider_Validation(t *testing.T) { t.Parallel() if _, err := NewRefreshingResourceProvider(nil, "https://data.example", "aud", nil, false); err == nil { if _, err := NewRefreshingResourceProvider(nil, "https://data.example", nil, false); err == nil { t.Fatal("want error for nil context") } if _, err := NewRefreshingResourceProvider(&contexts.Context{Name: "x", CoreURL: "https://core.example"}, "https://data.example", "aud", nil, false); err == nil { if _, err := NewRefreshingResourceProvider(&contexts.Context{Name: "x", CoreURL: "https://core.example"}, "https://data.example", nil, false); err == nil { t.Fatal("want error for a context with no keychain slot") } }

7 unmodified lines

t.Cleanup(restore)

c := &contexts.Context{Name: "me@core", CoreURL: "https://core.example", Handle: "me", KeychainService: "kc:me"} provider, err := NewRefreshingResourceProvider(c, "https://data.example", "https://data.example", nil, false) provider, err := NewRefreshingResourceProvider(c, "https://data.example", nil, false) if err != nil { t.Fatalf("NewRefreshingResourceProvider: %v", err) } """

Mcmd/entire/cli/auth/data_api_test.go+15/-17

"""

238 unmodified lines

238 unmodified lines

// NewRefreshingResourceProvider returns a provider that mints a bearer valid // for resourceOrigin carrying the given audience, by exchanging context c's // login JWT at c's own core (RFC 8693). It is NewRefreshingLoginProvider's // sibling for resource servers: where that returns the bare login JWT (the // control plane / cluster cases, where the host is the core), this performs // the token exchange the data API requires. // for resourceOrigin, by exchanging context c's login JWT at c's own core (RFC // 8693). It is NewRefreshingLoginProvider's sibling for resource servers: where // that returns the bare login JWT (the control plane / cluster cases, where the // host is the core), this performs the token exchange the data API requires. // // Both the silent login-JWT re-mint and the exchange run through the shared // per-context tokenmanager (newContextTokenManager). resourceOrigin must // already be origin-only (no path); audience is passed verbatim as the RFC // 8693 audience param. Exchanged tokens are cached in-process by the // tokenmanager for the life of this process. // already be origin-only (no path). No audience is passed: the token manager // defaults the RFC 8693 audience to the resource origin, which is exactly what // the data API requires (aud == its base URI), so the audience is derived from // the host being dialed rather than read from discovery. Exchanged tokens are // cached in-process by the tokenmanager for the life of this process. // // transport carries the caller's TLS configuration; allowInsecureHTTP permits // an http:// core/resource for loopback/dev. func NewRefreshingResourceProvider(c *contexts.Context, resourceOrigin, audience string, transport http.RoundTripper, allowInsecureHTTP bool) (func(context.Context) (string, error), error) { func NewRefreshingResourceProvider(c *contexts.Context, resourceOrigin string, transport http.RoundTripper, allowInsecureHTTP bool) (func(context.Context) (string, error), error) {

mgr, err := newContextTokenManager(c, transport, allowInsecureHTTP) if err != nil { return nil, err } req := tokenmanager.TokenRequest{Resource: resourceOrigin, Audience: audience} req := tokenmanager.TokenRequest{Resource: resourceOrigin} return func(ctx context.Context) (string, error) { tok, err := mgr.Token(ctx, req) if mapped := contextReauthError(c, err); mapped != nil { """

Mcmd/entire/cli/auth/refresh.go+11/-10

"""

19 unmodified lines

20 21 22 23 23 24 25 26 61 unmodified lines

88 89 90 91 92 93 91 92 93 94 95 96 97 98 99 100 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 104 105 111 112 113 114 115 116 117 118 119 111 112 113 120 121 122 123 115 116 117 118 124 125 126 127 128 129 130 131 2 unmodified lines

134 135 136 127 137 138 139 140 141 130 131 142

19 unmodified lines

|---|---|---|---| | Core — IdP and control-plane API, co-located | entire-core (us.auth.entire.io) | org / repo / project / grant, auth *, login | none needed — the host is the core | | Resource: git cluster | entire-server / entiredb | git-remote-entire (clone/push) | /.well-known/entire-cluster.json → core_urls | | Resource: web/data API | entire.io (partial.to) | activity / search / trail / dispatch | /.well-known/entire-api.json → trusted_issuers + audience | | Resource: web/data API | entire.io (partial.to) | activity / search / trail / dispatch | /.well-known/entire-api.json → trusted_issuers (audience = the host origin) |

contexts.json ($ENTIRE_CONFIG_DIR/contexts.json, shared with entiredb's CLIs) stores each login as {Name, CoreURL, Handle, KeychainService} plus a 61 unmodified lines

}

"""

The CLI reads only trusted_issuers and audience; jwks_uris is a server-side verification concern (the CLI never fetches JWKS) and is ignored on decode, so the server can evolve that field freely. The CLI reads only trusted_issuers — exactly the way the git path reads a cluster's core_urls. issuer, audience, and jwks_uris are advertised but ignored on decode (see the audience note below).

Audience = the data host origin, not an opaque string. entire.io's ENTIRE_CORE_JWT_AUDIENCE is https://entire.io (prod) / https://partial.to (staging). The core's STS exchange stamps aud = the requested audience for api-access exchanges, so the CLI requests exactly the advertised value. We advertise it (rather than assume host==audience) so the server can change it without a CLI release. Audience = the data host origin. entire.io's ENTIRE_CORE_JWT_AUDIENCE is https://entire.io (prod) / https://partial.to (staging) — the data host's own base URI, on both environments. The token manager already defaults the RFC 8693 audience to the resource origin it's exchanging for, so dialing https://entire.io produces aud = https://entire.io with no special handling. The CLI therefore derives the audience from the host it's already dialing rather than reading the advertised audience field. (This trades away the "server changes audience without a CLI release" flexibility — acceptable because aud == base URI is a hard requirement on both environments.)

Because the only field the CLI consumes is the trusted-issuer list — which is a set of core URLs — the data-API discovery cache is literally the git cluster's cores cache (ClusterCoresCache), in a separate file (api_discovery.json).

Resolution (auth.ResolveDataAPIToken):

1. Fetch the API host's /.well-known/entire-api.json (TLS-authenticated — it's a trust root) and read trusted_issuers + audience.
2. Resolve the API host's trusted issuers: api_discovery.json when fresh, else a live /.well-known/entire-api.json fetch (TLS-authenticated — it's a trust root; redirects refused), cached with a 24h TTL and stale-fallback on a failed re-fetch. Same resolveClusterCores shape the git path uses.
3. Pick the context with the same cluster semantics as the git path: active-context-wins-if-eligible → sole eligible → explicit-choice error. This is the lever that makes ENTIRE_API_BASE_URL=https://partial.to entire authenticate as the partial.to login even while the active context is a prod entire.io login — without also setting ENTIRE_AUTH_BASE_URL`.
4. Exchange that context's login JWT at its core for the advertised audience (auth.NewRefreshingResourceProvider, keyed on c.CoreURL like the control-plane provider, plus the RFC 8693 step the data API requires).
5. Exchange that context's login JWT at its core for the data host origin (auth.NewRefreshingResourceProvider, keyed on c.CoreURL like the control-plane provider; the token manager sets aud = that origin).
6. Fallback: if the host doesn't advertise discovery (404 / unreachable / 503 / malformed — a deployment predating the well-known), fall back to the pre-discovery static path (TokenForResource via the singleton manager), so behaviour is never worse than before. A reachable host whose context selection fails surfaces that error — the user must log in or pick one. 503 / malformed) and no cache entry exists, fall back to the pre-discovery static path (TokenForResource via the singleton manager), so behaviour is never worse than before. A reachable host whose context selection fails surfaces that error — the user must log in or pick one. (A transient outage with a warm cache uses the stale entry, not the fallback.)

The selection rule differs from the control plane (where the active context always wins because there's no host to match): here a host is matched, so

Key files: cmd/entire/cli/auth/data_api.go (ResolveDataAPIToken + fallback), cmd/entire/cli/auth/refresh.go (NewRefreshingResourceProvider), internal/entireclient/clusterdiscovery/api_discovery.go (DiscoverAPI, ResolveContextForAPI, sharing selectContext with the cluster path). Seams: ResolveContextForAPI, sharing selectContext and the cores cache with the cluster path), internal/entireclient/discovery/cluster_cores.go (LoadAPICores/ModifyAPICores). Seams: NewAuthenticatedAPIClient (activity/trail/search-completion), dispatch/mode_local.go lookupResourceToken (dispatch), search_cmd.go resolveSearchToken (search). Server: entire.io api/src/app.ts + buildAPIDiscoveryDoc in api/src/lib/core-jwt.ts. search_cmd.go resolveSearchToken (search).

APIPath is the well-known path a data/web API (entire.io) serves to

advertise its trust roots, mirroring entire.io's api/src/app.ts route.

Unlike the cluster blob (core_urls only) it also carries the audience

the CLI must exchange its core token for, since a resource API

validates a fixed aud claim.

const APIPath = "/.well-known/entire-api.json"

// APIResponse is the parsed shape of /.well-known/entire-api.json. New // fields may be added by the server; unknown ones are ignored. type APIResponse struct { // Issuer is the API's home core (its preferred login server). Issuer string json:"issuer" // TrustedIssuers is every core whose JWTs the API accepts. Used the // same way cluster discovery uses core_urls: to pick the local // context whose CoreURL the API will honour. TrustedIssuers []string json:"trusted_issuers" // Audience is the aud the exchanged token must carry. The CLI // passes this verbatim as the RFC 8693 audience so the issued token // matches what the API validates — today the data host origin, but // advertised (not assumed) so the server can change it without a CLI // release. Audience string json:"audience" }

// ErrDiscoveryUnavailable wraps every "the API didn't give us a usable // the user must see. var ErrDiscoveryUnavailable = errors.New("api discovery unavailable")

// DiscoverAPI fetches and parses an API host's /.well-known/entire-api.json. // On success it returns a body with a non-empty TrustedIssuers and // Audience. Every failure mode (transport, non-200, decode, empty // required fields) is folded under ErrDiscoveryUnavailable so the caller // has a single sentinel to fall back on. func DiscoverAPI(ctx context.Context, apiHost string, c *http.Client, debugf DebugFunc) (*APIResponse, error) { var body APIResponse if err := fetchWellKnownJSON(ctx, apiHost, APIPath, c, &body, debugf); err != nil { return nil, fmt.Errorf("%w: %w", ErrDiscoveryUnavailable, err) } if len(body.TrustedIssuers) == 0 || body.Audience == "" { debugf("api discovery: incomplete document from https://%s%s (trusted_issuers=%d, audience=%q)", apiHost, APIPath, len(body.TrustedIssuers), body.Audience) debugf("api discovery: no trusted_issuers in response from https://%s%s", apiHost, APIPath) return nil, fmt.Errorf("%w: incomplete /.well-known/entire-api.json from %s", ErrDiscoveryUnavailable, apiHost) } return &body, nil }

// resolveAPIDoc returns apiHost's trust-root document, from api_discovery.json // when fresh, otherwise via a live /.well-known/entire-api.json fetch (which is // then cached). A stale-but-present cache entry is used as a fallback when the // live fetch fails, so a brief outage doesn't break a command whose trust roots // we already knew. Mirrors resolveClusterCores. Every cold failure stays folded // under ErrDiscoveryUnavailable (from DiscoverAPI) for the caller's fallback. func resolveAPIDoc(ctx context.Context, cacheDir, apiHost string, httpClient *http.Client, debugf DebugFunc) (*APIResponse, error) { cache, err := discovery.LoadAPIDiscovery(cacheDir) if err != nil { // A cache read problem must not block resolution — discover live. debugf("api-discovery cache load failed: %v; discovering live", err) cache = nil } var stale *APIResponse if cache != nil { if entry, fresh, ok := cache.Get(apiHost); ok { doc := &APIResponse{Issuer: entry.Issuer, TrustedIssuers: entry.TrustedIssuers, Audience: entry.Audience} if fresh { debugf("api host %s trust roots from cache", apiHost) return doc, nil } stale = doc debugf("api host %s trust-roots cache expired; re-fetching %s", apiHost, APIPath) } } doc, err := DiscoverAPI(ctx, apiHost, httpClient, debugf) if err != nil { if stale != nil { debugf("api discovery for %s failed (%v); falling back to stale cached trust roots", apiHost, err) return stale, nil } return nil, err } if mErr := discovery.ModifyAPIDiscovery(cacheDir, func(c discovery.APIDiscoveryCache) error { c.Set(apiHost, discovery.APIDiscoveryEntry{Issuer: doc.Issuer, TrustedIssuers: doc.TrustedIssuers, Audience: doc.Audience}) return nil }); mErr != nil { // Non-fatal: we resolved the doc, the next command just re-fetches. debugf("api-discovery cache write for %s failed: %v", apiHost, mErr) } return doc, nil }