# data-api: fold discovery cache into the cluster cores cache

`6cc6266`→[main](/content/gh/entireio/cli/commits/main/index.html)·
toothbrush·1mo ago·9 files·+198 added/-351 removed

aud == base URI holds on both data-API environments (entire.io / partial.to),
and the token manager already defaults the RFC 8693 audience to the resource
origin it's dialing. So the CLI never needs the advertised `audience`: it
derives it from the host. That removes the one field distinguishing the API
discovery cache from the git cluster's cores cache.

Fold accordingly:
\- discovery: api_discovery.json reuses ClusterCoresCache via LoadAPICores/
ModifyAPICores (deleted the bespoke APIDiscoveryCache/Entry); cores cache now
serves both clusters and data APIs, two files.
\- clusterdiscovery: APIResponse slims to {trusted_issuers}; DiscoverAPI requires
only trusted_issuers; resolveAPITrustedIssuers mirrors resolveClusterCores;
ResolveContextForAPI returns just the context (no doc).
\- auth: NewRefreshingResourceProvider drops the audience param (token manager
defaults aud to the resource origin); ResolveDataAPIToken + seams updated.

Trades away "server changes audience without a CLI release" — fine, aud == base
URI is a hard requirement both envs. Doc updated.

## Sessions

75020e07c077View transcript

## Changes

9

- cmd/entire/cli/auth

- Mdata_api.go+7/-5

- Mdata_api_test.go+15/-17

- Mrefresh.go+11/-10

- docs/architecture

- Mupstream-host-resolution.md+33/-22

- internal/entireclient

- clusterdiscovery

- Mapi_discovery.go+54/-70

- Mapi_discovery_test.go+26/-38

- discovery

- Dapi_discovery.go-86

- Mapi_discovery_test.go+22/-98

- Mcluster_cores.go+30/-5

"""
20 unmodified lines

16 unmodified lines

44
45
46
47
48
47
48
49
50
51
36 unmodified lines

88
89
90
91
91
92
93
94
2 unmodified lines

97
98
99
100
101
102
101
103
104
105
106

20 unmodified lines

// resolveContextForAPIFunc is the shape of the discovery seam: it mirrors
// clusterdiscovery.ResolveContextForAPI (ctx, configDir, cacheDir, apiHost,
// httpClient, debugf).
type resolveContextForAPIFunc func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, *clusterdiscovery.APIResponse, error)
type resolveContextForAPIFunc func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, error)

// resolveContextForAPI is the discovery seam, swapped in tests so they don't
// reach the network. See SetResolveContextForAPIForTest for cross-package tests.

// DiscoveryUnavailableForTest is a ready-made SetResolveContextForAPIForTest
// value that forces the discovery-unavailable fallback (no network), so a
// cross-package test exercises the static TokenForResource path deterministically.
func DiscoveryUnavailableForTest(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, *clusterdiscovery.APIResponse, error) {
return nil, nil, clusterdiscovery.ErrDiscoveryUnavailable

func DiscoveryUnavailableForTest(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, error) {
return nil, clusterdiscovery.ErrDiscoveryUnavailable
}

// ResolveDataAPIToken returns a bearer for the data API at dataBaseURL.

36 unmodified lines

defer cancel()
httpClient := &http.Client{Timeout: dataAPIDiscoveryTimeout}

selected, doc, err := resolveContextForAPI(dctx, contexts.DefaultConfigDir(), discovery.DefaultCacheDir(), host, httpClient, nil)
selected, err := resolveContextForAPI(dctx, contexts.DefaultConfigDir(), discovery.DefaultCacheDir(), host, httpClient, nil)
if errors.Is(err, clusterdiscovery.ErrDiscoveryUnavailable) {
// Old deployment / not rolled out / transient — preserve today's behaviour.
return TokenForResource(ctx, dataOrigin)
2 unmodified lines

return "", err
}

// Exchange for the data host origin; the token manager derives the RFC 8693
// audience from it, which is the aud the API requires (aud == base URI).
allowInsecure := insecureHTTPEnabled() || isLoopbackHTTP(selected.CoreURL)
provider, err := NewRefreshingResourceProvider(selected, dataOrigin, doc.Audience, nil, allowInsecure)
provider, err := NewRefreshingResourceProvider(selected, dataOrigin, nil, allowInsecure)
if err != nil {
return "", err
}
"""

Mcmd/entire/cli/auth/data_api.go+7/-5

"""

41 unmodified lines

24 unmodified lines

74
75
76
77
78
77
78
79
80
81
3 unmodified lines

85
86
87
88
89
90
88
89
90
91
92
93
94
95
24 unmodified lines

120
121
122
121
122
123
124
125
126
123
124
125
126
127
8 unmodified lines

136
137
138
141
139
140
141
142
2 unmodified lines

145
146
147
150
148
149
150
153
151
152
153
154
7 unmodified lines

162
163
164
167
165
166
167
168

41 unmodified lines

restore := tokenstore.UseFileBackendForTesting(filepath.Join(t.TempDir(), "tokens.json"))
t.Cleanup(restore)

stubResolveContextForAPI(t, func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, *clusterdiscovery.APIResponse, error) {
return nil, nil, fmt.Errorf("%w: 404", clusterdiscovery.ErrDiscoveryUnavailable)
stubResolveContextForAPI(t, func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, error) {
return nil, fmt.Errorf("%w: 404", clusterdiscovery.ErrDiscoveryUnavailable)
})

// Pin the singleton manager to an empty store so the static fallback's
24 unmodified lines

t.Cleanup(restore)

sentinel := errors.New("multiple login contexts can authenticate against API host entire.io")
stubResolveContextForAPI(t, func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, *clusterdiscovery.APIResponse, error) {
return nil, nil, sentinel
stubResolveContextForAPI(t, func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, error) {
return nil, sentinel
})

_, err := ResolveDataAPIToken(context.Background(), "https://entire.io")
3 unmodified lines

// The success path: discovery picks a context, and the provider exchanges that
// context's login JWT at its core for the advertised audience, returning the
// exchanged token.
func TestResolveDataAPIToken_ExchangesForAdvertisedAudience(t *testing.T) {
// context's login JWT at its core for an audience equal to the data host
// origin (the aud the API requires), returning the exchanged token. The
audience is derived from the resource origin by the token manager, not read
// from discovery.
func TestResolveDataAPIToken_ExchangesForDataHostOrigin(t *testing.T) {
t.Setenv("ENTIRE_CONFIG_DIR", t.TempDir())
restore := tokenstore.UseFileBackendForTesting(filepath.Join(t.TempDir(), "tokens.json"))
t.Cleanup(restore)
24 unmodified lines

}
ctxObj := &contexts.Context{Name: "me@core", CoreURL: srv.URL, Handle: "me", KeychainService: svc};

stubResolveContextForAPI(t, func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, *clusterdiscovery.APIResponse, error) {
return ctxObj, &clusterdiscovery.APIResponse{
Issuer: srv.URL,
TrustedIssuers: []string{srv.URL},
Audience: wantAudience,
}, nil
stubResolveContextForAPI(t, func(context.Context, string, string, string, *http.Client, clusterdiscovery.DebugFunc) (*contexts.Context, error) {
return ctxObj, nil
})

// allowInsecure flows from the loopback http core (srv.URL) automatically.
8 unmodified lines

t.Fatalf("grant_type = %q, want token-exchange", gotGrant)
}
if gotAudience != wantAudience {
t.Fatalf("audience = %q, want the advertised audience %q", gotAudience, wantAudience)
t.Fatalf("audience = %q, want the data host origin %q (derived from the resource)", gotAudience, wantAudience)
}
if want := mustOrigin(t, dataOrigin); gotResource != want {
t.Fatalf("resource = %q, want the data origin %q", gotResource, want)
2 unmodified lines

func TestNewRefreshingResourceProvider_Validation(t *testing.T) {
t.Parallel()
if _, err := NewRefreshingResourceProvider(nil, "https://data.example", "aud", nil, false); err == nil {
if _, err := NewRefreshingResourceProvider(nil, "https://data.example", nil, false); err == nil {
t.Fatal("want error for nil context")
}
if _, err := NewRefreshingResourceProvider(&contexts.Context{Name: "x", CoreURL: "https://core.example"}, "https://data.example", "aud", nil, false); err == nil {
if _, err := NewRefreshingResourceProvider(&contexts.Context{Name: "x", CoreURL: "https://core.example"}, "https://data.example", nil, false); err == nil {
t.Fatal("want error for a context with no keychain slot")
}
}

7 unmodified lines

t.Cleanup(restore)

c := &contexts.Context{Name: "me@core", CoreURL: "https://core.example", Handle: "me", KeychainService: "kc:me"}
provider, err := NewRefreshingResourceProvider(c, "https://data.example", "https://data.example", nil, false)
provider, err := NewRefreshingResourceProvider(c, "https://data.example", nil, false)
if err != nil {
t.Fatalf("NewRefreshingResourceProvider: %v", err)
}
"""

Mcmd/entire/cli/auth/data_api_test.go+15/-17

"""

238 unmodified lines

238 unmodified lines

// NewRefreshingResourceProvider returns a provider that mints a bearer valid
// for resourceOrigin carrying the given audience, by exchanging context c's
// login JWT at c's own core (RFC 8693). It is NewRefreshingLoginProvider's
// sibling for resource servers: where that returns the bare login JWT (the
// control plane / cluster cases, where the host is the core), this performs
// the token exchange the data API requires.
// for resourceOrigin, by exchanging context c's login JWT at c's own core (RFC
// 8693). It is NewRefreshingLoginProvider's sibling for resource servers: where
// that returns the bare login JWT (the control plane / cluster cases, where the
// host is the core), this performs the token exchange the data API requires.
//
// Both the silent login-JWT re-mint and the exchange run through the shared
// per-context tokenmanager (newContextTokenManager). resourceOrigin must
// already be origin-only (no path); audience is passed verbatim as the RFC
// 8693 audience param. Exchanged tokens are cached in-process by the
// tokenmanager for the life of this process.
// already be origin-only (no path). No audience is passed: the token manager
// defaults the RFC 8693 audience to the resource origin, which is exactly what
// the data API requires (aud == its base URI), so the audience is derived from
// the host being dialed rather than read from discovery. Exchanged tokens are
// cached in-process by the tokenmanager for the life of this process.
//
// transport carries the caller's TLS configuration; allowInsecureHTTP permits
// an http:// core/resource for loopback/dev.
func NewRefreshingResourceProvider(c *contexts.Context, resourceOrigin, audience string, transport http.RoundTripper, allowInsecureHTTP bool) (func(context.Context) (string, error), error) {
func NewRefreshingResourceProvider(c *contexts.Context, resourceOrigin string, transport http.RoundTripper, allowInsecureHTTP bool) (func(context.Context) (string, error), error) {

mgr, err := newContextTokenManager(c, transport, allowInsecureHTTP)
if err != nil {
return nil, err
}
req := tokenmanager.TokenRequest{Resource: resourceOrigin, Audience: audience}
req := tokenmanager.TokenRequest{Resource: resourceOrigin}
return func(ctx context.Context) (string, error) {
tok, err := mgr.Token(ctx, req)
if mapped := contextReauthError(c, err); mapped != nil {
"""

Mcmd/entire/cli/auth/refresh.go+11/-10

"""

19 unmodified lines

20
21
22
23
23
24
25
26
61 unmodified lines

88
89
90
91
92
93
91
92
93
94
95
96
97
98
99
100
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
104
105
111
112
113
114
115
116
117
118
119
111
112
113
120
121
122
123
115
116
117
118
124
125
126
127
128
129
130
131
2 unmodified lines

134
135
136
127
137
138
139
140
141
130
131
142

19 unmodified lines

|---|---|---|---|
| **Core** — IdP **and** control-plane API, co-located | `entire-core` (`us.auth.entire.io`) | `org` / `repo` / `project` / `grant`, `auth *`, `login` | none needed — the host *is* the core |
| **Resource: git cluster** | `entire-server` / `entiredb` | `git-remote-entire` (clone/push) | `/.well-known/entire-cluster.json` → `core_urls` |
| **Resource: web/data API** | `entire.io` (`partial.to`) | `activity` / `search` / `trail` / `dispatch` | `/.well-known/entire-api.json` → `trusted_issuers` + `audience` |
| **Resource: web/data API** | `entire.io` (`partial.to`) | `activity` / `search` / `trail` / `dispatch` | `/.well-known/entire-api.json` → `trusted_issuers` (audience = the host origin) |

`contexts.json` (`$ENTIRE_CONFIG_DIR/contexts.json`, shared with entiredb's
CLIs) stores each login as `{Name, CoreURL, Handle, KeychainService}` plus a
61 unmodified lines

}

"""

The CLI reads only `trusted_issuers` and `audience`; `jwks_uris` is a
server-side verification concern (the CLI never fetches JWKS) and is ignored
on decode, so the server can evolve that field freely.
The CLI reads **only `trusted_issuers`** — exactly the way the git path reads a
cluster's `core_urls`. `issuer`, `audience`, and `jwks_uris` are advertised but
ignored on decode (see the audience note below).

> **Audience = the data host origin, not an opaque string.** entire.io's
> `ENTIRE_CORE_JWT_AUDIENCE` is `https://entire.io` (prod) / `https://partial.to`
> (staging). The core's STS exchange stamps `aud` = the requested audience for
> api-access exchanges, so the CLI requests exactly the advertised value. We
> advertise it (rather than assume host==audience) so the server can change it
> without a CLI release.
> **Audience = the data host origin.** entire.io's `ENTIRE_CORE_JWT_AUDIENCE` is
> `https://entire.io` (prod) / `https://partial.to` (staging) — the data host's
> own base URI, on both environments. The token manager already defaults the RFC
> 8693 audience to the resource origin it's exchanging for, so dialing
> `https://entire.io` produces `aud = https://entire.io` with no special
> handling. The CLI therefore **derives** the audience from the host it's already
dialing rather than reading the advertised `audience` field. (This trades away
the "server changes audience without a CLI release" flexibility — acceptable
> because `aud == base URI` is a hard requirement on both environments.)

Because the only field the CLI consumes is the trusted-issuer list — which *is*
a set of core URLs — the data-API discovery cache is literally the git cluster's
cores cache (`ClusterCoresCache`), in a separate file (`api_discovery.json`).

Resolution (`auth.ResolveDataAPIToken`):

1. Fetch the API host's `/.well-known/entire-api.json` (TLS-authenticated —
it's a trust root) and read `trusted_issuers` + `audience`.
1. Resolve the API host's trusted issuers: `api_discovery.json` when fresh, else
a live `/.well-known/entire-api.json` fetch (TLS-authenticated — it's a trust
root; redirects refused), cached with a 24h TTL and stale-fallback on a failed
re-fetch. Same `resolveClusterCores` shape the git path uses.
2. Pick the context with the **same cluster semantics** as the git path:
active-context-wins-if-eligible → sole eligible → explicit-choice error.
This is the lever that makes `ENTIRE_API_BASE_URL=https://partial.to entire
authenticate as the partial.to login even while the active context is a prod
entire.io login — without also setting `ENTIRE_AUTH_BASE_URL`.
3. Exchange that context's login JWT at **its** core for the advertised
audience (`auth.NewRefreshingResourceProvider`, keyed on `c.CoreURL` like the
control-plane provider, plus the RFC 8693 step the data API requires).
3. Exchange that context's login JWT at **its** core for the data host origin
(`auth.NewRefreshingResourceProvider`, keyed on `c.CoreURL` like the
control-plane provider; the token manager sets `aud` = that origin).
4. **Fallback**: if the host doesn't advertise discovery (404 / unreachable /
503 / malformed — a deployment predating the well-known), fall back to the
pre-discovery static path (`TokenForResource` via the singleton manager), so
behaviour is never worse than before. A *reachable* host whose context
selection fails surfaces that error — the user must log in or pick one.
503 / malformed) *and* no cache entry exists, fall back to the pre-discovery
static path (`TokenForResource` via the singleton manager), so behaviour is
never worse than before. A *reachable* host whose context selection fails
surfaces that error — the user must log in or pick one. (A transient outage
with a warm cache uses the stale entry, not the fallback.)

The selection rule differs from the control plane (where the active context
*always* wins because there's no host to match): here a host **is** matched, so

Key files: `cmd/entire/cli/auth/data_api.go` (`ResolveDataAPIToken` + fallback),
`cmd/entire/cli/auth/refresh.go` (`NewRefreshingResourceProvider`),
`internal/entireclient/clusterdiscovery/api_discovery.go` (`DiscoverAPI`,
`ResolveContextForAPI`, sharing `selectContext` with the cluster path). Seams:
`ResolveContextForAPI`, sharing `selectContext` *and* the cores cache with the
cluster path), `internal/entireclient/discovery/cluster_cores.go`
(`LoadAPICores`/`ModifyAPICores`). Seams:
`NewAuthenticatedAPIClient` (activity/trail/search-completion),
`dispatch/mode_local.go` `lookupResourceToken` (dispatch),
`search_cmd.go` `resolveSearchToken` (search). Server:
entire.io `api/src/app.ts` + `buildAPIDiscoveryDoc` in `api/src/lib/core-jwt.ts`.
`search_cmd.go` `resolveSearchToken` (search).

# APIPath is the well-known path a data/web API (entire.io) serves to
# advertise its trust roots, mirroring entire.io's api/src/app.ts route.
# Unlike the cluster blob (core_urls only) it also carries the audience
# the CLI must exchange its core token for, since a resource API
# validates a fixed `aud` claim.
const APIPath = "/.well-known/entire-api.json"

// APIResponse is the parsed shape of /.well-known/entire-api.json. New
// fields may be added by the server; unknown ones are ignored.
type APIResponse struct {
  // Issuer is the API's home core (its preferred login server).
  Issuer string `json:"issuer"`
  // TrustedIssuers is every core whose JWTs the API accepts. Used the
  // same way cluster discovery uses core_urls: to pick the local
  // context whose CoreURL the API will honour.
  TrustedIssuers []string `json:"trusted_issuers"`
  // Audience is the `aud` the exchanged token must carry. The CLI
  // passes this verbatim as the RFC 8693 audience so the issued token
  // matches what the API validates — today the data host origin, but
  // advertised (not assumed) so the server can change it without a CLI
  // release.
  Audience string `json:"audience"`
}

// ErrDiscoveryUnavailable wraps every "the API didn't give us a usable
// the user must see.
var ErrDiscoveryUnavailable = errors.New("api discovery unavailable")

// DiscoverAPI fetches and parses an API host's /.well-known/entire-api.json.
// On success it returns a body with a non-empty TrustedIssuers and
// Audience. Every failure mode (transport, non-200, decode, empty
// required fields) is folded under ErrDiscoveryUnavailable so the caller
// has a single sentinel to fall back on.
func DiscoverAPI(ctx context.Context, apiHost string, c *http.Client, debugf DebugFunc) (*APIResponse, error) {
  var body APIResponse
  if err := fetchWellKnownJSON(ctx, apiHost, APIPath, c, &body, debugf); err != nil {
    return nil, fmt.Errorf("%w: %w", ErrDiscoveryUnavailable, err)
}
  if len(body.TrustedIssuers) == 0 || body.Audience == "" {
    debugf("api discovery: incomplete document from https://%s%s (trusted_issuers=%d, audience=%q)",
      apiHost, APIPath, len(body.TrustedIssuers), body.Audience)
    debugf("api discovery: no trusted_issuers in response from https://%s%s", apiHost, APIPath)
    return nil, fmt.Errorf("%w: incomplete /.well-known/entire-api.json from %s", ErrDiscoveryUnavailable, apiHost)
  }
  return &body, nil
}

// resolveAPIDoc returns apiHost's trust-root document, from api_discovery.json
// when fresh, otherwise via a live /.well-known/entire-api.json fetch (which is
// then cached). A stale-but-present cache entry is used as a fallback when the
// live fetch fails, so a brief outage doesn't break a command whose trust roots
// we already knew. Mirrors resolveClusterCores. Every cold failure stays folded
// under ErrDiscoveryUnavailable (from DiscoverAPI) for the caller's fallback.
func resolveAPIDoc(ctx context.Context, cacheDir, apiHost string, httpClient *http.Client, debugf DebugFunc) (*APIResponse, error) {
  cache, err := discovery.LoadAPIDiscovery(cacheDir)
  if err != nil {
    // A cache read problem must not block resolution — discover live.
    debugf("api-discovery cache load failed: %v; discovering live", err)
    cache = nil
  }
  var stale *APIResponse
  if cache != nil {
    if entry, fresh, ok := cache.Get(apiHost); ok {
      doc := &APIResponse{Issuer: entry.Issuer, TrustedIssuers: entry.TrustedIssuers, Audience: entry.Audience}
      if fresh {
        debugf("api host %s trust roots from cache", apiHost)
        return doc, nil
      }
      stale = doc
      debugf("api host %s trust-roots cache expired; re-fetching %s", apiHost, APIPath)
    }
  }
  doc, err := DiscoverAPI(ctx, apiHost, httpClient, debugf)
  if err != nil {
    if stale != nil {
      debugf("api discovery for %s failed (%v); falling back to stale cached trust roots", apiHost, err)
      return stale, nil
    }
    return nil, err
  }
  if mErr := discovery.ModifyAPIDiscovery(cacheDir, func(c discovery.APIDiscoveryCache) error {
    c.Set(apiHost, discovery.APIDiscoveryEntry{Issuer: doc.Issuer, TrustedIssuers: doc.TrustedIssuers, Audience: doc.Audience})
    return nil
  }); mErr != nil {
    // Non-fatal: we resolved the doc, the next command just re-fetches.
    debugf("api-discovery cache write for %s failed: %v", apiHost, mErr)
  }
  return doc, nil
}