Merge pull request #1751 from entireio/fix/redact-layer-vocab-and-pins

2c03352→main·

gtrrz-victor·2d ago·21 files·+179 added/-90 removed

fix(redact): reconcile layer-count vocabulary and pin provider-token boundaries

Changes

21

• cmd/entire/cli

• checkpoint

• Mcheckpoint_test.go+1/-1

• Mpersistent.go+16/-14

• Mpersistent_opf_trailer_test.go+3/-3

• Mprompts.go+3/-2

• Mprompts_test.go+3/-2

• settings

• Msettings.go+1/-1

• strategy

• Mcommon.go+1/-1

• Mmanual_commit_condensation.go+7/-5

• Mmanual_commit_hooks.go+4/-4

• Mmanual_commit_opf_prompt.go+2/-2

• Mmanual_commit_opf_rewrite.go+11/-11

• Mmanual_commit_push.go+4/-4

• trailers

• Mtrailers.go+2/-1

• Mtrailers_test.go+3/-2

• redact

• Mbatch.go+12/-11

• Mbatch_test.go+7/-7

• Mopf.go+4/-4

• Mopf_test.go+2/-2

• Mproviders.go+15/-6

• Mredact_test.go+76/-5

4694 unmodified lines

4695
4696
4697
4698
4698
4699
4700
4701

4694 unmodified lines

// Summary.Intent and ReviewPrompt that previously bypassed redaction because
// the dispatcher only matched .jsonl. The PR 1236 fix extended the JSON-aware
// branch to .json. We assert via a low-entropy AWS-key shaped secret (catches
// the 7-layer pipeline) so the test stays deterministic without the OPF binary.
// the regex-only pipeline) so the test stays deterministic without the OPF binary.
func TestRedactBlobBytes_JSONMetadata(t *testing.T) {
    t.Parallel()

Mcmd/entire/cli/checkpoint/checkpoint_test.go+1/-1

447 unmodified lines

448
449
450
451
451
452
453
454
260 unmodified lines

715
716
717
718
718
719
720
721
1532 unmodified lines

2254
2255
2256
2257
2258
2257
2258
2259
2260
2261
2261
2262
2262
2263
2264
2265
2266
15 unmodified lines

2282
2283
2284
2284
2285
2286
2287
2288
2289
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
35 unmodified lines

2330
2331
2332
2331
2332
2333
2334
2335
2336
2337

447 unmodified lines

}

// Replace prompts with 7-layer-redacted content.
// Replace prompts with regex-only-redacted content.
if len(opts.Prompts) > 0 {
    promptContent := RedactedJoinedPrompts(opts.Prompts)
    blobHash, err := CreateBlobFromContent(s.repo, []byte(promptContent))
260 unmodified lines
}
filePaths.AssetsManifest = manifestPath

// Write prompts via the 7-layer pipeline. OPF runs only in the
// Write prompts via the regex-only pipeline. OPF runs only in the
// pre-push rewrite path (manual_commit_opf_rewrite.go).
if len(opts.Prompts) > 0 {
    promptContent := RedactedJoinedPrompts(opts.Prompts)
1532 unmodified lines
}

return nil
}

// createRedactedBlobFromFile reads a file, applies the 7-layer redaction
// pipeline, and creates a git blob. Used by committed-checkpoint writes
// at post-commit time. The OpenAI Privacy Filter is intentionally NOT
// run here — OPF lives in the pre-push rewrite path
// (strategy/manual_commit_opf_rewrite.go), which re-redacts the 7-layer
// blobs into 8-layer commits before they leave the local machine.
// createRedactedBlobFromFile reads a file, applies the regex-only redaction
// pipeline (the eight always-on/opt-in layers), and creates a git blob.
// Used by committed-checkpoint writes at post-commit time. The OpenAI Privacy
// Filter is intentionally NOT run here — OPF lives in the pre-push rewrite
// path (strategy/manual_commit_opf_rewrite.go), which re-redacts the
// regex-only blobs into OPF-applied (9-layer) commits before they leave the
// local machine.
// JSONL files get JSONL-aware redaction; all other files get plain byte redaction.
func createRedactedBlobFromFile(ctx context.Context, repo *git.Repository, filePath, treePath string) (plumbing.Hash, filemode.FileMode, error) {
    info, err := os.Stat(filePath)
35 unmodified lines
// JSON-shaped files (.jsonl or .json) get JSON-aware redaction (falling
// back to plain bytes on parse failure so regex/credential layers
// still apply); other files get plain byte redaction. When
// usePrivacyFilter is true the full 8-layer pipeline (including OPF)
// runs; otherwise the 7-layer pipeline.
// usePrivacyFilter is true the full 9-layer pipeline (the eight regex
// layers plus OPF) runs; otherwise just the eight regex layers.

// .json is handled alongside .jsonl because checkpoint metadata files
// (metadata.json, per-session metadata.json) carry free-form fields

Mcmd/entire/cli/checkpoint/persistent.go+16/-14

17 unmodified lines

18
19
20
21
21
22
23
24
25
26
25
26
27
28
29

17 unmodified lines

// TestWriteCommitted_DoesNotEmitOPFAppliedTrailer is the regression guard
// for the architectural promise: standard post-commit condensation writes
// 7-layer-only blobs and MUST NOT mark them with the Entire-OPF-Applied
// regex-only blobs and MUST NOT mark them with the Entire-OPF-Applied
// trailer. The trailer is emitted exclusively by the pre-push rewrite
// path; if a future change accidentally added it to the standard writer,
// the pre-push rewrite would skip those commits (HasOPFApplied true →
// reparent-only, no actual OPF run) and ship 7-layer content as if it
// were 8-layer. This test pins down that contract.
// reparent-only, no actual OPF run) and ship regex-only content as if it
// were OPF-applied. This test pins down that contract.
func TestWriteCommitted_DoesNotEmitOPFAppliedTrailer(t *testing.T) {
    t.Parallel()

Mcmd/entire/cli/checkpoint/persistent_opf_trailer_test.go+3/-3

22 unmodified lines

23
24
25
26
27
26
27
28
29
30
31

22 unmodified lines

return prompts
}

// RedactedJoinedPrompts joins prompts and runs the 7-layer redaction
// pipeline. OPF runs exclusively in the pre-push rewrite (not here),
// RedactedJoinedPrompts joins prompts and runs them through the regex-only
// redaction pipeline (the eight always-on/opt-in layers). OPF runs exclusively in
// the pre-push rewrite (not here),
// so the writer's hot path stays predictable. Exported so alternate
// persistent backends produce identically-redacted prompt blobs.
func RedactedJoinedPrompts(prompts []string) string {