feat(trail): add `entire trail tune` to tailor runner prompts to the repo · Entire
feat(trail): add entire trail tune to tailor runner prompts to the repo
72117b4·Soph·3w ago·7 files·+1,212 added/-0 removed
entire trail tune [<runner>] gathers signal about the current repo across four best-effort, gracefully-degrading tiers — repo docs/structure, merged PRs and issues (via gh), checkpoint churn hotspots, and past trail findings — and produces a prompt that rewrites the .entire/runners/*.json templates so their dimensions and score bands fit this repo instead of the generic defaults.
By default it prints the prompt for pasting into an agent. With --run it executes the prompt headlessly through the configured summary provider and surgically rewrites only each runner's prompt.template via byte-level replacement, leaving all other fields and formatting byte-for-byte intact (minimal git diff; files are git-tracked so the user reviews via git diff).
The CLI has no runner struct/loader (the backend consumes these files and substitutes {{placeholders}}), so runners are treated as opaque text.
Unit tests cover output parsing, surgical template replacement, runner loading/filtering, source-flag parsing, and prompt assembly.
Sessions
Changes
7
- cmd/entire/cli
65 unmodified lines
67
68
69
70
71
72
65 unmodified lines
cmd.AddCommand(newTrailDeleteCmd())
cmd.AddCommand(newTrailFindingCmd())
cmd.AddCommand(newTrailWatchCmd())
cmd.AddCommand(newTrailTuneCmd())
return cmd
}
package cli
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"regexp"
"sort"
"strings"
)
// parseTuneOutput extracts the runner-id -> new-template map the tuning model
// is instructed to emit as a single JSON object. The model may wrap the object
// in prose or code fences, so we slice from the first "{" to the last "}". An
// empty object ({}) is valid: the model is told to omit unchanged runners, so
// "{}" is the legitimate "no changes" result, not an error.
func parseTuneOutput(text string) (map[string]string, error) {
obj := extractJSONObject(text)
if obj == "" {
return nil, errors.New("no JSON object found in model output")
}
var m map[string]string
if err := json.Unmarshal([]byte(obj), &m); err != nil {
return nil, fmt.Errorf("parse model output as {runner: template}: %w", err)
}
return m, nil
}
var placeholderRe = regexp.MustCompile(`{{[^{}]+}}`)
// validateNewTemplate rejects a rewritten template that is empty or whose set
// of {{placeholder}} tokens differs from the original. The backend substitutes
// those placeholders at run time, so a dropped one silently breaks the runner
// and an invented one leaves unresolved template text in later runs — and the
// model is only *asked* to preserve them exactly, not forced to.
func validateNewTemplate(oldTemplate, newTemplate string) error {
if strings.TrimSpace(newTemplate) == "" {
return errors.New("rewritten template is empty")
}
oldSet := placeholderSet(oldTemplate)
newSet := placeholderSet(newTemplate)
var missing, added []string
for ph := range oldSet {
if !newSet[ph] {
missing = append(missing, ph)
}
}
for ph := range newSet {
if !oldSet[ph] {
added = append(added, ph)
}
}
sort.Strings(missing)
sort.Strings(added)
if len(missing) > 0 {
return fmt.Errorf("rewritten template dropped placeholder(s): %s", strings.Join(missing, ", "))
}
if len(added) > 0 {
return fmt.Errorf("rewritten template added unknown placeholder(s): %s", strings.Join(added, ", "))
}
return nil
}
func placeholderSet(s string) map[string]bool {
set := make(map[string]bool)
for _, ph := range placeholderRe.FindAllString(s, -1) {
set[ph] = true
}
return set
}
// extractJSONObject returns the outermost {...} span in text, after stripping
// any surrounding markdown code fences. Returns "" when none is found.
func extractJSONObject(text string) string {
text = stripCodeFences(strings.TrimSpace(text))
start := strings.Index(text, "{")
end := strings.LastIndex(text, "}")
if start < 0 || end <= start {
return ""
}
return text[start : end+1]
}
func stripCodeFences(text string) string {
if !strings.HasPrefix(text, "```") {
return text
}
// Drop the opening fence line (``` or ```json) and the closing fence.
if nl := strings.IndexByte(text, '\n'); nl >= 0 {
text = text[nl+1:]
}
if i := strings.LastIndex(text, "```); i >= 0 {
text = text[:i]
}
return strings.TrimSpace(text)
}
// replaceRunnerTemplate swaps only the prompt.template value inside a runner
// JSON document, leaving every other field and the file's formatting
// byte-for-byte intact. It works on the raw bytes (not a re-marshal) so unknown
// or backend-managed fields are never dropped and the git diff stays scoped to
// the prompt change. Returns the original bytes unchanged when newTemplate
// matches the current template.
func replaceRunnerTemplate(raw []byte, newTemplate string) ([]byte, error) {
var top map[string]json.RawMessage
if err := json.Unmarshal(raw, &top); err != nil {
return nil, fmt.Errorf("parse runner JSON: %w", err)
}
promptRaw, ok := top["prompt"]
if !ok {
return nil, errors.New("runner has no \"prompt\" object")
}
var promptObj map[string]json.RawMessage
if err := json.Unmarshal(promptRaw, &promptObj); err != nil {
return nil, fmt.Errorf("parse runner prompt object: %w", err)
}
// oldVal holds the original on-disk bytes of the template value, so it is a
// guaranteed substring of raw.
oldVal, ok := promptObj["template"]
if !ok {
return nil, errors.New("runner has no \"prompt.template\" field")
}
newVal, err := encodeJSONString(newTemplate)
if err != nil {
return nil, err
}
if bytes.Equal(oldVal, newVal) {
return raw, nil
}
if n := bytes.Count(raw, oldVal); n != 1 {
return nil, fmt.Errorf("expected exactly one occurrence of the current template, found %d", n)
}
out := bytes.Replace(raw, oldVal, newVal, 1)
if !json.Valid(out) {
return nil, errors.New("template replacement produced invalid JSON")
}
return out, nil
}
// encodeJSONString encodes s as a JSON string without HTML escaping, so
// characters like <, >, and & stay literal — matching the style the runner
// files are authored in and keeping diffs minimal.
func encodeJSONString(s string) ([]byte, error) {
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
enc.SetEscapeHTML(false)
if err := enc.Encode(s); err != nil {
return nil, fmt.Errorf("encode template string: %w", err)
}
return bytes.TrimRight(buf.Bytes(), "\n"), nil
}
package cli
import (
"encoding/json"
"strings"
"testing"
)
const sampleRunner = `{
"id": "trail-risk",
"display_name": "Risk Eval",
"enabled": true,
"runtime": {
"kind": "prompt_runner",
"model": "haiku"
},
"prompt": {
"template": "Old template with a \"quote\" and <angle> & ampersand — and an em-dash."
},
"output": {
"trail_monitor": {
"key": "risk",
"polarity": "lower_is_better"
}
}
}
`
func TestReplaceRunnerTemplate_SurgicalPreservesOtherFields(t *testing.T) {
t.Parallel()
const newTemplate = "Brand new template with <angle>, & ampersand, \"quotes\", and — em-dash."
out, err := replaceRunnerTemplate([]byte(sampleRunner), newTemplate)
if err != nil {
t.Fatalf("replaceRunnerTemplate: %v", err)
}
if !json.Valid(out) {
t.Fatalf("output is not valid JSON:\n%s", out)
}
// Every non-template field must survive byte-for-byte.
for _, want := range []string{
`"id": "trail-risk"`,
`"display_name": "Risk Eval"`,
`"model": "haiku"`,
`"polarity": "lower_is_better"`,
} {
if !strings.Contains(string(out), want) {
t.Errorf("expected output to preserve %q, got:\n%s", want, out)
}
}
// And the template must now be the new one (decoded), with special chars literal.
var doc struct {
Prompt struct {
Template string `json:"template"`
} `json:"prompt"`
}
if err := json.Unmarshal(out, &doc); err != nil {
t.Fatalf("unmarshal result: %v", err)
}
if doc.Prompt.Template != newTemplate {
t.Errorf("template = %q, want %q", doc.Prompt.Template, newTemplate)
}
// Literal <angle> present (rather than <angle>) proves
// SetEscapeHTML(false) kept special chars unescaped.
if !strings.Contains(string(out), `<angle>`) {
t.Errorf("expected literal <angle> (not HTML-escaped) in output:\n%s", out)
}
}
func TestReplaceRunnerTemplate_NoChangeReturnsIdentical(t *testing.T) {
t.Parallel()
const same = "Old template with a \"quote\" and <angle> & ampersand — and an em-dash."
out, err := replaceRunnerTemplate([]byte(sampleRunner), same)
if err != nil {
t.Fatalf("replaceRunnerTemplate: %v", err)
}
if string(out) != sampleRunner {
t.Errorf("expected identical bytes when template unchanged")
}
}
func TestReplaceRunnerTemplate_Errors(t *testing.T) {
t.Parallel()
if _, err := replaceRunnerTemplate([]byte(`{"prompt": {}}`), "x"); err == nil {
t.Error("expected error when prompt.template is missing")
}
if _, err := replaceRunnerTemplate([]byte(`{}`), "x"); err == nil {
t.Error("expected error when prompt object is missing")
}
if _, err := replaceRunnerTemplate([]byte(`not json`), "x"); err == nil {
t.Error("expected error on invalid JSON")
}
}
func TestValidateNewTemplate(t *testing.T) {
t.Parallel()
const old = "Analyze {{branch}} vs {{base_branch}}. Use {{previous_findings}}. Output JSON."
tests := []struct {
name string
newTemplate string
wantErr bool
}{
{name: "all placeholders preserved", newTemplate: "New text {{branch}} {{base_branch}} {{previous_findings}} done", wantErr: false},
{name: "empty", newTemplate: " ", wantErr: true},
{name: "dropped placeholder", newTemplate: "New text {{branch}} done", wantErr: true},
{name: "invented placeholder", newTemplate: "New {{branch}} {{base_branch}} {{previous_findings}} {{secrets}}", wantErr: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
err := validateNewTemplate(old, tc.newTemplate)
if tc.wantErr != (err != nil) {
t.Errorf("validateNewTemplate err=%v, wantErr=%v", err, tc.wantErr)
}
})
}
}
func TestParseTuneOutput(t *testing.T) {
t.Parallel()
tests := []struct {
name string
in string
want map[string]string
wantErr bool
}{
{
name: "plain object",
in: `{"trail-risk": "new risk", "trail-drift": "new drift"}`,
want: map[string]string{"trail-risk": "new risk", "trail-drift": "new drift"},
wantErr: false,
},
{
name: "fenced",
in: "```json\n{\"trail-risk\": \"new risk\"}\n```",
want: map[string]string{"trail-risk": "new risk"},
wantErr: false,
},
{
name: "prose wrapped",
in: "Here are the changes:\n{\"trail-risk\": \"new risk\"}\nDone.",
want: map[string]string{"trail-risk": "new risk"},
wantErr: false,
},
{name: "no json", in: "no object here", wantErr: true},
{name: "empty object is a valid no-op", in: "{}", want: map[string]string{}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got, err := parseTuneOutput(tc.in)
if tc.wantErr {
if err == nil {
t.Fatalf("expected error, got %v", got)
}
return
}
if err != nil {
t.Fatalf("parseTuneOutput: %v", err)
}
if len(got) != len(tc.want) {
t.Fatalf("got %v, want %v", got, tc.want)
}
for k, v := range tc.want {
if got[k] != v {
t.Errorf("key %q = %q, want %q", k, got[k], v)
}
}
})
}
}