Add a judge panel to inspect (engine + scripted config) · Entire
Add a judge panel to inspect (engine + scripted config)
2595b9e→main·
dipree·1mo ago·6 files·+401 added/-85 removed
Judges replace the single master: each judge independently evaluates the inspectors' reports, and when there are >=2 a chair merges their verdicts into one final verdict with the panel's agreement/dissent surfaced.
- settings: ReviewProfileConfig.Judges []ReviewConfig + Chair (legacy Master/MasterAgent/MasterModel kept for back-compat).
- profile: profileJudges resolves the panel + chair index; profileMaster-Identity now reports the chair/single judge on top of it.
- synthesis: PanelSynthesisProvider fans out to judges in parallel, the chair merges, failures are dropped (tested with stub providers).
- run path: runMultiAgentPath builds the panel provider.
- scripted: --set-judge (repeatable) + --set-chair replace --set-master.
Sessions
d672b0f57f7eView transcript
[?
List me all the potential command combinations for review.Pi·Opus 4.8·12 steps](/content/gh/entireio/cli/session/019e9fb5-004d-7145-8f48-db6a240d0ab3#timeline-d672b0f57f7e/index.html)
Changes
6
cmd/entire/cli
review
Mcmd.go+70/-41
Mconfigure_test.go+39/-4
Mprofile.go+64/-33
Asynthesis_panel.go+121
Asynthesis_panel_test.go+93
settings
Msettings.go+14/-7
78 unmodified lines
79
80
81
82
82
83
84
85
86
25 unmodified lines
112
113
114
114
115
116
117
118
119
63 unmodified lines
183
184
185
184
186
187
188
189
190
11 unmodified lines
202
203
204
202
205
206
207
208
209
12 unmodified lines
222
223
224
221
222
225
226
227
228
224
225
229
230
231
232
233
229
234
235
236
237
283 unmodified lines
521
522
523
519
524
525
526
527
82 unmodified lines
610
611
612
608
609
610
611
612
613
614
613
614
615
616
617
618
619
620
621
622
623
624
616
617
618
625
626
620
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
167 unmodified lines
812
813
814
794
795
815
816
817
818
819
820
797
798
799
800
801
802
803
804
805
806
807
808
809
810
821
822
823
824
302 unmodified lines
1127
1128
1129
1119
1120
1121
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1 unmodified line
1155
1156
1157
1129
1158
1159
1160
1161
78 unmodified lines
var listAgents bool
var listProfiles bool
var setAgents []string
var setMaster string
var setJudges []string
var setChair string
var setTask string
var setModels []string
var setSlots []string
25 unmodified lines
With --set-* flags it writes the profile non-interactively; otherwise it opens the wizard (interactive) without starting agents.
--set-agents with --configure: comma-separated worker agents for the profile
--set-master with --configure: master agent that writes the final report
--set-judge with --configure: a judge as agent[=model] (repeatable; >1 = panel)
--set-chair with --configure: the judge that merges a multi-judge panel
--set-task with --configure: the profile's canonical task text
--set-model with --configure: per-worker model as agent=model (repeatable)
--set-slot with --configure: a worker slot as agent[=model] (repeatable; same agent/model may repeat)
63 unmodified lines
if configure {
return runReviewConfigure(ctx, cmd, profileName, reviewConfigureOptions{
Agents: setAgents,
Master: setMaster,
Judges: setJudges,
Chair: setChair,
Task: setTask,
Models: setModels,
Slots: setSlots,
}
}
cmd.Flags().BoolVar(&configure, "configure", false, "set up a review profile; shows available agents and accepts --set-* flags for non-interactive config")
cmd.Flags().StringSliceVar(&setAgents, "set-agents", nil, "with --configure: worker agents for the profile (comma-separated)")
cmd.Flags().StringVar(&setMaster, "set-master", "", "with --configure: master agent that writes the final report")
cmd.Flags().StringArrayVar(&setJudges, "set-judge", nil, "with --configure: a judge as agent[=model] (repeatable; multiple judges form a panel)")
cmd.Flags().StringVar(&setChair, "set-chair", "", "with --configure: the judge (agent[=model]) that merges a multi-judge panel")
cmd.Flags().StringVar(&setTask, "set-task", "", "with --configure: the profile's canonical task text")
cmd.Flags().StringArrayVar(&setModels, "set-model", nil, "with --configure: per-worker model as agent=model (repeatable)")
cmd.Flags().StringArrayVar(&setSlots, "set-slot", nil, "with --configure: a worker slot as agent[=model] (repeatable; same agent/model may repeat)")
// reviewConfigureOptions carries the non-interactive `--configure` inputs.
type reviewConfigureOptions struct {
Agents []string // worker agent names (--set-agents)
Master string // master agent (--set-master)
Agents []string // inspector agent names (--set-agents)
Judges []string // judge slots as "agent[=model]" entries (--set-judge)
Chair string // chair judge as agent[=model] (--set-chair)
Task string // profile task text (--set-task)
Models []string // per-worker "agent=model" entries (--set-model)
Slots []string // worker slots as "agent[=model]" entries (--set-slot)
Models []string // per-inspector "agent=model" entries (--set-model)
Slots []string // inspector slots as "agent[=model]" entries (--set-slot)
}
func (o reviewConfigureOptions) scripted() bool {
return len(o.Agents) > 0 || o.Master != "" || o.Task != "" || len(o.Models) > 0 || len(o.Slots) > 0
}
return len(o.Agents) > 0 || len(o.Judges) > 0 || o.Chair != "" || o.Task != "" || len(o.Models) > 0 || len(o.Slots) > 0
}
func runReviewConfigure(ctx context.Context, cmd *cobra.Command, profileOverride string, opts reviewConfigureOptions, deps Deps) error {
283 unmodified lines
fmt.Fprintln(out)
fmt.Fprintf(out, "Configure %q non-interactively, e.g.:\n", profileName)
fmt.Fprintf(out, " entire inspect --configure --profile %s --set-agents %s --set-master <agent>\n",
fmt.Fprintf(out, " entire inspect --configure --profile %s --set-agents %s --set-judge <agent>\n",
profileName, exampleAgentList(catalog))
}
82 unmodified lines
profile.Task = profileTask(profileName, settings.ReviewProfileConfig{})
if opts.Master != "" {
profile.Master = opts.Master
}
if len(nonZeroAgentConfigs(profile.Agents)) > 1 {
if strings.TrimSpace(profile.MasterAgent) == "" {
if strings.TrimSpace(profile.Master) == "" {
profile.Master = defaultReviewMaster(ctx, profile.Agents)
// Judges: explicit --set-judge defines the panel; otherwise fall back to the
// legacy single-master default picked from the inspectors.
inspectorCount := len(nonZeroAgentConfigs(profile.Agents))
switch {
case len(opts.Judges) > 0:
judges := make([]settings.ReviewConfig, 0, len(opts.Judges))
for _, raw := range opts.Judges {
rawName, model, _ := strings.Cut(raw, "=")
name := strings.TrimSpace(rawName)
if name == "" {
continue
}
if _, _, masterErr := selectProfileWorker(profile, profile.Master); masterErr != nil {
return settings.ReviewProfileConfig{}, fmt.Errorf("master %q is not one of the profile workers (%s)", profile.Master, strings.Join(sortedProfileAgentNames(profile), ", "))
}
judges = append(judges, settings.ReviewConfig{Agent: name, Model: strings.TrimSpace(model)})
}
} else {
if len(judges) == 0 {
return settings.ReviewProfileConfig{}, errors.New("--set-judge listed no usable judges")
}
profile.Judges = judges
profile.Chair = strings.TrimSpace(opts.Chair)
profile.Master = ""
profile.MasterAgent = ""
case inspectorCount > 1 && len(profile.Judges) == 0 && strings.TrimSpace(profile.MasterAgent) == "":
if strings.TrimSpace(profile.Master) == "" {
profile.Master = defaultReviewMaster(ctx, profile.Agents)
}
if _, _, masterErr := selectProfileWorker(profile, profile.Master); masterErr != nil {
return settings.ReviewProfileConfig{}, fmt.Errorf("judge %q is not one of the profile inspectors (%s)", profile.Master, strings.Join(sortedProfileAgentNames(profile), ", "))
}
case inspectorCount <= 1:
profile.Master = ""
}
return profile, nil
// More unmodified lines omitted for brevity...
package review
import (
"context"
"errors"
"fmt"
"strings"
"sync"
)
// PanelSynthesisProvider runs a panel of judges over the inspectors' reports. // Each judge independently produces a verdict from the same synthesis prompt; // when two or more verdicts come back, the chair judge merges them into one // final verdict and the individual verdicts are appended as a panel. // It implements SynthesisProvider, so the existing SynthesisSink uses it // unchanged — a panel is just a provider that happens to consult several // judges. A single judge collapses to that judge's verdict (today's behavior).
type PanelSynthesisProvider struct {
Judges []SynthesisProvider // one per judge; index aligns with Labels
Labels []string // display label per judge (e.g. "codex · gpt-5")
ChairIdx int // index of the judge that merges the panel
}
// Synthesize fans out to each judge in parallel, then has the chair merge the // verdicts. Failed judges are dropped; if only one verdict survives it is // returned directly. If every judge fails, the error is returned so the caller // can report "final report unavailable".
func (p PanelSynthesisProvider) Synthesize(ctx context.Context, prompt string) (string, error) {
switch len(p.Judges) {
case 0:
return "", errors.New("no judges configured")
case 1:
return p.Judges[0].Synthesize(ctx, prompt) //nolint:wrapcheck // transparent single-judge passthrough
}
verdicts := make([]string, len(p.Judges))
errs := make([]error, len(p.Judges))
var wg sync.WaitGroup
for i := range p.Judges {
wg.Add(1)
go func(i int) {
defer wg.Done()
verdicts[i], errs[i] = p.Judges[i].Synthesize(ctx, prompt)
}(i)
}
wg.Wait()
ok := make([]int, 0, len(p.Judges))
for i := range verdicts {
if errs[i] == nil && strings.TrimSpace(verdicts[i]) != "" {
ok = append(ok, i)
}
}
switch len(ok) {
case 0:
return "", fmt.Errorf("all judges failed: %w", firstNonNilErr(errs))
case 1:
return verdicts[ok[0]], nil
}
// Pick the chair; fall back to the first successful judge if the configured // chair failed or is out of range.
chair := p.ChairIdx
if chair < 0 || chair >= len(p.Judges) || errs[chair] != nil || strings.TrimSpace(verdicts[chair]) == "" {
chair = ok[0]
}
final, err := p.Judges[chair].Synthesize(ctx, composeChairPrompt(verdicts, p.Labels, ok))
if err != nil || strings.TrimSpace(final) == "" {
// Chair merge failed: surface the panel rather than nothing.
final = "The judges could not be merged into a single verdict; see each judge's verdict below."
}
var b strings.Builder
b.WriteString(strings.TrimSpace(final))
b.WriteString("\n\n## Panel\n\n")
for _, i := range ok {
fmt.Fprintf(&b, "### %s\n\n%s\n\n", p.labelAt(i), strings.TrimSpace(verdicts[i]))
}
return b.String(), nil
}