diff --git a/README.details.md b/README.details.md index 1dfea50..300dc53 100644 --- a/README.details.md +++ b/README.details.md @@ -325,6 +325,25 @@ The role routing applies `high` to Scout/Review and `medium` to Implement: } ``` +An optional `efforts` object overrides the per-role reasoning defaults — +Scout `high`, Implement `medium`, Review `high` — with `medium`, `high`, or +`xhigh`: + +```json +"efforts": { + "implement": "xhigh", + "scout": "medium" +} +``` + +Omitted roles keep the defaults, and existing attempts keep their recorded +effort on restart. A configured effort the routed models do not support +falls back to the role default with a diagnostic instead of failing the run; +when only a stronger fallback model supports it, routing advances along the +normal profile chain (for example Implement moves from Terra to Sol for +`xhigh`). Raise effort for hard implementation work and lower it for cheap +scouting when your provider and workload justify it. + For advanced Claude or GLM setup, configure the bundled Pi CLI under the daemon account with `bun x --no-install pi`. Use its `/login` and `/model` commands where supported and save the default. Provider keys must be in the daemon environment. diff --git a/README.md b/README.md index 1583119..1e6b19c 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,11 @@ acceptance evidence, and review PRs in GitHub. The board reads shared checkpoint it does not launch another worker or grant approval. New Codex setups select GPT-6 Astra; existing model settings are preserved. -New Scout/Review attempts use `high` reasoning; Implement uses `medium`. +New Scout/Review attempts use `high` reasoning; Implement uses `medium`. An +optional `efforts` object in `~/.config/roc/settings.json` overrides these +per-role defaults with `medium`, `high`, or `xhigh` (for example +`"efforts": { "implement": "xhigh" }`); an effort the routed models do not +support falls back to the role default with a diagnostic. Roc uses Pi's tools and agent loop. It does not launch Codex CLI or Claude Code. Pi automatically summarizes older context as a session approaches its context diff --git a/src/agents/pi/backend.ts b/src/agents/pi/backend.ts index f9cac74..7f6ed25 100644 --- a/src/agents/pi/backend.ts +++ b/src/agents/pi/backend.ts @@ -1,5 +1,9 @@ import { AgileError } from "../../runtime/errors"; -import type { CatalogModel, ModelMapping } from "../../scheduler/model-routing"; +import type { + CatalogModel, + ModelMapping, + RoleEfforts, +} from "../../scheduler/model-routing"; import { loadRocSettings } from "../../settings"; import { buildDefaultSkillConfig, @@ -88,6 +92,7 @@ export const startPiBackend: BackendFactory = async (context) => { skillPaths, allowUnsandboxed, models: settings.models, + efforts: settings.efforts, })(context); }; @@ -101,6 +106,7 @@ export function buildPiBackendFactory(input: { skillPaths?: readonly string[]; allowUnsandboxed?: boolean; models?: ModelMapping; + efforts?: RoleEfforts; startAttemptClient?: (cwd: string) => Promise; }): BackendFactory { return async ({ branches }: { branches: TaskBranchManager }) => { @@ -236,6 +242,7 @@ export function buildPiBackendFactory(input: { return { catalog, modelMapping, + efforts: input.efforts, harness: createPiHarness({ branches, startClient: startAttemptClient }), close: () => { closed ??= (async () => { diff --git a/src/agents/types.ts b/src/agents/types.ts index 4799a70..09b8149 100644 --- a/src/agents/types.ts +++ b/src/agents/types.ts @@ -1,5 +1,9 @@ import type { AgentHarness } from "../harness/contracts"; -import type { CatalogModel, ModelMapping } from "../scheduler/model-routing"; +import type { + CatalogModel, + ModelMapping, + RoleEfforts, +} from "../scheduler/model-routing"; import type { TaskBranchManager } from "../workspace/task-branch"; /** @@ -16,6 +20,8 @@ export type BackendRuntime = { * catalog instead of inferring profiles from provider model names. */ readonly modelMapping?: ModelMapping; + /** Per-role reasoning effort overrides from Roc settings; unset roles keep defaults. */ + readonly efforts?: RoleEfforts; readonly harness: AgentHarness; /** Releases owned resources idempotently, rejecting if child exit or cleanup cannot be confirmed. */ close(): Promise; diff --git a/src/cli/commands/onboard.ts b/src/cli/commands/onboard.ts index 883fd2d..de74b31 100644 --- a/src/cli/commands/onboard.ts +++ b/src/cli/commands/onboard.ts @@ -137,6 +137,9 @@ async function executeOnboard( ...(priorSettings?.models === undefined ? {} : { models: priorSettings.models }), + ...(priorSettings?.efforts === undefined + ? {} + : { efforts: priorSettings.efforts }), }, homeRoot, ); diff --git a/src/cli/runtime.ts b/src/cli/runtime.ts index 76fc7de..9b592ee 100644 --- a/src/cli/runtime.ts +++ b/src/cli/runtime.ts @@ -161,20 +161,25 @@ export async function runBackendSession( retain = false; stop.throwIfAborted(); const lastProgress = new Map(); + const emitDiagnostic = (message: string) => + process.stderr.write(`${message}\n`); const runner = new GitHubTaskPool({ concurrency: input.concurrency, autoMerge: input.autoMerge, store, branches, harness: backend.harness, - advisor: createModelAdvisor(backend.catalog, backend.modelMapping), + advisor: createModelAdvisor(backend.catalog, backend.modelMapping, { + efforts: backend.efforts, + onDiagnostic: emitDiagnostic, + }), publisher: options.publisherFactory?.(branches) ?? new GitHubPullRequestPublisher(baseBranch, branches, command), command, cwd: input.repoPath, baseBranch, - diagnostic: (message) => process.stderr.write(`${message}\n`), + diagnostic: emitDiagnostic, /** Emits confirmed phase changes and wait reasons once while tool activity stays local. */ progress(record) { const summary = `Phase: ${record.phase}${record.failure ? ` · ${record.failure}` : ""}`; diff --git a/src/domain/agile-cycle.ts b/src/domain/agile-cycle.ts index 05ab2a2..dd75d23 100644 --- a/src/domain/agile-cycle.ts +++ b/src/domain/agile-cycle.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { AgentRoleSchema, ReasoningEffortSchema } from "../harness/contracts"; import { ModelProfileSchema } from "./schemas"; import { SkillSettingsSchema } from "./skill-allowlist"; @@ -45,6 +46,7 @@ export const RocSettingsSchema = z models: z .partialRecord(ModelProfileSchema, z.string().regex(/^[^\s/]+\/[^\s]+$/u)) .optional(), + efforts: z.partialRecord(AgentRoleSchema, ReasoningEffortSchema).optional(), }) .strict(); diff --git a/src/scheduler/model-routing.ts b/src/scheduler/model-routing.ts index b614d82..e3d0f52 100644 --- a/src/scheduler/model-routing.ts +++ b/src/scheduler/model-routing.ts @@ -2,13 +2,18 @@ import type { z } from "zod"; import type { ModelProfileSchema } from "../domain/schemas"; export type ModelProfile = z.infer; +export type AgentRole = "scout" | "implement" | "review"; +/** Every Roc effort is also a Pi thinking level; no other levels exist. */ +export type ReasoningEffort = "medium" | "high" | "xhigh"; export type CatalogModel = Readonly<{ id: string; supportedReasoningEfforts: readonly string[]; }>; export type ModelMapping = Readonly>>; +/** Per-role configured efforts from Roc settings; unset roles keep defaults. */ +export type RoleEfforts = Readonly>>; export type AdvisorInput = { - role: "scout" | "implement" | "review"; + role: AgentRole; risk: "low" | "medium" | "high"; retryIndex: 0 | 1 | 2; priorProfile?: ModelProfile; @@ -17,11 +22,16 @@ export type AdvisorInput = { export type Route = { profile: ModelProfile; model: string; - effort: "medium" | "high"; + effort: ReasoningEffort; fallbacks: string[]; rationale: string[]; }; export type ModelAdvisor = { decide(input: AdvisorInput): Route | undefined }; +export type AdvisorOptions = Readonly<{ + efforts?: RoleEfforts; + /** Receives one bounded message per role when its configured effort is unsupported. */ + onDiagnostic?: (message: string) => void; +}>; const profileOrder: ModelProfile[] = ["luna", "terra", "sol"]; @@ -76,16 +86,44 @@ function routeRationale( ]; } +/** Explains how the effective effort relates to the configured one. */ +function effortRationale( + configured: ReasoningEffort | undefined, + applied: ReasoningEffort, +): string[] { + if (configured === undefined) return []; + if (configured === applied) return [`effort ${applied} (configured)`]; + return [ + `configured effort ${configured} unsupported`, + `effort ${applied} (default)`, + ]; +} + /** Creates a model advisor from a stable catalog snapshot and optional mappings. */ export function createModelAdvisor( catalog: readonly CatalogModel[], mapping: ModelMapping = {}, + options: AdvisorOptions = {}, ): ModelAdvisor { const catalogSnapshot = catalog.map((model) => ({ id: model.id, supportedReasoningEfforts: [...model.supportedReasoningEfforts], })); const mappingSnapshot: ModelMapping = { ...mapping }; + const effortsSnapshot: RoleEfforts = { ...(options.efforts ?? {}) }; + const reportedRoles = new Set(); + /** Reports an unsupported configured effort once per role, never failing the run. */ + const reportUnsupported = ( + role: AgentRole, + configured: ReasoningEffort, + fallback: ReasoningEffort, + ): void => { + if (reportedRoles.has(role)) return; + reportedRoles.add(role); + options.onDiagnostic?.( + `Configured ${role} effort "${configured}" is unsupported by the routed models; using the default "${fallback}" instead.`, + ); + }; /** Finds the configured or inferred catalog model supporting a profile and effort. */ const modelForProfile = ( profile: ModelProfile, @@ -109,22 +147,37 @@ export function createModelAdvisor( return { /** Chooses the first compatible routed model and records its fallbacks and rationale. */ decide(input) { - const effort: Route["effort"] = + const defaultEffort: ReasoningEffort = input.role === "implement" ? "medium" : "high"; - const choices = routeProfiles(input).flatMap((profile) => { - const model = modelForProfile(profile, effort); - return model === undefined ? [] : [{ profile, model }]; - }); - const choice = choices[0]; - if (choice === undefined) return undefined; + const configured = effortsSnapshot[input.role]; + const candidates: ReasoningEffort[] = + configured !== undefined && configured !== defaultEffort + ? [configured, defaultEffort] + : [defaultEffort]; + for (const effort of candidates) { + const choices = routeProfiles(input).flatMap((profile) => { + const model = modelForProfile(profile, effort); + return model === undefined ? [] : [{ profile, model }]; + }); + const choice = choices[0]; + if (choice === undefined) { + if (effort !== defaultEffort) + reportUnsupported(input.role, effort, defaultEffort); + continue; + } - return { - profile: choice.profile, - model: choice.model, - effort, - fallbacks: choices.slice(1).map((fallback) => fallback.model), - rationale: routeRationale(input, choice.profile), - }; + return { + profile: choice.profile, + model: choice.model, + effort, + fallbacks: choices.slice(1).map((fallback) => fallback.model), + rationale: [ + ...routeRationale(input, choice.profile), + ...effortRationale(configured, effort), + ], + }; + } + return undefined; }, }; } diff --git a/src/settings.ts b/src/settings.ts index 3a709c5..74a9417 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -32,6 +32,10 @@ const publicFieldNames = new Set([ "luna", "terra", "sol", + "efforts", + "scout", + "implement", + "review", ]); const repairGuidance = "Back up this file, then repair it manually using the supported settings format in README.details.md (Progress and recovery) and retry; onboarding reads the same file and cannot repair it."; diff --git a/test/agents/pi/backend.test.ts b/test/agents/pi/backend.test.ts index bab2255..d344d5b 100644 --- a/test/agents/pi/backend.test.ts +++ b/test/agents/pi/backend.test.ts @@ -223,6 +223,24 @@ test("saved execution consent permits the probe without an environment override" } }); +test("surfaces configured per-role efforts on the returned runtime", async () => { + const probe = new ScriptedProbeClient(probeModels, probeDefaultModel); + const runtime = await buildPiBackendFactory({ + allowUnsandboxed: true, + efforts: { implement: "xhigh", scout: "medium" }, + startProbeClient: async () => probe, + })({ branches: memoryBranches() }); + try { + expect(runtime.efforts).toEqual({ + implement: "xhigh", + scout: "medium", + }); + } finally { + await runtime.close(); + } + expect(probe.closeCount).toBe(1); +}); + test("the factory fails closed and closes the probe when no default model resolves", async () => { const previous = process.env.ROC_ALLOW_UNSANDBOXED; const probe = new ScriptedProbeClient(probeModels, null); diff --git a/test/scheduler/model-routing.test.ts b/test/scheduler/model-routing.test.ts index 7a61fbb..707eead 100644 --- a/test/scheduler/model-routing.test.ts +++ b/test/scheduler/model-routing.test.ts @@ -206,3 +206,136 @@ test("keeps an immutable snapshot of catalog and mapping inputs", () => { model: "model-a", }); }); + +test("applies configured per-role efforts and leaves unset roles unchanged", () => { + const advisor = createModelAdvisor(catalog, undefined, { + efforts: { implement: "high", scout: "medium" }, + }); + + expect( + advisor.decide({ role: "implement", risk: "medium", retryIndex: 0 }), + ).toMatchObject({ + profile: "terra", + model: "gpt-5.6-terra", + effort: "high", + rationale: [ + "implement baseline", + "medium risk", + "effort high (configured)", + ], + }); + expect( + advisor.decide({ role: "scout", risk: "medium", retryIndex: 0 }), + ).toMatchObject({ + profile: "luna", + model: "gpt-5.6-luna", + effort: "medium", + }); + expect( + advisor.decide({ role: "review", risk: "medium", retryIndex: 0 }), + ).toMatchObject({ + profile: "sol", + model: "gpt-5.6-sol", + effort: "high", + rationale: ["review baseline", "medium risk"], + }); +}); + +test("routes a configured xhigh effort and advances the chain for support", () => { + const advisor = createModelAdvisor(catalog, undefined, { + efforts: { implement: "xhigh" }, + }); + expect( + advisor.decide({ role: "implement", risk: "medium", retryIndex: 0 }), + ).toMatchObject({ + profile: "terra", + model: "gpt-5.6-terra", + effort: "xhigh", + }); + + const terraWithoutXhigh = createModelAdvisor( + [ + { id: "gpt-5.6-luna", supportedReasoningEfforts: ["medium", "high"] }, + { id: "gpt-5.6-terra", supportedReasoningEfforts: ["medium", "high"] }, + { + id: "gpt-5.6-sol", + supportedReasoningEfforts: ["medium", "high", "xhigh"], + }, + ], + undefined, + { efforts: { implement: "xhigh" } }, + ); + expect( + terraWithoutXhigh.decide({ + role: "implement", + risk: "medium", + retryIndex: 0, + }), + ).toMatchObject({ + profile: "sol", + model: "gpt-5.6-sol", + effort: "xhigh", + }); +}); + +test("falls back to the role default with one diagnostic per role when the configured effort is unsupported", () => { + const mediumHighOnly = [ + { id: "gpt-5.6-luna", supportedReasoningEfforts: ["medium", "high"] }, + { id: "gpt-5.6-terra", supportedReasoningEfforts: ["medium", "high"] }, + { id: "gpt-5.6-sol", supportedReasoningEfforts: ["medium", "high"] }, + ]; + const diagnostics: string[] = []; + const advisor = createModelAdvisor(mediumHighOnly, undefined, { + efforts: { implement: "xhigh" }, + onDiagnostic: (message) => diagnostics.push(message), + }); + + expect( + advisor.decide({ role: "implement", risk: "medium", retryIndex: 0 }), + ).toMatchObject({ + profile: "terra", + model: "gpt-5.6-terra", + effort: "medium", + rationale: [ + "implement baseline", + "medium risk", + "configured effort xhigh unsupported", + "effort medium (default)", + ], + }); + expect(diagnostics).toEqual([ + 'Configured implement effort "xhigh" is unsupported by the routed models; using the default "medium" instead.', + ]); + + advisor.decide({ + role: "implement", + risk: "high", + retryIndex: 1, + priorProfile: "terra", + }); + expect(diagnostics).toHaveLength(1); + + expect( + advisor.decide({ role: "scout", risk: "medium", retryIndex: 0 }), + ).toMatchObject({ profile: "luna", model: "gpt-5.6-luna", effort: "high" }); + expect(diagnostics).toHaveLength(1); +}); + +test("returns undefined when both configured and default efforts are unsupported", () => { + const diagnostics: string[] = []; + const advisor = createModelAdvisor( + [{ id: "provider/astra", supportedReasoningEfforts: ["high"] }], + { luna: "provider/astra", terra: "provider/astra", sol: "provider/astra" }, + { + efforts: { implement: "xhigh" }, + onDiagnostic: (message) => diagnostics.push(message), + }, + ); + + expect( + advisor.decide({ role: "implement", risk: "low", retryIndex: 0 }), + ).toBeUndefined(); + expect(diagnostics).toEqual([ + 'Configured implement effort "xhigh" is unsupported by the routed models; using the default "medium" instead.', + ]); +}); diff --git a/test/settings.test.ts b/test/settings.test.ts index 8cac00e..eae52dd 100644 --- a/test/settings.test.ts +++ b/test/settings.test.ts @@ -24,6 +24,7 @@ test("round-trips optional profile mappings and rejects malformed configuration" const settings = { cycle: { type: "weekly" as const }, models: { luna: "openai-codex/gpt-5.6-luna" }, + efforts: { implement: "xhigh" as const }, }; await saveRocSettings(settings, homeRoot); expect(await loadRocSettings(homeRoot)).toEqual(settings); @@ -38,6 +39,44 @@ test("round-trips optional profile mappings and rejects malformed configuration" .success, ).toBe(false); } + for (const efforts of [ + { coder: "high" }, + { implement: "low" }, + { implement: "max" }, + { scout: "medium", review: "secret-effort" }, + "high", + ]) { + expect( + RocSettingsSchema.safeParse({ cycle: { type: "weekly" }, efforts }) + .success, + ).toBe(false); + } +}); + +test("rejects malformed efforts with bounded diagnostics", async () => { + const homeRoot = await mkdtemp(join(tmpdir(), "roc-effort-settings-")); + await saveRocSettings({ cycle: { type: "weekly" } }, homeRoot); + const path = rocSettingsPath(homeRoot); + + for (const [source, expected] of [ + [ + '{"cycle":{"type":"weekly"},"efforts":{"implementer":"secret-effort"}}', + "Unsupported fields: 1 other field name(s) hidden.", + ], + [ + '{"cycle":{"type":"weekly"},"efforts":{"implement":"ultra-secret"}}', + "Invalid settings data; check the supported settings types and structure.", + ], + ] as const) { + await writeFile(path, source); + const error = await loadRocSettings(homeRoot).then( + () => undefined, + (failure: AgileError) => failure, + ); + expect(error).toBeInstanceOf(AgileError); + expect(error?.message).toContain(expected); + expect(error?.message).not.toContain("secret"); + } }); test("saves and loads strict global settings", async () => {