From b6089bf0e72a2b91c6a04e98a489b0f54953a1eb Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Wed, 6 May 2026 15:58:07 -0700 Subject: [PATCH 01/49] init --- .../benchmarks/swe_bench/scripts/run_infer.sh | 106 ++++ packages/opencode/src/bench/cli.ts | 389 +++++++++++++ packages/opencode/src/provider/provider.ts | 5 + .../src/provider/sdk/nemo-gym/index.ts | 62 +++ .../provider/sdk/nemo-gym/language-model.ts | 510 ++++++++++++++++++ 5 files changed, 1072 insertions(+) create mode 100755 evaluation/benchmarks/swe_bench/scripts/run_infer.sh create mode 100644 packages/opencode/src/bench/cli.ts create mode 100644 packages/opencode/src/provider/sdk/nemo-gym/index.ts create mode 100644 packages/opencode/src/provider/sdk/nemo-gym/language-model.ts diff --git a/evaluation/benchmarks/swe_bench/scripts/run_infer.sh b/evaluation/benchmarks/swe_bench/scripts/run_infer.sh new file mode 100755 index 000000000000..25bd5b001e09 --- /dev/null +++ b/evaluation/benchmarks/swe_bench/scripts/run_infer.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Bench entry script — invoked by gym's OpenCodeHarnessProcessor.get_run_command(). +# +# Args (positional, must match the order in app.py's get_run_command): +# $1 COMMIT_HASH opencode commit (informational; checkout is done at setup) +# $2 AGENT agent class name (informational) +# $3 MAX_ITER max agent turns +# $4 DATASET dataset name (informational; dispatch already done by gym) +# $5 SPLIT dataset split (informational) +# $6 EVAL_OUTPUT_DIR where to write trajectories (relative to opencode dir) +# $7 SELECTED_ID instance_id to run +# $8 INSTANCE_DICT_PATH /root/dataset/data.jsonl (single-line JSONL) +# $9 CONFIG_FILE opencode model config JSON (written by gym) +# $10 USER_PROMPT_PATH optional +# $11 SYSTEM_PROMPT_PATH optional +# +# Environment (set by gym): +# NEMO_GYM_MODEL_SERVER_NAME proxy name on the gym head server +# NEMO_GYM_MODEL_SERVER_BASE_URL base http://host:port for the model server +# NEMO_GYM_METRICS_FPATH path to the metrics JSON to update +# NEMO_GYM_CONFIG_DICT (informational) the gym YAML config blob +# COMMAND_EXEC_TIMEOUT per-bash-command timeout in seconds +# DIVERSIFY_TOOL_NAMES optional: rename tools for RL diversity +# CAMEL_CASE_TOOL_NAMES optional: camelCase tool names + +set -eo pipefail + +COMMIT_HASH="${1:-}" +AGENT="${2:-OpenCodeAgent}" +MAX_ITER="${3:-100}" +DATASET="${4:-}" +SPLIT="${5:-test}" +EVAL_OUTPUT_DIR="${6:-evaluation/oh}" +SELECTED_ID="${7:-}" +INSTANCE_DICT_PATH="${8:-/root/dataset/data.jsonl}" +CONFIG_FILE="${9:-/tmp/oc_config.json}" +USER_PROMPT_PATH="${10:-}" +SYSTEM_PROMPT_PATH="${11:-}" + +if [ -z "$SELECTED_ID" ]; then + echo "ERROR: SELECTED_ID (\$7) is required." + exit 64 +fi +if [ -z "${NEMO_GYM_MODEL_SERVER_NAME:-}" ]; then + echo "ERROR: NEMO_GYM_MODEL_SERVER_NAME not set in env." + exit 65 +fi +if [ -z "${NEMO_GYM_MODEL_SERVER_BASE_URL:-}" ]; then + echo "ERROR: NEMO_GYM_MODEL_SERVER_BASE_URL not set in env." + exit 66 +fi + +# Resolve the opencode root directory. The script lives at +# evaluation/benchmarks/swe_bench/scripts/run_infer.sh — go up four levels. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OPENCODE_DIR="$(cd "$SCRIPT_DIR/../../../.." && pwd)" +BENCH_CLI="$OPENCODE_DIR/packages/opencode/src/bench/cli.ts" + +if [ ! -f "$BENCH_CLI" ]; then + echo "ERROR: bench cli.ts not found at $BENCH_CLI" + exit 67 +fi +if ! command -v bun >/dev/null 2>&1; then + echo "ERROR: bun not on PATH (expected /opencode_setup/bun/bin/bun)" + exit 68 +fi + +# Make EVAL_OUTPUT_DIR absolute (relative to opencode dir). +case "$EVAL_OUTPUT_DIR" in + /*) ABS_OUTPUT_DIR="$EVAL_OUTPUT_DIR" ;; + *) ABS_OUTPUT_DIR="$OPENCODE_DIR/$EVAL_OUTPUT_DIR" ;; +esac +mkdir -p "$ABS_OUTPUT_DIR" + +# Echo the resolved config for log analysis. +echo "OPENCODE_DIR: $OPENCODE_DIR" +echo "BENCH_CLI: $BENCH_CLI" +echo "AGENT: $AGENT COMMIT: $COMMIT_HASH MAX_ITER: $MAX_ITER" +echo "DATASET: $DATASET SPLIT: $SPLIT SELECTED_ID: $SELECTED_ID" +echo "EVAL_OUTPUT_DIR: $ABS_OUTPUT_DIR" +echo "INSTANCE_DICT_PATH: $INSTANCE_DICT_PATH" +echo "CONFIG_FILE: $CONFIG_FILE" +echo "USER_PROMPT_PATH: $USER_PROMPT_PATH" +echo "SYSTEM_PROMPT_PATH: $SYSTEM_PROMPT_PATH" +echo "MODEL_SERVER: $NEMO_GYM_MODEL_SERVER_NAME @ $NEMO_GYM_MODEL_SERVER_BASE_URL" + +cmd=( + bun "$BENCH_CLI" + --instance-dict-path "$INSTANCE_DICT_PATH" + --output-dir "$ABS_OUTPUT_DIR" + --config "$CONFIG_FILE" + --max-turns "$MAX_ITER" + --agent-cls "$AGENT" + --dataset "$DATASET" + --split "$SPLIT" + --selected-id "$SELECTED_ID" +) +if [ -n "$USER_PROMPT_PATH" ]; then + cmd+=(--user-prompt "$USER_PROMPT_PATH") +fi +if [ -n "$SYSTEM_PROMPT_PATH" ]; then + cmd+=(--system-prompt "$SYSTEM_PROMPT_PATH") +fi + +echo "Executing: ${cmd[*]}" +exec "${cmd[@]}" diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts new file mode 100644 index 000000000000..33dbb92d6e6b --- /dev/null +++ b/packages/opencode/src/bench/cli.ts @@ -0,0 +1,389 @@ +/** + * SWE-bench bench CLI driver. + * + * Drives a single SWE-bench instance to completion using opencode's REAL + * agentic loop. We spawn `bun .../src/index.ts run` as a subprocess (with a + * per-instance opencode config that registers our `nemo-gym` provider, a + * SWE-bench agent, and disables compaction) and let it run to idle. + * + * Why subprocess instead of in-process Server.Default? Subprocess is the + * model the user-facing `opencode run` already uses (cli/cmd/run.ts:670–675 + * also uses an in-process fetch but the public entry is `bun .../index.ts`). + * A subprocess gives us: + * - clean process isolation per instance (matters for many parallel SIFs) + * - identical bootstrapping path to `opencode run`, so we don't drift + * - the JSON event stream on stdout for free (--format json) + * + * Trajectory capture: the nemo-gym provider (registered via this config) + * writes `/.json` per LLM call BEFORE returning. On + * exit we capture `git diff` and write `output.jsonl`. + */ + +import { promises as fs, readFileSync } from "node:fs" +import path from "node:path" +import os from "node:os" +import { spawn } from "node:child_process" + +interface CliArgs { + instanceDictPath: string + outputDir: string + config: string + maxTurns: number + agentCls: string + dataset: string + split: string + selectedId: string + userPromptPath?: string + systemPromptPath?: string +} + +function parseArgs(argv: string[]): CliArgs { + const out: Partial = { maxTurns: 100, agentCls: "OpenCodeAgent", dataset: "", split: "test" } + for (let i = 0; i < argv.length; i++) { + const a = argv[i] + const next = () => argv[++i] + switch (a) { + case "--instance-dict-path": + out.instanceDictPath = next() + break + case "--output-dir": + out.outputDir = next() + break + case "--config": + out.config = next() + break + case "--max-turns": + out.maxTurns = parseInt(next(), 10) + break + case "--agent-cls": + out.agentCls = next() + break + case "--dataset": + out.dataset = next() + break + case "--split": + out.split = next() + break + case "--selected-id": + out.selectedId = next() + break + case "--user-prompt": + out.userPromptPath = next() + break + case "--system-prompt": + out.systemPromptPath = next() + break + default: + if (a.startsWith("--")) throw new Error(`Unknown flag: ${a}`) + } + } + for (const required of ["instanceDictPath", "outputDir", "config", "selectedId"] as const) { + if (!out[required]) + throw new Error(`Missing required arg --${required.replace(/[A-Z]/g, (c) => "-" + c.toLowerCase())}`) + } + return out as CliArgs +} + +interface InstanceDict { + instance_id: string + problem_statement: string + repo?: string + repo_name?: string + workspace?: string + [key: string]: unknown +} + +async function readInstance(instanceDictPath: string, selectedId: string): Promise { + const text = await fs.readFile(instanceDictPath, "utf8") + const lines = text + .split("\n") + .map((l) => l.trim()) + .filter(Boolean) + const records = lines.map((l) => JSON.parse(l) as InstanceDict) + const match = records.find((r) => r.instance_id === selectedId) ?? records[0] + if (!match) throw new Error(`No instance found in ${instanceDictPath}`) + return match +} + +function detectWorkspaceRoot(instance: InstanceDict): string { + if (instance.workspace) return instance.workspace + // SWE-bench SIFs check the repo out at /testbed by convention. + return "/testbed" +} + +function loadGymConfig(configPath: string): Record { + return JSON.parse(readFileSync(configPath, "utf8")) +} + +const DEFAULT_SYSTEM_PROMPT = `You are an autonomous software engineer fixing a known issue in a checked-out git repository. + +Work in small, deliberate steps: +1. Read the issue and explore the relevant files. +2. Reproduce the issue if applicable. +3. Edit the source to fix the issue. +4. Run the project's tests to verify the fix. +5. Iterate until the issue is resolved. + +Use the available tools (bash, edit, read, glob, grep) to investigate and act. Do NOT modify the test files unless the task explicitly says so. The harness will capture the final \`git diff\` of the workspace as your patch — do not commit or format the diff yourself. +` + +async function buildConfigDir(args: { + instanceId: string + workspaceRoot: string + modelName: string + baseURL: string + completionsDir: string + maxTurns: number + problemStatement: string + systemPromptPath?: string + userPromptPath?: string +}): Promise<{ tmpRoot: string; configFile: string; userPrompt: string }> { + const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), `bench-${args.instanceId}-`)) + await fs.mkdir(tmpRoot, { recursive: true }) + + const systemPrompt = args.systemPromptPath + ? await fs.readFile(args.systemPromptPath, "utf8") + : DEFAULT_SYSTEM_PROMPT + + const userPromptTemplate = args.userPromptPath + ? await fs.readFile(args.userPromptPath, "utf8") + : `\n{{problem_statement}}\n\n\nThe workspace is at ${args.workspaceRoot}. Investigate, fix, run tests, and stop when the issue is resolved.` + + const cfg: Record = { + $schema: "https://opencode.ai/config.json", + provider: { + "nemo-gym": { + npm: "@opencode-ai/nemo-gym", + options: { + baseURL: args.baseURL, + completionsDir: args.completionsDir, + instanceId: args.instanceId, + }, + models: { + [args.modelName]: { + id: args.modelName, + name: args.modelName, + limit: { context: 131072, output: 32768 }, + tool_call: true, + temperature: true, + }, + }, + }, + }, + agent: { + "swe-bench": { + mode: "primary", + model: `nemo-gym/${args.modelName}`, + prompt: systemPrompt, + // Allow the read+write tool set; disable web/skill/task to keep the + // agent focused on local code editing. + permission: { + edit: { "**": "allow" }, + bash: { "*": "allow" }, + webfetch: { "*": "deny" }, + websearch: { "*": "deny" }, + }, + tools: { + bash: true, + edit: true, + read: true, + glob: true, + grep: true, + write: true, + apply_patch: true, + webfetch: false, + websearch: false, + task: false, + skill: false, + todowrite: false, + }, + steps: args.maxTurns, + options: {}, + }, + }, + compaction: { auto: false }, + share: "manual", + } + + const configFile = path.join(tmpRoot, "opencode.jsonc") + await fs.writeFile(configFile, JSON.stringify(cfg, null, 2)) + + return { + tmpRoot, + configFile, + userPrompt: userPromptTemplate.replace(/\{\{problem_statement\}\}/g, args.problemStatement), + } +} + +function runOpencode(args: { + workspaceRoot: string + modelName: string + message: string + env: NodeJS.ProcessEnv + opencodeBin: string + agent: string +}): Promise<{ exitCode: number; stdout: string; stderr: string }> { + return new Promise((resolve) => { + const child = spawn( + "bun", + [ + args.opencodeBin, + "run", + args.message, + "--agent", + args.agent, + "--model", + `nemo-gym/${args.modelName}`, + "--format", + "json", + "--dangerously-skip-permissions", + "--dir", + args.workspaceRoot, + ], + { + cwd: args.workspaceRoot, + env: args.env, + stdio: ["ignore", "pipe", "pipe"], + }, + ) + let stdout = "" + let stderr = "" + child.stdout?.on("data", (b) => { + const chunk = b.toString("utf8") + stdout += chunk + // Forward to our stdout so the gym log captures the event stream. + process.stdout.write(chunk) + }) + child.stderr?.on("data", (b) => { + const chunk = b.toString("utf8") + stderr += chunk + process.stderr.write(chunk) + }) + child.on("close", (code) => resolve({ exitCode: code ?? 0, stdout, stderr })) + child.on("error", (err) => { + stderr += String(err) + resolve({ exitCode: 999, stdout, stderr }) + }) + }) +} + +async function captureGitDiff(workspaceRoot: string): Promise { + return new Promise((resolve) => { + const child = spawn("git", ["-C", workspaceRoot, "diff"], { + env: { ...process.env, GIT_PAGER: "cat" }, + }) + let stdout = "" + child.stdout?.on("data", (b) => (stdout += b.toString("utf8"))) + child.on("close", () => resolve(stdout)) + child.on("error", () => resolve("")) + }) +} + +interface OutputJsonl { + instance_id: string + test_result: { git_patch: string } + metadata: { llm_config: { model: string } } + metrics: Record + error: string | null +} + +async function writeOutputJsonl(evalOutputDir: string, instanceId: string, payload: OutputJsonl): Promise { + const runDir = path.join(evalOutputDir, instanceId, "bench_run") + await fs.mkdir(runDir, { recursive: true }) + const outPath = path.join(runDir, "output.jsonl") + const tmp = `${outPath}.tmp` + await fs.writeFile(tmp, JSON.stringify(payload) + "\n") + await fs.rename(tmp, outPath) + return outPath +} + +function completionsDirFor(evalOutputDir: string, instanceId: string): string { + // Match openhands' on-host glob: /*/*/*/llm_completions//*.json + return path.join(evalOutputDir, instanceId, "bench_run", "llm_completions", instanceId) +} + +function detectOpencodeBin(): string { + // bench/cli.ts runs from packages/opencode/src/bench/. The opencode index + // entry sits at packages/opencode/src/index.ts. From this script's url we + // resolve up two levels. + const here = path.dirname(new URL(import.meta.url).pathname) + return path.resolve(here, "..", "index.ts") +} + +async function main() { + const args = parseArgs(process.argv.slice(2)) + const instance = await readInstance(args.instanceDictPath, args.selectedId) + const workspaceRoot = detectWorkspaceRoot(instance) + const gymConfig = loadGymConfig(args.config) + const llmModelCfg = ((gymConfig as Record>).llm?.model ?? {}) as Record< + string, + unknown + > + const modelName = String(llmModelCfg.model ?? "unknown-model") + const baseURL = process.env.NEMO_GYM_MODEL_SERVER_BASE_URL + if (!baseURL) throw new Error("NEMO_GYM_MODEL_SERVER_BASE_URL not set in env (gym harness sets this).") + + const completionsDir = completionsDirFor(args.outputDir, instance.instance_id) + await fs.mkdir(completionsDir, { recursive: true }) + + // We pre-render the user message so opencode's prompt machinery doesn't + // need to know about SWE-bench-specific templating. + const problemStatement = (instance.problem_statement ?? "").toString() + + const { tmpRoot, configFile, userPrompt } = await buildConfigDir({ + instanceId: instance.instance_id, + workspaceRoot, + modelName, + baseURL, + completionsDir, + maxTurns: args.maxTurns, + problemStatement, + systemPromptPath: args.systemPromptPath, + userPromptPath: args.userPromptPath, + }) + + const startedAt = Date.now() + const childEnv: NodeJS.ProcessEnv = { + ...process.env, + // Run-isolated opencode state. + OPENCODE_DB: ":memory:", + OPENCODE_DATA: path.join(tmpRoot, "data"), + OPENCODE_CONFIG: configFile, + // Disable opencode's built-in plugin loaders; the bench harness doesn't need them. + OPENCODE_PURE: "1", + } + + const opencodeBin = detectOpencodeBin() + const result = await runOpencode({ + workspaceRoot, + modelName, + message: userPrompt, + env: childEnv, + opencodeBin, + agent: "swe-bench", + }) + + const patch = await captureGitDiff(workspaceRoot) + const benchRunTime = (Date.now() - startedAt) / 1000 + + const error: string | null = result.exitCode === 0 ? null : `opencode_exit_${result.exitCode}` + const outPath = await writeOutputJsonl(args.outputDir, instance.instance_id, { + instance_id: instance.instance_id, + test_result: { git_patch: patch }, + metadata: { llm_config: { model: modelName } }, + metrics: { + bench_run_time: benchRunTime, + opencode_exit_code: result.exitCode, + }, + error, + }) + + console.log(`[bench] wrote ${outPath} (patch=${patch.length} bytes, error=${error ?? "none"})`) + + if (result.exitCode !== 0) process.exit(1) +} + +main().catch((err) => { + console.error(`[bench] fatal: ${err?.stack ?? err}`) + process.exit(2) +}) diff --git a/packages/opencode/src/provider/provider.ts b/packages/opencode/src/provider/provider.ts index 4013dcee36e7..dbcb319ff284 100644 --- a/packages/opencode/src/provider/provider.ts +++ b/packages/opencode/src/provider/provider.ts @@ -114,6 +114,11 @@ const BUNDLED_PROVIDERS: Record Promise<(opts: any) => BundledSDK> "gitlab-ai-provider": () => import("gitlab-ai-provider").then((m) => m.createGitLab), "@ai-sdk/github-copilot": () => import("./sdk/copilot/copilot-provider").then((m) => m.createOpenaiCompatible), "venice-ai-sdk-provider": () => import("venice-ai-sdk-provider").then((m) => m.createVenice), + // NeMo-Gym custom provider used by the SWE-bench RL rollout harness. + // Routes chat completions through the gym's vllm model server while + // threading prompt/generation token IDs into providerMetadata. The + // bench cli (`bench/cli.ts`) configures this provider per-instance. + "@opencode-ai/nemo-gym": () => import("./sdk/nemo-gym/index").then((m) => m.createNemoGym), } type CustomModelLoader = (sdk: any, modelID: string, options?: Record) => Promise diff --git a/packages/opencode/src/provider/sdk/nemo-gym/index.ts b/packages/opencode/src/provider/sdk/nemo-gym/index.ts new file mode 100644 index 000000000000..0d7001b8dc28 --- /dev/null +++ b/packages/opencode/src/provider/sdk/nemo-gym/index.ts @@ -0,0 +1,62 @@ +/** + * NeMo-Gym opencode provider entry. + * + * Provider id: `nemo-gym`. Used by the bench harness for SWE-bench RL rollouts. + * Registered in `provider/provider.ts:BUNDLED_PROVIDERS`. + * + * The factory mirrors `@ai-sdk/openai-compatible`'s shape: `createNemoGym(opts)` + * returns a provider with `.languageModel(modelId)` so opencode's existing + * provider plumbing (Provider.Service.getModel) works without special-casing. + */ + +import { NemoGymLanguageModel, type NemoGymLanguageModelConfig } from "./language-model" + +export interface CreateNemoGymOptions { + /** Base URL of the gym model server (`http://host:port`). */ + baseURL: string + /** Optional name of the model server (informational; useful for logs). */ + modelServerName?: string + /** Custom request headers. */ + headers?: () => Record + /** + * Where to dump per-call llm_completions/.json files. + * Set per-instance by the bench harness; if absent, no trajectory dump. + */ + completionsDir?: string + /** instance_id to embed in trajectory dump paths/file names. */ + instanceId?: string + /** Per-call HTTP timeout in ms. */ + requestTimeoutMs?: number + /** HTTP retry count on transient errors. */ + retries?: number + /** Optional turn counter shared across all model calls in a session. */ + turnCounter?: { next(): number } + /** Optional callback invoked after each successful chat-completion. */ + onCompletion?: NemoGymLanguageModelConfig["onCompletion"] +} + +export interface NemoGymProvider { + languageModel: (modelId: string) => NemoGymLanguageModel +} + +export function createNemoGym(opts: CreateNemoGymOptions): NemoGymProvider { + if (!opts.baseURL) { + throw new Error("createNemoGym: baseURL is required (e.g. http://host:port)") + } + return { + languageModel(modelId: string) { + return new NemoGymLanguageModel(modelId, { + provider: "nemo-gym", + baseURL: opts.baseURL, + modelServerName: opts.modelServerName, + headers: opts.headers, + completionsDir: opts.completionsDir, + instanceId: opts.instanceId, + requestTimeoutMs: opts.requestTimeoutMs, + retries: opts.retries, + turnCounter: opts.turnCounter, + onCompletion: opts.onCompletion, + }) + }, + } +} diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts new file mode 100644 index 000000000000..0ba88a5d1edb --- /dev/null +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -0,0 +1,510 @@ +/** + * NeMo-Gym LanguageModelV3 implementation. + * + * The opencode `processor.ts` agentic loop is unmodified — this is the only + * piece that swaps. Internally we POST to NeMo Gym's `/v1/chat/completions` + * non-streaming, capture token IDs (`prompt_token_ids` / `generation_token_ids` + * / `generation_log_probs`) from the response, and emit a single-shot synthetic + * stream so opencode's streaming handler is happy. + * + * Why non-streaming? RL training requires contiguous, exact token IDs across + * turns. Streaming has them drip in across SSE chunks; non-streaming returns + * them in the final response cleanly. The opencode loop doesn't notice — it + * receives all stream parts at once. + * + * Trajectory dump: every doStream call writes + * `//.json` BEFORE the stream finishes, + * so a tool crash later cannot lose this turn's token IDs. The shape matches + * openhands' `llm_completions//*.json` exactly so gym's + * `get_openhands_trajectory_from_completions` reads it without changes. + */ + +import { + type LanguageModelV3, + type LanguageModelV3CallOptions, + type LanguageModelV3StreamPart, + type LanguageModelV3Content, + type SharedV3ProviderMetadata, + type SharedV3Warning, +} from "@ai-sdk/provider" +import { promises as fs } from "node:fs" +import path from "node:path" +import { convertToOpenAICompatibleChatMessages } from "../copilot/chat/convert-to-openai-compatible-chat-messages" +import { prepareTools } from "../copilot/chat/openai-compatible-prepare-tools" + +// --------------------------------------------------------------------------- +// Wire types +// --------------------------------------------------------------------------- + +interface ChatRequestMessage { + role: "system" | "user" | "assistant" | "tool" + content?: string | Array | null + tool_calls?: Array<{ + id: string + type: "function" + function: { name: string; arguments: string } + }> + tool_call_id?: string + name?: string + prompt_token_ids?: number[] + generation_token_ids?: number[] + generation_log_probs?: number[] + [key: string]: unknown +} + +interface ChatResponseChoice { + index?: number + finish_reason?: string | null + message: { + role: string + content?: string | null + reasoning_text?: string | null + tool_calls?: Array<{ + id?: string + type?: string + function: { name: string; arguments: string } + }> + prompt_token_ids?: number[] + generation_token_ids?: number[] + generation_log_probs?: number[] + [key: string]: unknown + } +} + +interface ChatResponseUsage { + prompt_tokens?: number | null + completion_tokens?: number | null + total_tokens?: number | null +} + +interface ChatResponse { + id?: string + model?: string + created?: number + choices: ChatResponseChoice[] + usage?: ChatResponseUsage +} + +// --------------------------------------------------------------------------- +// Config +// --------------------------------------------------------------------------- + +const TOKEN_ID_FIELDS = ["prompt_token_ids", "generation_token_ids", "generation_log_probs"] as const + +export interface NemoGymLanguageModelConfig { + /** Provider id used to namespace providerMetadata. Defaults to "nemo-gym". */ + provider: string + /** Full base URL of the model server (e.g. `http://gym-host:18086`). */ + baseURL: string + /** Optional gym head-server-style model server name; informational only. */ + modelServerName?: string + /** Custom request headers (auth, etc). */ + headers?: () => Record + /** Per-call HTTP timeout in ms. */ + requestTimeoutMs?: number + /** Number of HTTP retry attempts on transient errors. */ + retries?: number + /** + * Where per-call llm_completions JSONs land. The bench harness builds this + * path; it must match what gym's host-side glob expects. If unset, no + * trajectory dump happens (useful for dev/test). + */ + completionsDir?: string + /** instance_id for the dump file naming + path. Required when completionsDir set. */ + instanceId?: string + /** Optional sink that the bench harness uses to count turns globally. */ + turnCounter?: { next(): number } + /** Optional callback fired after each successful chat completion. */ + onCompletion?: (info: { + turn: number + messages: ChatRequestMessage[] + response: ChatResponse + providerSpecificFields: Record + requestParams: Record + }) => void | Promise +} + +// --------------------------------------------------------------------------- +// Implementation +// --------------------------------------------------------------------------- + +export class NemoGymLanguageModel implements LanguageModelV3 { + readonly specificationVersion = "v3" + readonly modelId: string + readonly provider: string + + private readonly cfg: NemoGymLanguageModelConfig + private cookies: Record = {} + + constructor(modelId: string, cfg: NemoGymLanguageModelConfig) { + this.modelId = modelId + this.provider = cfg.provider + this.cfg = { + requestTimeoutMs: 600_000, + retries: 3, + ...cfg, + } + } + + get supportedUrls() { + return {} as Record + } + + // The streamText path in `session/llm.ts` only calls doStream. We still + // implement doGenerate for completeness / future direct-use. + async doGenerate(options: LanguageModelV3CallOptions) { + const { warnings, messages, requestParams } = await this._buildRequestParams(options) + const { responseJson } = await this._postChat(requestParams) + + const choice = responseJson.choices[0] + if (!choice) throw new Error("nemo-gym: empty choices in response") + const msg: ChatResponseChoice["message"] = choice.message ?? ({ role: "assistant" } as ChatResponseChoice["message"]) + + const content: LanguageModelV3Content[] = [] + if (msg.content) content.push({ type: "text", text: msg.content }) + if (msg.reasoning_text) content.push({ type: "reasoning", text: msg.reasoning_text }) + if (msg.tool_calls) { + for (const tc of msg.tool_calls) { + content.push({ + type: "tool-call", + toolCallId: tc.id ?? `call_${Math.random().toString(36).slice(2, 10)}`, + toolName: tc.function.name, + input: tc.function.arguments, + }) + } + } + + const providerSpecificFields = this._extractProviderFields(msg) + const providerMetadata = this._buildProviderMetadata(providerSpecificFields) + + await this._dumpAndNotify({ + messages, + response: responseJson, + providerSpecificFields, + requestParams, + }) + + return { + content, + finishReason: this._mapFinishReason(choice.finish_reason ?? null), + usage: this._mapUsage(responseJson.usage), + providerMetadata, + request: { body: JSON.stringify(requestParams) }, + response: { body: responseJson }, + warnings, + } + } + + async doStream(options: LanguageModelV3CallOptions) { + const { warnings, messages, requestParams } = await this._buildRequestParams(options) + + // Fire the HTTP call eagerly so any error surfaces synchronously when the + // stream is consumed. We then synthesize parts in `start`. + const self = this + + const stream = new ReadableStream({ + async start(controller) { + controller.enqueue({ type: "stream-start", warnings }) + + try { + const { responseJson } = await self._postChat(requestParams) + + const choice = responseJson.choices[0] + if (!choice) throw new Error("nemo-gym: empty choices in response") + const msg: ChatResponseChoice["message"] = + choice.message ?? ({ role: "assistant" } as ChatResponseChoice["message"]) + + const providerSpecificFields = self._extractProviderFields(msg) + const providerMetadata = self._buildProviderMetadata(providerSpecificFields) + + // Emit response-metadata first. + controller.enqueue({ + type: "response-metadata", + id: responseJson.id, + modelId: responseJson.model, + timestamp: responseJson.created ? new Date(responseJson.created * 1000) : undefined, + }) + + // Reasoning content. + if (msg.reasoning_text) { + controller.enqueue({ type: "reasoning-start", id: "reasoning-0" }) + controller.enqueue({ type: "reasoning-delta", id: "reasoning-0", delta: msg.reasoning_text }) + controller.enqueue({ type: "reasoning-end", id: "reasoning-0" }) + } + + // Text content. + if (msg.content) { + controller.enqueue({ type: "text-start", id: "txt-0" }) + controller.enqueue({ type: "text-delta", id: "txt-0", delta: msg.content }) + controller.enqueue({ type: "text-end", id: "txt-0" }) + } + + // Tool calls. + if (msg.tool_calls) { + for (const tc of msg.tool_calls) { + const tcId = tc.id ?? `call_${Math.random().toString(36).slice(2, 10)}` + controller.enqueue({ + type: "tool-input-start", + id: tcId, + toolName: tc.function.name, + }) + controller.enqueue({ + type: "tool-input-delta", + id: tcId, + delta: tc.function.arguments, + }) + controller.enqueue({ type: "tool-input-end", id: tcId }) + controller.enqueue({ + type: "tool-call", + toolCallId: tcId, + toolName: tc.function.name, + input: tc.function.arguments, + }) + } + } + + // Persist trajectory BEFORE finishing so a downstream tool crash + // cannot lose this turn's token IDs. + await self._dumpAndNotify({ + messages, + response: responseJson, + providerSpecificFields, + requestParams, + }) + + controller.enqueue({ + type: "finish", + finishReason: self._mapFinishReason(choice.finish_reason ?? null), + usage: self._mapUsage(responseJson.usage), + providerMetadata, + }) + + controller.close() + } catch (err) { + controller.enqueue({ type: "error", error: err instanceof Error ? err.message : String(err) }) + controller.enqueue({ + type: "finish", + finishReason: { unified: "error", raw: undefined }, + usage: self._mapUsage(undefined), + providerMetadata: {}, + }) + controller.close() + } + }, + }) + + return { + stream, + request: { body: JSON.stringify(requestParams) }, + response: {}, + } + } + + // ----------------------------------------------------------------------- + // Helpers + // ----------------------------------------------------------------------- + + private async _buildRequestParams(options: LanguageModelV3CallOptions): Promise<{ + warnings: SharedV3Warning[] + messages: ChatRequestMessage[] + tools: unknown + toolChoice: unknown + requestParams: Record + }> { + const warnings: SharedV3Warning[] = [] + // Reuse opencode's existing OpenAI-compatible message converter so all + // tool-call / multi-content shapes map identically to the rest of opencode. + const messages = convertToOpenAICompatibleChatMessages(options.prompt) as unknown as ChatRequestMessage[] + + const { tools, toolChoice, toolWarnings } = prepareTools({ + tools: options.tools, + toolChoice: options.toolChoice, + }) + warnings.push(...toolWarnings) + + // Strip token-ID fields from all assistant messages EXCEPT the most recent. + // Mirrors nemo_gym_client.py:85-97. Wire-payload dedup; the most recent + // message keeps its IDs so the server can verify continuity. + { + let lastSeen = false + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] as Record + const hasAll = TOKEN_ID_FIELDS.every((f) => f in m) + if (lastSeen) { + for (const f of TOKEN_ID_FIELDS) delete m[f] + } else if (hasAll) { + lastSeen = true + } + } + } + + const requestParams: Record = { + model: this.modelId, + messages, + max_tokens: options.maxOutputTokens, + temperature: options.temperature, + top_p: options.topP, + stop: options.stopSequences, + seed: options.seed, + } + if (tools && (tools as unknown[]).length) requestParams.tools = tools + if (toolChoice) requestParams.tool_choice = toolChoice + + // Strip undefineds — vllm errors on null/undefined keys. + for (const k of Object.keys(requestParams)) { + if (requestParams[k] === undefined) delete requestParams[k] + } + + return { warnings, messages, tools, toolChoice, requestParams } + } + + private async _postChat(params: Record): Promise<{ responseJson: ChatResponse }> { + const url = this._urlFor("/v1/chat/completions") + const headers: Record = { + "Content-Type": "application/json", + Accept: "application/json", + } + const cfgHeaders = this.cfg.headers?.() + if (cfgHeaders) { + for (const [k, v] of Object.entries(cfgHeaders)) if (v != null) headers[k] = v + } + if (Object.keys(this.cookies).length) { + headers.Cookie = Object.entries(this.cookies) + .map(([k, v]) => `${k}=${v}`) + .join("; ") + } + + const retries = this.cfg.retries ?? 3 + let lastErr: unknown = null + for (let attempt = 0; attempt < retries; attempt++) { + const ac = new AbortController() + const timer = setTimeout(() => ac.abort(), this.cfg.requestTimeoutMs) + try { + const res = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify(params), + signal: ac.signal, + }) + clearTimeout(timer) + if (!res.ok) { + const text = await res.text().catch(() => "") + throw new Error(`NeMoGym ${url} ${res.status}: ${text.slice(0, 500)}`) + } + const setCookie = res.headers.get("set-cookie") + if (setCookie) { + for (const part of setCookie.split(/,(?=[^;]+=)/)) { + const [kv] = part.split(";") + const [k, v] = kv.split("=") + if (k && v) this.cookies[k.trim()] = v.trim() + } + } + const responseJson = (await res.json()) as ChatResponse + return { responseJson } + } catch (err) { + clearTimeout(timer) + lastErr = err + if (attempt === retries - 1) break + await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt)) + } + } + throw new Error(`NeMoGym chat completions failed after ${retries} attempts: ${String(lastErr)}`) + } + + private _urlFor(p: string): string { + const base = this.cfg.baseURL.endsWith("/") ? this.cfg.baseURL : `${this.cfg.baseURL}/` + return new URL(p.replace(/^\//, ""), base).toString() + } + + private _extractProviderFields(msg: ChatResponseChoice["message"]): Record { + const out: Record = {} + if (Array.isArray(msg.prompt_token_ids)) { + for (const f of TOKEN_ID_FIELDS) { + const v = (msg as Record)[f] + if (v !== undefined) out[f] = v + } + } + return out + } + + private _buildProviderMetadata(providerSpecific: Record): SharedV3ProviderMetadata { + const md: SharedV3ProviderMetadata = { [this.provider]: {} } + for (const [k, v] of Object.entries(providerSpecific)) { + ;(md[this.provider] as Record)[k] = v as never + } + return md + } + + private _mapFinishReason(raw: string | null): { unified: "stop" | "length" | "tool-calls" | "error" | "other"; raw: string | undefined } { + if (!raw) return { unified: "other", raw: undefined } + switch (raw) { + case "stop": + return { unified: "stop", raw } + case "length": + return { unified: "length", raw } + case "tool_calls": + case "function_call": + return { unified: "tool-calls", raw } + default: + return { unified: "other", raw } + } + } + + private _mapUsage(raw?: ChatResponseUsage) { + return { + inputTokens: { + total: raw?.prompt_tokens ?? undefined, + noCache: raw?.prompt_tokens ?? undefined, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { + total: raw?.completion_tokens ?? undefined, + text: raw?.completion_tokens ?? undefined, + reasoning: undefined, + }, + } + } + + private async _dumpAndNotify(args: { + messages: ChatRequestMessage[] + response: ChatResponse + providerSpecificFields: Record + requestParams: Record + }) { + const turn = this.cfg.turnCounter ? this.cfg.turnCounter.next() : Date.now() + if (this.cfg.onCompletion) { + try { + await this.cfg.onCompletion({ turn, ...args }) + } catch (err) { + console.warn(`[nemo-gym] onCompletion hook threw: ${String(err)}`) + } + } + + if (!this.cfg.completionsDir || !this.cfg.instanceId) return + + try { + await fs.mkdir(this.cfg.completionsDir, { recursive: true }) + const turnStr = String(turn).padStart(4, "0") + const safeModel = this.modelId.replace(/\//g, "__") + const fname = `${safeModel}-${turnStr}-${Date.now()}.json` + const fpath = path.join(this.cfg.completionsDir, fname) + const kwargs: Record = {} + for (const [k, v] of Object.entries(args.requestParams)) { + if (k !== "messages") kwargs[k] = v + } + const payload = { + messages: args.messages, + response: args.response, + provider_specific_fields: args.providerSpecificFields, + kwargs, + timestamp: Date.now() / 1000, + } + const tmp = `${fpath}.tmp` + await fs.writeFile(tmp, JSON.stringify(payload)) + await fs.rename(tmp, fpath) + } catch (err) { + console.warn(`[nemo-gym] failed to dump completion: ${String(err)}`) + } + } +} From 1a1bf7a5f09b18f22eb70689cf7098c5327e78fc Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Wed, 6 May 2026 19:13:16 -0700 Subject: [PATCH 02/49] bench: add nemo-gym provider + run_infer entry --- .../benchmarks/swe_bench/scripts/run_infer.sh | 37 +++++++----- packages/opencode/src/bench/cli.ts | 56 +++++++++---------- 2 files changed, 48 insertions(+), 45 deletions(-) diff --git a/evaluation/benchmarks/swe_bench/scripts/run_infer.sh b/evaluation/benchmarks/swe_bench/scripts/run_infer.sh index 25bd5b001e09..8392a21b8b0e 100755 --- a/evaluation/benchmarks/swe_bench/scripts/run_infer.sh +++ b/evaluation/benchmarks/swe_bench/scripts/run_infer.sh @@ -5,14 +5,15 @@ # $1 COMMIT_HASH opencode commit (informational; checkout is done at setup) # $2 AGENT agent class name (informational) # $3 MAX_ITER max agent turns -# $4 DATASET dataset name (informational; dispatch already done by gym) +# $4 DATASET dataset name (informational; gym already dispatched) # $5 SPLIT dataset split (informational) # $6 EVAL_OUTPUT_DIR where to write trajectories (relative to opencode dir) # $7 SELECTED_ID instance_id to run # $8 INSTANCE_DICT_PATH /root/dataset/data.jsonl (single-line JSONL) # $9 CONFIG_FILE opencode model config JSON (written by gym) -# $10 USER_PROMPT_PATH optional -# $11 SYSTEM_PROMPT_PATH optional +# $10 WORKSPACE_ROOT resolved repo path inside the SIF (gym side decided) +# $11 USER_MESSAGE_PATH pre-rendered user prompt file (workspace baked in) +# $12 SYSTEM_PROMPT_PATH optional system-prompt override # # Environment (set by gym): # NEMO_GYM_MODEL_SERVER_NAME proxy name on the gym head server @@ -34,20 +35,29 @@ EVAL_OUTPUT_DIR="${6:-evaluation/oh}" SELECTED_ID="${7:-}" INSTANCE_DICT_PATH="${8:-/root/dataset/data.jsonl}" CONFIG_FILE="${9:-/tmp/oc_config.json}" -USER_PROMPT_PATH="${10:-}" -SYSTEM_PROMPT_PATH="${11:-}" +WORKSPACE_ROOT="${10:-}" +USER_MESSAGE_PATH="${11:-}" +SYSTEM_PROMPT_PATH="${12:-}" if [ -z "$SELECTED_ID" ]; then echo "ERROR: SELECTED_ID (\$7) is required." exit 64 fi +if [ -z "$WORKSPACE_ROOT" ]; then + echo "ERROR: WORKSPACE_ROOT (\$10) is required — gym side resolves the dataset-aware repo path." + exit 65 +fi +if [ -z "$USER_MESSAGE_PATH" ]; then + echo "ERROR: USER_MESSAGE_PATH (\$11) is required — gym side renders the user prompt." + exit 66 +fi if [ -z "${NEMO_GYM_MODEL_SERVER_NAME:-}" ]; then echo "ERROR: NEMO_GYM_MODEL_SERVER_NAME not set in env." - exit 65 + exit 67 fi if [ -z "${NEMO_GYM_MODEL_SERVER_BASE_URL:-}" ]; then echo "ERROR: NEMO_GYM_MODEL_SERVER_BASE_URL not set in env." - exit 66 + exit 68 fi # Resolve the opencode root directory. The script lives at @@ -58,11 +68,11 @@ BENCH_CLI="$OPENCODE_DIR/packages/opencode/src/bench/cli.ts" if [ ! -f "$BENCH_CLI" ]; then echo "ERROR: bench cli.ts not found at $BENCH_CLI" - exit 67 + exit 69 fi if ! command -v bun >/dev/null 2>&1; then echo "ERROR: bun not on PATH (expected /opencode_setup/bun/bin/bun)" - exit 68 + exit 70 fi # Make EVAL_OUTPUT_DIR absolute (relative to opencode dir). @@ -72,7 +82,6 @@ case "$EVAL_OUTPUT_DIR" in esac mkdir -p "$ABS_OUTPUT_DIR" -# Echo the resolved config for log analysis. echo "OPENCODE_DIR: $OPENCODE_DIR" echo "BENCH_CLI: $BENCH_CLI" echo "AGENT: $AGENT COMMIT: $COMMIT_HASH MAX_ITER: $MAX_ITER" @@ -80,7 +89,8 @@ echo "DATASET: $DATASET SPLIT: $SPLIT SELECTED_ID: $SELECTED_ID" echo "EVAL_OUTPUT_DIR: $ABS_OUTPUT_DIR" echo "INSTANCE_DICT_PATH: $INSTANCE_DICT_PATH" echo "CONFIG_FILE: $CONFIG_FILE" -echo "USER_PROMPT_PATH: $USER_PROMPT_PATH" +echo "WORKSPACE_ROOT: $WORKSPACE_ROOT" +echo "USER_MESSAGE_PATH: $USER_MESSAGE_PATH" echo "SYSTEM_PROMPT_PATH: $SYSTEM_PROMPT_PATH" echo "MODEL_SERVER: $NEMO_GYM_MODEL_SERVER_NAME @ $NEMO_GYM_MODEL_SERVER_BASE_URL" @@ -94,10 +104,9 @@ cmd=( --dataset "$DATASET" --split "$SPLIT" --selected-id "$SELECTED_ID" + --workspace-root "$WORKSPACE_ROOT" + --user-message-file "$USER_MESSAGE_PATH" ) -if [ -n "$USER_PROMPT_PATH" ]; then - cmd+=(--user-prompt "$USER_PROMPT_PATH") -fi if [ -n "$SYSTEM_PROMPT_PATH" ]; then cmd+=(--system-prompt "$SYSTEM_PROMPT_PATH") fi diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 33dbb92d6e6b..1961a6acaf60 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -33,7 +33,10 @@ interface CliArgs { dataset: string split: string selectedId: string - userPromptPath?: string + /** Resolved repo path inside the SIF — gym side decided based on dataset_name. */ + workspaceRoot: string + /** Pre-rendered user message file (workspace_path baked in by gym). */ + userMessageFile: string systemPromptPath?: string } @@ -67,8 +70,11 @@ function parseArgs(argv: string[]): CliArgs { case "--selected-id": out.selectedId = next() break - case "--user-prompt": - out.userPromptPath = next() + case "--workspace-root": + out.workspaceRoot = next() + break + case "--user-message-file": + out.userMessageFile = next() break case "--system-prompt": out.systemPromptPath = next() @@ -77,7 +83,14 @@ function parseArgs(argv: string[]): CliArgs { if (a.startsWith("--")) throw new Error(`Unknown flag: ${a}`) } } - for (const required of ["instanceDictPath", "outputDir", "config", "selectedId"] as const) { + for (const required of [ + "instanceDictPath", + "outputDir", + "config", + "selectedId", + "workspaceRoot", + "userMessageFile", + ] as const) { if (!out[required]) throw new Error(`Missing required arg --${required.replace(/[A-Z]/g, (c) => "-" + c.toLowerCase())}`) } @@ -105,12 +118,6 @@ async function readInstance(instanceDictPath: string, selectedId: string): Promi return match } -function detectWorkspaceRoot(instance: InstanceDict): string { - if (instance.workspace) return instance.workspace - // SWE-bench SIFs check the repo out at /testbed by convention. - return "/testbed" -} - function loadGymConfig(configPath: string): Record { return JSON.parse(readFileSync(configPath, "utf8")) } @@ -129,15 +136,12 @@ Use the available tools (bash, edit, read, glob, grep) to investigate and act. D async function buildConfigDir(args: { instanceId: string - workspaceRoot: string modelName: string baseURL: string completionsDir: string maxTurns: number - problemStatement: string systemPromptPath?: string - userPromptPath?: string -}): Promise<{ tmpRoot: string; configFile: string; userPrompt: string }> { +}): Promise<{ tmpRoot: string; configFile: string }> { const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), `bench-${args.instanceId}-`)) await fs.mkdir(tmpRoot, { recursive: true }) @@ -145,10 +149,6 @@ async function buildConfigDir(args: { ? await fs.readFile(args.systemPromptPath, "utf8") : DEFAULT_SYSTEM_PROMPT - const userPromptTemplate = args.userPromptPath - ? await fs.readFile(args.userPromptPath, "utf8") - : `\n{{problem_statement}}\n\n\nThe workspace is at ${args.workspaceRoot}. Investigate, fix, run tests, and stop when the issue is resolved.` - const cfg: Record = { $schema: "https://opencode.ai/config.json", provider: { @@ -208,11 +208,7 @@ async function buildConfigDir(args: { const configFile = path.join(tmpRoot, "opencode.jsonc") await fs.writeFile(configFile, JSON.stringify(cfg, null, 2)) - return { - tmpRoot, - configFile, - userPrompt: userPromptTemplate.replace(/\{\{problem_statement\}\}/g, args.problemStatement), - } + return { tmpRoot, configFile } } function runOpencode(args: { @@ -313,7 +309,8 @@ function detectOpencodeBin(): string { async function main() { const args = parseArgs(process.argv.slice(2)) const instance = await readInstance(args.instanceDictPath, args.selectedId) - const workspaceRoot = detectWorkspaceRoot(instance) + // workspaceRoot is decided gym-side based on dataset_name; we use it verbatim. + const workspaceRoot = args.workspaceRoot const gymConfig = loadGymConfig(args.config) const llmModelCfg = ((gymConfig as Record>).llm?.model ?? {}) as Record< string, @@ -326,20 +323,17 @@ async function main() { const completionsDir = completionsDirFor(args.outputDir, instance.instance_id) await fs.mkdir(completionsDir, { recursive: true }) - // We pre-render the user message so opencode's prompt machinery doesn't - // need to know about SWE-bench-specific templating. - const problemStatement = (instance.problem_statement ?? "").toString() + // The user message is fully rendered by gym (workspace_path baked in based + // on dataset_name); we just read it as-is and pass it to opencode. + const userPrompt = await fs.readFile(args.userMessageFile, "utf8") - const { tmpRoot, configFile, userPrompt } = await buildConfigDir({ + const { tmpRoot, configFile } = await buildConfigDir({ instanceId: instance.instance_id, - workspaceRoot, modelName, baseURL, completionsDir, maxTurns: args.maxTurns, - problemStatement, systemPromptPath: args.systemPromptPath, - userPromptPath: args.userPromptPath, }) const startedAt = Date.now() From c55a4bab7d0bcd134d08298fd58bc7a16b79c0d4 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Wed, 6 May 2026 19:23:51 -0700 Subject: [PATCH 03/49] bench: drop eager TUI command imports from CLI entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The opencode CLI registers TUI subcommands (Attach, TuiThread) via eager imports in index.ts. Loading them transitively imports cli/cmd/tui/app.tsx, whose JSX is meant to compile against @opentui/solid (per the package tsconfig's jsxImportSource) but is incorrectly resolved against react/jsx-dev-runtime when Bun runs the un-bundled .ts file at runtime — react isn't a dep of packages/opencode, so the bench harness crashes at startup before the run command ever executes. The bench harness (packages/opencode/src/bench/cli.ts) only ever spawns \`bun src/index.ts run\` as a subprocess; it never invokes the TUI. Remove the imports + command registrations entirely so the cli/cmd/tui/ subtree is unreachable from the bench code path. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/opencode/src/index.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 4c8e447041c0..444e9730d29c 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -22,8 +22,13 @@ import { McpCommand } from "./cli/cmd/mcp" import { GithubCommand } from "./cli/cmd/github" import { ExportCommand } from "./cli/cmd/export" import { ImportCommand } from "./cli/cmd/import" -import { AttachCommand } from "./cli/cmd/tui/attach" -import { TuiThreadCommand } from "./cli/cmd/tui/thread" +// TUI subcommands (Attach, TuiThread) are dropped from this build of opencode. +// The bench harness (`packages/opencode/src/bench/cli.ts`) only invokes the +// `run` command via subprocess, never the TUI; loading them eagerly here drags +// in `cli/cmd/tui/app.tsx` at startup, which JSX-compiles against +// `@opentui/solid` and trips Bun's runtime JSX resolver into looking for +// `react/jsx-dev-runtime` (a bug we hit when running the un-bundled .ts). +// Removed entirely rather than lazy-loaded — bench has no use for them. import { AcpCommand } from "./cli/cmd/acp" import { EOL } from "os" import { WebCommand } from "./cli/cmd/web" @@ -156,8 +161,6 @@ const cli = yargs(args) .completion("completion", "generate shell completion script") .command(AcpCommand) .command(McpCommand) - .command(TuiThreadCommand) - .command(AttachCommand) .command(RunCommand) .command(GenerateCommand) .command(DebugCommand) From ab84347402bb65b85452642083935749a8a8928a Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Wed, 6 May 2026 19:33:49 -0700 Subject: [PATCH 04/49] bench: prefer pre-bundled opencode.js over running src/index.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running opencode's CLI un-bundled (`bun src/index.ts run`) triggers cascading runtime-resolution failures: tsconfig's `jsxImportSource` isn't honored for inline `.tsx` JIT compilation (we already removed the TUI commands to dodge that), and bun's isolated install layout under `node_modules/.bun/@/...` breaks `..`-relative `.mjs` imports inside packages like @anthropic-ai/sdk (the `internal/to-file.mjs` import from `core/uploads.mjs` fails to traverse the symlink the way Node does at runtime). opencode is meant to be shipped as a pre-bundled single file (their own `bin/opencode` is built the same way) — `bun build` resolves every transitive import statically and inlines the result, so runtime never has to. The companion gym `setup_scripts/opencode.sh` now invokes `bun build packages/opencode/src/index.ts --outdir .bench-build --entry-naming opencode.js`. This commit teaches `bench/cli.ts` to prefer that bundle when it exists, and falls back to `src/index.ts` for dev / when the build step hasn't run. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/opencode/src/bench/cli.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 1961a6acaf60..4d2901ebe9e9 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -19,7 +19,7 @@ * exit we capture `git diff` and write `output.jsonl`. */ -import { promises as fs, readFileSync } from "node:fs" +import { existsSync, promises as fs, readFileSync } from "node:fs" import path from "node:path" import os from "node:os" import { spawn } from "node:child_process" @@ -299,10 +299,18 @@ function completionsDirFor(evalOutputDir: string, instanceId: string): string { } function detectOpencodeBin(): string { - // bench/cli.ts runs from packages/opencode/src/bench/. The opencode index - // entry sits at packages/opencode/src/index.ts. From this script's url we - // resolve up two levels. + // Prefer the pre-bundled artifact at /.bench-build/opencode.js. + // Running un-bundled `src/index.ts` triggers cascading runtime resolution + // failures (TUI JSX runtime not honored, @anthropic-ai/sdk relative .mjs + // paths failing across the isolated install layout). The bundle inlines + // every transitive dep and is opencode's intended deployment shape. + // Falls back to src/index.ts only for dev / when setup_scripts/opencode.sh + // hasn't run. const here = path.dirname(new URL(import.meta.url).pathname) + // bench/cli.ts → packages/opencode/src/bench → packages/opencode/src → packages/opencode → packages → + const opencodeRoot = path.resolve(here, "..", "..", "..", "..") + const bundled = path.resolve(opencodeRoot, ".bench-build", "opencode.js") + if (existsSync(bundled)) return bundled return path.resolve(here, "..", "index.ts") } From 85e8e32179a28454744166f254ae61f6cd3196ec Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Thu, 7 May 2026 17:41:30 -0700 Subject: [PATCH 05/49] bench: commit empty models-snapshot stubs (force-added) `provider/models.ts:137` does `import("./models-snapshot.js")` for a static cache of models.dev metadata. Upstream generates this file at build time via `script/generate.ts` (network fetch from models.dev), and `.gitignore` correctly excludes it as a build artifact. For the bench harness we register a single custom provider (`@opencode-ai/nemo-gym`) in the per-instance opencode config and never consult the snapshot. But `bun build --target=bun packages/opencode/src/index.ts ...` still needs the import target to exist at static-analysis time, otherwise it errors with: error: Could not resolve: "./models-snapshot.js" Force-add empty stubs so the bundle build succeeds without running `script/generate.ts` (which needs network + adds 5+ seconds). The runtime `try:` lambda in models.ts handles an empty snapshot fine. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../opencode/src/provider/models-snapshot.d.ts | 3 +++ .../opencode/src/provider/models-snapshot.js | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 packages/opencode/src/provider/models-snapshot.d.ts create mode 100644 packages/opencode/src/provider/models-snapshot.js diff --git a/packages/opencode/src/provider/models-snapshot.d.ts b/packages/opencode/src/provider/models-snapshot.d.ts new file mode 100644 index 000000000000..508ab6ee22fe --- /dev/null +++ b/packages/opencode/src/provider/models-snapshot.d.ts @@ -0,0 +1,3 @@ +// Empty stub committed for the bench harness build path. See models-snapshot.js +// for the rationale. +export declare const snapshot: Record diff --git a/packages/opencode/src/provider/models-snapshot.js b/packages/opencode/src/provider/models-snapshot.js new file mode 100644 index 000000000000..c48d54a8f72a --- /dev/null +++ b/packages/opencode/src/provider/models-snapshot.js @@ -0,0 +1,18 @@ +// @ts-nocheck +// Empty stub committed for the bench harness build path. +// +// Upstream opencode generates this file at build time via `script/generate.ts` +// (which fetches https://models.dev/api.json). For the nemo-gym bench harness +// we only register a single custom provider in the per-instance opencode +// config, so the snapshot is unused — but `bun build` still has to resolve +// `import("./models-snapshot.js")` from `provider/models.ts:137` at static +// analysis time. An empty snapshot satisfies that requirement; the runtime +// `try:` lambda in models.ts handles an empty snapshot gracefully. +// +// `.gitignore` excludes this file because upstream regenerates it. We +// force-add it on the bench branch (sdd/dev) so `bun build --target=bun +// packages/opencode/src/index.ts ...` succeeds without running generate.ts +// (which requires network access to models.dev). If you ever DO want real +// model metadata, run `bun run script/generate.ts` and don't commit the +// regenerated file. +export const snapshot = {} From 7c9b883e5376b8aa6bbd7fd07911efc3bebbb2bb Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sun, 10 May 2026 15:51:55 -0700 Subject: [PATCH 06/49] bench: drop webfetch/websearch permission entries from per-instance config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PermissionActionConfig` accepts a glob-keyed map only for file/shell-style permissions (edit, bash). webfetch / websearch use a different shape (a single literal action), so emitting `{"*": "deny"}` for them trips the config validator at startup: Error: Configuration is invalid at .../opencode.jsonc ↳ Expected PermissionActionConfig | undefined, got {"*":"deny"} agent.swe-bench.permission.webfetch ↳ Expected PermissionActionConfig | undefined, got {"*":"deny"} agent.swe-bench.permission.websearch Both tools are already disabled via the agent's `tools:` block (webfetch: false, websearch: false), so the permission entries are redundant. Removing them lets the per-instance config load cleanly and the run command actually start. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/opencode/src/bench/cli.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 4d2901ebe9e9..e3e5b0f23f94 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -178,10 +178,12 @@ async function buildConfigDir(args: { // Allow the read+write tool set; disable web/skill/task to keep the // agent focused on local code editing. permission: { + // Glob-keyed `PermissionActionConfig` for file/shell access. edit: { "**": "allow" }, bash: { "*": "allow" }, - webfetch: { "*": "deny" }, - websearch: { "*": "deny" }, + // webfetch / websearch use a different schema (single action, not + // a glob map) and we already disable them in `tools` below — no + // need for an explicit entry here. }, tools: { bash: true, From f59e8c6e7f81a6261219c72a71fb4a0a12815ea6 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sun, 10 May 2026 15:59:12 -0700 Subject: [PATCH 07/49] bench: accept opts.headers as either function or plain object opencode's bundled-provider loader invokes the factory with \`{ name, ...options }\` where `options` is the merged user + custom provider defaults from `provider.ts`. For some providers opencode injects an empty/static `headers: {}` into that merged options dict (distinct from upstream openai-compatible's schema where `headers` is a function `() => Record`). We were unconditionally invoking `this.cfg.headers?.()`. The optional chain only short-circuits on null/undefined, so when opencode passed a plain object it threw at session-start: Error: this.cfg.headers is not a function. (In 'this.cfg.headers?.()', 'this.cfg.headers' is an instance of Object) Branch on `typeof headers` to support both shapes: call it if it's a function, spread it directly if it's a plain object. Either way the resulting kv pairs are merged into the per-request HTTP headers. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/provider/sdk/nemo-gym/language-model.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 0ba88a5d1edb..af81208bf6b2 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -364,7 +364,17 @@ export class NemoGymLanguageModel implements LanguageModelV3 { "Content-Type": "application/json", Accept: "application/json", } - const cfgHeaders = this.cfg.headers?.() + // opencode's bundled-provider loader can pass `headers` as either a + // function (matching upstream openai-compatible's schema) OR a plain + // object (when opencode injects defaults from its provider merge layer). + // Handle both — `?.()` would throw on a non-callable object. + let cfgHeaders: Record | undefined + const rawHeaders = this.cfg.headers as unknown + if (typeof rawHeaders === "function") { + cfgHeaders = (rawHeaders as () => Record)() + } else if (rawHeaders && typeof rawHeaders === "object") { + cfgHeaders = rawHeaders as Record + } if (cfgHeaders) { for (const [k, v] of Object.entries(cfgHeaders)) if (v != null) headers[k] = v } From b401866bbb7b5fcb5f6d88cd9c4344e522115402 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sun, 10 May 2026 16:28:27 -0700 Subject: [PATCH 08/49] bench: fix default-merge bug that made every fetch abort instantly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The constructor merged caller config over defaults via: this.cfg = { requestTimeoutMs: 600_000, retries: 3, ...cfg } opencode's provider loader invokes the factory with options that include explicit `undefined` for optional fields. Our factory in `provider/sdk/nemo-gym/index.ts` faithfully passes those through: new NemoGymLanguageModel(modelId, { ..., requestTimeoutMs: opts.requestTimeoutMs, // undefined retries: opts.retries, // undefined }) The `...cfg` spread then overwrites the 600_000 / 3 defaults with `undefined`. At runtime `setTimeout(fn, undefined)` is treated as `setTimeout(fn, 0)` — so the abort timer fires before `fetch` can even hand off the request. Result: NeMoGym chat completions failed after 3 attempts: AbortError: The operation was aborted. with ~10 ms total elapsed between step_start and the error event. Swap to the spread-first, coalesce-with-`??` pattern so undefineds fall through to the defaults instead of clobbering them. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../opencode/src/provider/sdk/nemo-gym/language-model.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index af81208bf6b2..fdb1ff9e034d 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -139,10 +139,14 @@ export class NemoGymLanguageModel implements LanguageModelV3 { constructor(modelId: string, cfg: NemoGymLanguageModelConfig) { this.modelId = modelId this.provider = cfg.provider + // Spread first, then coalesce — opencode's provider loader passes + // optional fields explicitly as `undefined`, and a default-then-spread + // pattern lets those undefineds overwrite the defaults. `??` only + // replaces null/undefined, preserving any real caller-supplied value. this.cfg = { - requestTimeoutMs: 600_000, - retries: 3, ...cfg, + requestTimeoutMs: cfg.requestTimeoutMs ?? 600_000, + retries: cfg.retries ?? 3, } } From a2451878ef61447eaf3634c05d0b4537bf03761d Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sun, 10 May 2026 16:59:15 -0700 Subject: [PATCH 09/49] bench: omit body.model when modelId is empty / 'default' When the input row's responses_create_params.model is unset, the gym side hands opencode an empty model string. opencode's session/provider resolver then falls back to its sentinel `"default"` (or threads the empty string straight through). We were faithfully POSTing \`model: "default"\` to the gym openai_model server, which forwarded it to OpenAI: The model `default` does not exist or you do not have access to it. The gym openai_model server already has the right pattern for this: body_dict.setdefault("model", self.config.openai_model) so omitting `model` from the outbound body lets the server fill in the policy-configured model name (e.g. gpt-4.1-2025-04-14) without any client-side knowledge of what the policy actually points to. Guard the model assignment in `_buildRequestParams` to do exactly that. The companion gym-side change is in OpenCodeHarnessProcessor.get_run_command: when body.model is empty, fall back to the resolved model_server_cfg.openai_model (or .model for vllm) before writing the per-instance config, so opencode also sees a real string when it's populating the agent's model field. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/provider/sdk/nemo-gym/language-model.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index fdb1ff9e034d..fc940f986320 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -343,7 +343,6 @@ export class NemoGymLanguageModel implements LanguageModelV3 { } const requestParams: Record = { - model: this.modelId, messages, max_tokens: options.maxOutputTokens, temperature: options.temperature, @@ -351,6 +350,16 @@ export class NemoGymLanguageModel implements LanguageModelV3 { stop: options.stopSequences, seed: options.seed, } + // Only include `model` when the caller-supplied modelId is a real value. + // opencode's session resolver falls back to its sentinel `"default"` + // (and to empty string with some misconfigured agents) when no model is + // pinned. We DO NOT want either of those leaking through to OpenAI as a + // literal `model: "default"` — the gym openai_model server's + // `body_dict.setdefault("model", self.config.openai_model)` will fill in + // the policy-configured model name when we omit it instead. + if (this.modelId && this.modelId !== "default") { + requestParams.model = this.modelId + } if (tools && (tools as unknown[]).length) requestParams.tools = tools if (toolChoice) requestParams.tool_choice = toolChoice From 06724b5a2a058f8b876084269d20c6eeebcf42a6 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sun, 10 May 2026 17:57:08 -0700 Subject: [PATCH 10/49] bench: enable todowrite tool for SWE-bench agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The todowrite tool is just an in-session planning aid — explicit multi-step task lists help the model decompose non-trivial fixes without affecting token-ID contiguity or subagent dispatch. Was disabled defensively; flipping it on. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index e3e5b0f23f94..d8055570a769 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -197,7 +197,7 @@ async function buildConfigDir(args: { websearch: false, task: false, skill: false, - todowrite: false, + todowrite: true, }, steps: args.maxTurns, options: {}, From 2fec7806fe2c75c00644435dfa7dda2a28069647 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sun, 10 May 2026 18:29:34 -0700 Subject: [PATCH 11/49] bench: port _deep_reset_to_base_commit from nv-OpenHands Strip git history past base_commit so the agent can't reach future commits via `git checkout ` / `git log` exploration. Two-pass: careful per-ref iteration first; nuclear batch-delete fallback for monorepos with thousands of refs (e.g. datadog-agent with 5k+ release tags would time out the careful pass). Runs once in cli.ts before spawning the opencode session, scoped to the per-instance workspace, using the base_commit field from the instance JSONL. `|| true` at the tail so a busted git state can't kill the rollout. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/cli.ts | 5 ++ packages/opencode/src/bench/deep_reset.ts | 91 +++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 packages/opencode/src/bench/deep_reset.ts diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index d8055570a769..f3ca8995ad77 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -23,6 +23,7 @@ import { existsSync, promises as fs, readFileSync } from "node:fs" import path from "node:path" import os from "node:os" import { spawn } from "node:child_process" +import { runDeepReset } from "./deep_reset" interface CliArgs { instanceDictPath: string @@ -103,6 +104,7 @@ interface InstanceDict { repo?: string repo_name?: string workspace?: string + base_commit?: string [key: string]: unknown } @@ -357,6 +359,9 @@ async function main() { OPENCODE_PURE: "1", } + // Prune git history past base_commit so the agent can't reach future commits. + await runDeepReset(workspaceRoot, String(instance.base_commit ?? "")) + const opencodeBin = detectOpencodeBin() const result = await runOpencode({ workspaceRoot, diff --git a/packages/opencode/src/bench/deep_reset.ts b/packages/opencode/src/bench/deep_reset.ts new file mode 100644 index 000000000000..89a9c9d1834a --- /dev/null +++ b/packages/opencode/src/bench/deep_reset.ts @@ -0,0 +1,91 @@ +/** + * Strip git history past base_commit so the agent can't reach future commits. + * + * Port of nv-OpenHands' `_deep_reset_to_base_commit` + * (evaluation/benchmarks/swe_bench/run_infer.py:774). Two-pass design: + * + * - Careful pass: per-ref iteration with `git for-each-ref`. Preserves + * local branches that don't descend from base, resets branches that do, + * deletes tags/remote-tracking/stash/notes refs past base. + * - Nuclear fallback: batch-delete every tag/remote/stash/notes ref + every + * local branch in two `git update-ref --stdin` calls. Microseconds + * regardless of ref count — handles monorepos with thousands of refs + * where the careful pass times out. + * + * `|| true` at the very end so a busted git state can't kill the agent run. + */ + +import { spawn } from "node:child_process" + +function carefulPass(baseCommit: string): string { + return ( + `BASE=$(git rev-parse --verify ${baseCommit}^{commit}) && ` + + `ORIG_BRANCH=$(git symbolic-ref --short -q HEAD || echo main) && ` + + `git checkout --detach "$BASE" && ` + + `git for-each-ref --format="%(refname)" refs/heads | while read -r ref; do ` + + ` tip=$(git rev-parse -q --verify "$ref^{commit}" 2>/dev/null || true); ` + + ` [ -z "$tip" ] && continue; ` + + ` if [ "$tip" != "$BASE" ] && git merge-base --is-ancestor "$BASE" "$tip"; then ` + + ` git update-ref "$ref" "$BASE"; ` + + ` fi; ` + + `done && ` + + `git for-each-ref --format="%(refname)" refs | while read -r ref; do ` + + ` case "$ref" in refs/heads/*) continue ;; esac; ` + + ` if git symbolic-ref -q "$ref" >/dev/null 2>&1; then continue; fi; ` + + ` tip=$(git rev-parse -q --verify "$ref^{commit}" 2>/dev/null || true); ` + + ` [ -z "$tip" ] && continue; ` + + ` if [ "$tip" != "$BASE" ] && git merge-base --is-ancestor "$BASE" "$tip"; then ` + + ` git update-ref -d "$ref"; ` + + ` fi; ` + + `done && ` + + `for r in $(git remote); do git remote remove "$r"; done; ` + + `gd=$(git rev-parse --git-dir) && ` + + `rm -f "$gd"/FETCH_HEAD "$gd"/ORIG_HEAD "$gd"/MERGE_HEAD "$gd"/CHERRY_PICK_HEAD ` + + `"$gd"/REVERT_HEAD "$gd"/BISECT_HEAD "$gd"/AUTO_MERGE && ` + + `git reflog expire --expire=now --expire-unreachable=now --all && ` + + `git repack -ad && git prune --expire=now && git gc --prune=now && ` + + `git checkout -B "$ORIG_BRANCH" "$BASE"` + ) +} + +function nuclearPass(baseCommit: string): string { + return ( + `BASE=$(git rev-parse --verify ${baseCommit}^{commit}) && ` + + `ORIG_BRANCH=$(git symbolic-ref --short -q HEAD || echo main) && ` + + `git checkout --detach "$BASE" && ` + + `for r in $(git remote); do git remote remove "$r"; done; ` + + `git for-each-ref --format="delete %(refname)" refs/tags refs/remotes refs/stash refs/notes 2>/dev/null ` + + `| git update-ref --stdin; ` + + `git for-each-ref --format="delete %(refname)" refs/heads | git update-ref --stdin; ` + + `gd=$(git rev-parse --git-dir) && ` + + `rm -f "$gd"/FETCH_HEAD "$gd"/ORIG_HEAD "$gd"/MERGE_HEAD "$gd"/CHERRY_PICK_HEAD ` + + `"$gd"/REVERT_HEAD "$gd"/BISECT_HEAD "$gd"/AUTO_MERGE && ` + + `git reflog expire --expire=now --expire-unreachable=now --all && ` + + `git repack -ad && git prune --expire=now && git gc --prune=now && ` + + `git checkout -B "$ORIG_BRANCH" "$BASE"` + ) +} + +export function buildDeepResetCmd(baseCommit: string): string { + return `( ${carefulPass(baseCommit)} ) || ( ${nuclearPass(baseCommit)} ) || true` +} + +export async function runDeepReset(workspaceRoot: string, baseCommit: string): Promise { + if (!baseCommit) return + const cmd = buildDeepResetCmd(baseCommit) + console.log(`[bench] deep_reset workspace=${workspaceRoot} base=${baseCommit}`) + await new Promise((resolve) => { + const child = spawn("bash", ["-c", cmd], { + cwd: workspaceRoot, + stdio: ["ignore", "inherit", "inherit"], + }) + child.on("close", (code) => { + console.log(`[bench] deep_reset exit=${code ?? 0}`) + resolve() + }) + child.on("error", (err) => { + console.warn(`[bench] deep_reset spawn error: ${err}`) + resolve() + }) + }) +} From 1b8998ad7d0c205286f7341f1de219381090afd3 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sun, 10 May 2026 18:30:57 -0700 Subject: [PATCH 12/49] bench: add progress echoes to deep_reset careful + nuclear passes Mark each phase + each ref delete/reset with a `[deep_reset:careful]` or `[deep_reset:nuclear]` line so the agent log shows what happened. Mirrors the per-step `echo` markers in nv-OpenHands' run_infer.py deep-reset path. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/deep_reset.ts | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/bench/deep_reset.ts b/packages/opencode/src/bench/deep_reset.ts index 89a9c9d1834a..c6dd07cda5d1 100644 --- a/packages/opencode/src/bench/deep_reset.ts +++ b/packages/opencode/src/bench/deep_reset.ts @@ -19,50 +19,65 @@ import { spawn } from "node:child_process" function carefulPass(baseCommit: string): string { return ( + `echo "[deep_reset:careful] start" && ` + `BASE=$(git rev-parse --verify ${baseCommit}^{commit}) && ` + `ORIG_BRANCH=$(git symbolic-ref --short -q HEAD || echo main) && ` + + `echo "[deep_reset:careful] base=$BASE orig_branch=$ORIG_BRANCH" && ` + `git checkout --detach "$BASE" && ` + + `echo "[deep_reset:careful] resetting local branches descending from base..." && ` + `git for-each-ref --format="%(refname)" refs/heads | while read -r ref; do ` + ` tip=$(git rev-parse -q --verify "$ref^{commit}" 2>/dev/null || true); ` + ` [ -z "$tip" ] && continue; ` + ` if [ "$tip" != "$BASE" ] && git merge-base --is-ancestor "$BASE" "$tip"; then ` + + ` echo "[deep_reset:careful] reset $ref -> $BASE"; ` + ` git update-ref "$ref" "$BASE"; ` + ` fi; ` + `done && ` + + `echo "[deep_reset:careful] deleting tags/remotes/stash/notes past base..." && ` + `git for-each-ref --format="%(refname)" refs | while read -r ref; do ` + ` case "$ref" in refs/heads/*) continue ;; esac; ` + ` if git symbolic-ref -q "$ref" >/dev/null 2>&1; then continue; fi; ` + ` tip=$(git rev-parse -q --verify "$ref^{commit}" 2>/dev/null || true); ` + ` [ -z "$tip" ] && continue; ` + ` if [ "$tip" != "$BASE" ] && git merge-base --is-ancestor "$BASE" "$tip"; then ` + + ` echo "[deep_reset:careful] delete $ref"; ` + ` git update-ref -d "$ref"; ` + ` fi; ` + `done && ` + - `for r in $(git remote); do git remote remove "$r"; done; ` + + `echo "[deep_reset:careful] removing remotes + transient refs..." && ` + + `for r in $(git remote); do echo "[deep_reset:careful] rm remote $r"; git remote remove "$r"; done; ` + `gd=$(git rev-parse --git-dir) && ` + `rm -f "$gd"/FETCH_HEAD "$gd"/ORIG_HEAD "$gd"/MERGE_HEAD "$gd"/CHERRY_PICK_HEAD ` + `"$gd"/REVERT_HEAD "$gd"/BISECT_HEAD "$gd"/AUTO_MERGE && ` + + `echo "[deep_reset:careful] expiring reflog + gc..." && ` + `git reflog expire --expire=now --expire-unreachable=now --all && ` + `git repack -ad && git prune --expire=now && git gc --prune=now && ` + - `git checkout -B "$ORIG_BRANCH" "$BASE"` + `git checkout -B "$ORIG_BRANCH" "$BASE" && ` + + `echo "[deep_reset:careful] done; HEAD=$ORIG_BRANCH at $BASE"` ) } function nuclearPass(baseCommit: string): string { return ( + `echo "[deep_reset:nuclear] careful pass failed; running batch-delete fallback" && ` + `BASE=$(git rev-parse --verify ${baseCommit}^{commit}) && ` + `ORIG_BRANCH=$(git symbolic-ref --short -q HEAD || echo main) && ` + + `echo "[deep_reset:nuclear] base=$BASE orig_branch=$ORIG_BRANCH" && ` + `git checkout --detach "$BASE" && ` + - `for r in $(git remote); do git remote remove "$r"; done; ` + + `for r in $(git remote); do echo "[deep_reset:nuclear] rm remote $r"; git remote remove "$r"; done; ` + + `echo "[deep_reset:nuclear] batch-delete tags/remotes/stash/notes..." && ` + `git for-each-ref --format="delete %(refname)" refs/tags refs/remotes refs/stash refs/notes 2>/dev/null ` + `| git update-ref --stdin; ` + + `echo "[deep_reset:nuclear] batch-delete local branches..." && ` + `git for-each-ref --format="delete %(refname)" refs/heads | git update-ref --stdin; ` + `gd=$(git rev-parse --git-dir) && ` + `rm -f "$gd"/FETCH_HEAD "$gd"/ORIG_HEAD "$gd"/MERGE_HEAD "$gd"/CHERRY_PICK_HEAD ` + `"$gd"/REVERT_HEAD "$gd"/BISECT_HEAD "$gd"/AUTO_MERGE && ` + + `echo "[deep_reset:nuclear] expiring reflog + gc..." && ` + `git reflog expire --expire=now --expire-unreachable=now --all && ` + `git repack -ad && git prune --expire=now && git gc --prune=now && ` + - `git checkout -B "$ORIG_BRANCH" "$BASE"` + `git checkout -B "$ORIG_BRANCH" "$BASE" && ` + + `echo "[deep_reset:nuclear] done; HEAD=$ORIG_BRANCH at $BASE"` ) } From 9b3d676695dd2e8e6fb6a6f702ba424bf0e7e250 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Mon, 11 May 2026 10:27:17 -0700 Subject: [PATCH 13/49] bench: capture per-session trajectories + add --enable-subagents flag - language-model.ts: read sessionID/parentSessionID from `x-session-affinity` / `x-parent-session-id` request headers (set by opencode's session/llm.ts for non-opencode providers). Per-session turn counter prevents subagent dumps from clobbering the main session's. Filename includes sessionID so each session's per-turn JSONs sit side-by-side; payload now carries session_id, parent_session_id, and turn for downstream reconstruction of the agent tree. - cli.ts: new --enable-subagents flag; toggles `task: ` in the per-instance opencode.jsonc. Default off. - run_infer.sh: forwards ENABLE_SUBAGENTS env -> --enable-subagents flag. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- .../benchmarks/swe_bench/scripts/run_infer.sh | 3 ++ packages/opencode/src/bench/cli.ts | 17 +++++++- .../provider/sdk/nemo-gym/language-model.ts | 42 ++++++++++++++++--- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/evaluation/benchmarks/swe_bench/scripts/run_infer.sh b/evaluation/benchmarks/swe_bench/scripts/run_infer.sh index 8392a21b8b0e..2edf91b721cc 100755 --- a/evaluation/benchmarks/swe_bench/scripts/run_infer.sh +++ b/evaluation/benchmarks/swe_bench/scripts/run_infer.sh @@ -110,6 +110,9 @@ cmd=( if [ -n "$SYSTEM_PROMPT_PATH" ]; then cmd+=(--system-prompt "$SYSTEM_PROMPT_PATH") fi +if [ "${ENABLE_SUBAGENTS:-0}" = "1" ] || [ "${ENABLE_SUBAGENTS:-}" = "true" ]; then + cmd+=(--enable-subagents) +fi echo "Executing: ${cmd[*]}" exec "${cmd[@]}" diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index f3ca8995ad77..f86afd9c311b 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -39,10 +39,18 @@ interface CliArgs { /** Pre-rendered user message file (workspace_path baked in by gym). */ userMessageFile: string systemPromptPath?: string + /** Enable opencode's `task` tool (spawns subagent sessions). */ + enableSubagents: boolean } function parseArgs(argv: string[]): CliArgs { - const out: Partial = { maxTurns: 100, agentCls: "OpenCodeAgent", dataset: "", split: "test" } + const out: Partial = { + maxTurns: 100, + agentCls: "OpenCodeAgent", + dataset: "", + split: "test", + enableSubagents: false, + } for (let i = 0; i < argv.length; i++) { const a = argv[i] const next = () => argv[++i] @@ -80,6 +88,9 @@ function parseArgs(argv: string[]): CliArgs { case "--system-prompt": out.systemPromptPath = next() break + case "--enable-subagents": + out.enableSubagents = true + break default: if (a.startsWith("--")) throw new Error(`Unknown flag: ${a}`) } @@ -143,6 +154,7 @@ async function buildConfigDir(args: { completionsDir: string maxTurns: number systemPromptPath?: string + enableSubagents: boolean }): Promise<{ tmpRoot: string; configFile: string }> { const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), `bench-${args.instanceId}-`)) await fs.mkdir(tmpRoot, { recursive: true }) @@ -197,7 +209,7 @@ async function buildConfigDir(args: { apply_patch: true, webfetch: false, websearch: false, - task: false, + task: args.enableSubagents, skill: false, todowrite: true, }, @@ -346,6 +358,7 @@ async function main() { completionsDir, maxTurns: args.maxTurns, systemPromptPath: args.systemPromptPath, + enableSubagents: args.enableSubagents, }) const startedAt = Date.now() diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index fc940f986320..6800fc568798 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -135,14 +135,14 @@ export class NemoGymLanguageModel implements LanguageModelV3 { private readonly cfg: NemoGymLanguageModelConfig private cookies: Record = {} + // Per-session turn counter. opencode's session header is `x-session-affinity`; + // subagents spawned via the task tool get their own sessionID, so keeping + // a Map keeps their dump filenames from clobbering the main session's. + private readonly turnCounters: Map = new Map() constructor(modelId: string, cfg: NemoGymLanguageModelConfig) { this.modelId = modelId this.provider = cfg.provider - // Spread first, then coalesce — opencode's provider loader passes - // optional fields explicitly as `undefined`, and a default-then-spread - // pattern lets those undefineds overwrite the defaults. `??` only - // replaces null/undefined, preserving any real caller-supplied value. this.cfg = { ...cfg, requestTimeoutMs: cfg.requestTimeoutMs ?? 600_000, @@ -150,6 +150,25 @@ export class NemoGymLanguageModel implements LanguageModelV3 { } } + private _nextTurn(sessionID: string): number { + const n = (this.turnCounters.get(sessionID) ?? -1) + 1 + this.turnCounters.set(sessionID, n) + return n + } + + private _sessionFromHeaders(headers: unknown): { sessionID: string; parentSessionID: string | undefined } { + let sid = "" + let pid: string | undefined + if (headers && typeof headers === "object") { + const h = headers as Record + const v = h["x-session-affinity"] ?? h["X-Session-Affinity"] + if (typeof v === "string") sid = v + const p = h["x-parent-session-id"] ?? h["X-Parent-Session-Id"] + if (typeof p === "string") pid = p + } + return { sessionID: sid || "main", parentSessionID: pid } + } + get supportedUrls() { return {} as Record } @@ -158,6 +177,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // implement doGenerate for completeness / future direct-use. async doGenerate(options: LanguageModelV3CallOptions) { const { warnings, messages, requestParams } = await this._buildRequestParams(options) + const session = this._sessionFromHeaders(options.headers) const { responseJson } = await this._postChat(requestParams) const choice = responseJson.choices[0] @@ -186,6 +206,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { response: responseJson, providerSpecificFields, requestParams, + session, }) return { @@ -201,6 +222,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { async doStream(options: LanguageModelV3CallOptions) { const { warnings, messages, requestParams } = await this._buildRequestParams(options) + const session = this._sessionFromHeaders(options.headers) // Fire the HTTP call eagerly so any error surfaces synchronously when the // stream is consumed. We then synthesize parts in `start`. @@ -274,6 +296,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { response: responseJson, providerSpecificFields, requestParams, + session, }) controller.enqueue({ @@ -494,8 +517,9 @@ export class NemoGymLanguageModel implements LanguageModelV3 { response: ChatResponse providerSpecificFields: Record requestParams: Record + session: { sessionID: string; parentSessionID: string | undefined } }) { - const turn = this.cfg.turnCounter ? this.cfg.turnCounter.next() : Date.now() + const turn = this._nextTurn(args.session.sessionID) if (this.cfg.onCompletion) { try { await this.cfg.onCompletion({ turn, ...args }) @@ -510,7 +534,10 @@ export class NemoGymLanguageModel implements LanguageModelV3 { await fs.mkdir(this.cfg.completionsDir, { recursive: true }) const turnStr = String(turn).padStart(4, "0") const safeModel = this.modelId.replace(/\//g, "__") - const fname = `${safeModel}-${turnStr}-${Date.now()}.json` + // sessionID is part of the filename so subagent dumps don't clobber the + // main session's. Sanitized for filesystem safety. + const safeSession = args.session.sessionID.replace(/[^A-Za-z0-9_-]/g, "_") + const fname = `${safeModel}-${safeSession}-${turnStr}-${Date.now()}.json` const fpath = path.join(this.cfg.completionsDir, fname) const kwargs: Record = {} for (const [k, v] of Object.entries(args.requestParams)) { @@ -521,6 +548,9 @@ export class NemoGymLanguageModel implements LanguageModelV3 { response: args.response, provider_specific_fields: args.providerSpecificFields, kwargs, + session_id: args.session.sessionID, + parent_session_id: args.session.parentSessionID ?? null, + turn, timestamp: Date.now() / 1000, } const tmp = `${fpath}.tmp` From 6495092f9b636c15917f653a4306fe645b3898a9 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Mon, 11 May 2026 13:53:20 -0700 Subject: [PATCH 14/49] bench: use opencode's anthropic.txt as default system prompt Replaces the 12-line bench-local DEFAULT_SYSTEM_PROMPT with the real anthropic system prompt opencode ships (`session/prompt/anthropic.txt`, ~105 lines), plus a short SWE-bench addendum (don't commit/format the diff, harness captures git diff as the patch, don't edit tests). `--system-prompt ` override still wins when supplied. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/cli.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index f86afd9c311b..aa569b3a221f 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -24,6 +24,9 @@ import path from "node:path" import os from "node:os" import { spawn } from "node:child_process" import { runDeepReset } from "./deep_reset" +// opencode's built-in anthropic system prompt — Bun bundles .txt as a string. +// Used as the default when no --system-prompt override is passed. +import PROMPT_ANTHROPIC from "../session/prompt/anthropic.txt" interface CliArgs { instanceDictPath: string @@ -135,18 +138,18 @@ function loadGymConfig(configPath: string): Record { return JSON.parse(readFileSync(configPath, "utf8")) } -const DEFAULT_SYSTEM_PROMPT = `You are an autonomous software engineer fixing a known issue in a checked-out git repository. +// Default is opencode's built-in anthropic system prompt + a short SWE-bench +// addendum (workspace is git-tracked, harness captures git diff as the patch, +// don't commit/format the diff yourself, don't modify the test files). +const SWE_BENCH_ADDENDUM = ` -Work in small, deliberate steps: -1. Read the issue and explore the relevant files. -2. Reproduce the issue if applicable. -3. Edit the source to fix the issue. -4. Run the project's tests to verify the fix. -5. Iterate until the issue is resolved. +# SWE-bench harness context -Use the available tools (bash, edit, read, glob, grep) to investigate and act. Do NOT modify the test files unless the task explicitly says so. The harness will capture the final \`git diff\` of the workspace as your patch — do not commit or format the diff yourself. +You are running inside a SWE-bench evaluation harness on a checked-out git repository. The harness will capture the final \`git diff\` of the workspace as your patch — do not commit, push, or format the diff yourself. Do NOT modify the test files unless the task explicitly says so. Stop calling tools once you are confident the issue is fully resolved. ` +const DEFAULT_SYSTEM_PROMPT = PROMPT_ANTHROPIC + SWE_BENCH_ADDENDUM + async function buildConfigDir(args: { instanceId: string modelName: string From d850c8fd717494b155221c3caa49ab33505262f2 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Mon, 11 May 2026 14:45:16 -0700 Subject: [PATCH 15/49] bench: gate dynamic system env block behind OPENCODE_DISABLE_ENV_PROMPT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The env block in session/system.ts (cwd, worktree, platform, today's date) is appended to the system prompt every turn. The `Today's date` field uses `new Date().toDateString()` — across a midnight rollover the system message shifts, which breaks the RL prompt-token-prefix invariant (turn[N+1].prompt_token_ids must extend turn[N]'s). Bench mode sets OPENCODE_DISABLE_ENV_PROMPT=1 in the child env so the environment() effect short-circuits to []. Normal `opencode run` is unaffected. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/cli.ts | 4 ++++ packages/opencode/src/session/system.ts | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index aa569b3a221f..3635bd1e6336 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -373,6 +373,10 @@ async function main() { OPENCODE_CONFIG: configFile, // Disable opencode's built-in plugin loaders; the bench harness doesn't need them. OPENCODE_PURE: "1", + // Skip the dynamic env block (working dir + Today's date) in the system + // prompt — keeps the RL prompt-token prefix invariant stable across turns + // (a midnight rollover would otherwise shift `Today's date: ...`). + OPENCODE_DISABLE_ENV_PROMPT: "1", } // Prune git history past base_commit so the agent can't reach future commits. diff --git a/packages/opencode/src/session/system.ts b/packages/opencode/src/session/system.ts index 06c71fa7dbdd..acde90b448d7 100644 --- a/packages/opencode/src/session/system.ts +++ b/packages/opencode/src/session/system.ts @@ -46,6 +46,11 @@ export const layer = Layer.effect( return Service.of({ environment: Effect.fn("SystemPrompt.environment")(function* (model: Provider.Model) { + // Bench / RL mode: skip the dynamic env block entirely. It includes + // `new Date().toDateString()` which would shift prompt tokens across a + // midnight rollover and break the RL contiguity invariant + // (prompt_token_ids[N+1] must extend prompt_token_ids[N]). + if (process.env.OPENCODE_DISABLE_ENV_PROMPT === "1") return [] const ctx = yield* InstanceState.context return [ [ From 01f0889a96cbb63c17ee54e70a52f7063864830b Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Mon, 11 May 2026 18:02:29 -0700 Subject: [PATCH 16/49] bench: spawn shell/bun/git via absolute paths to dodge SIF PATH quirks Some apptainer images ENOENT on bare program names through Bun's posix_spawn (libuv falls back to PATH-less execv on some kernels / musl builds). Symptoms in the wild: [bench] deep_reset spawn error: ENOENT posix_spawn 'bash' [bench] wrote ... error=opencode_exit_999 # bun spawn also ENOENT Fixes: - deep_reset.ts: probe /bin/bash, /usr/bin/bash, /bin/sh, /usr/bin/sh and spawn the first that exists. The script is POSIX-compatible so /bin/sh works fine when bash is absent (minimal/distroless SIFs). Skip deep_reset with a warning if no shell is found. - cli.ts runOpencode: use process.execPath (the currently-running bun binary) instead of bare "bun". - cli.ts captureGitDiff: probe /usr/bin/git, /bin/git, /usr/local/bin/git with bare "git" as last-ditch fallback. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/cli.ts | 18 ++++++++++++++++-- packages/opencode/src/bench/deep_reset.ts | 22 ++++++++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 3635bd1e6336..759bc753638b 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -230,6 +230,16 @@ async function buildConfigDir(args: { return { tmpRoot, configFile } } +// Some SIFs ship with bare PATH lookups that ENOENT on bare program names +// through Bun's posix_spawn. Resolve to an absolute path up front for any +// binary we shell out to. +function detectBin(candidates: string[]): string | null { + for (const p of candidates) { + if (existsSync(p)) return p + } + return null +} + function runOpencode(args: { workspaceRoot: string modelName: string @@ -238,9 +248,12 @@ function runOpencode(args: { opencodeBin: string agent: string }): Promise<{ exitCode: number; stdout: string; stderr: string }> { + // Use the same bun binary that's currently running — guaranteed to exist + // and avoids PATH lookup quirks under Bun's posix_spawn. + const bunPath = process.execPath return new Promise((resolve) => { const child = spawn( - "bun", + bunPath, [ args.opencodeBin, "run", @@ -283,8 +296,9 @@ function runOpencode(args: { } async function captureGitDiff(workspaceRoot: string): Promise { + const gitPath = detectBin(["/usr/bin/git", "/bin/git", "/usr/local/bin/git"]) ?? "git" return new Promise((resolve) => { - const child = spawn("git", ["-C", workspaceRoot, "diff"], { + const child = spawn(gitPath, ["-C", workspaceRoot, "diff"], { env: { ...process.env, GIT_PAGER: "cat" }, }) let stdout = "" diff --git a/packages/opencode/src/bench/deep_reset.ts b/packages/opencode/src/bench/deep_reset.ts index c6dd07cda5d1..77167e1530c6 100644 --- a/packages/opencode/src/bench/deep_reset.ts +++ b/packages/opencode/src/bench/deep_reset.ts @@ -16,6 +16,19 @@ */ import { spawn } from "node:child_process" +import { existsSync } from "node:fs" + +// Some SIFs are minimal and ship without `bash` on PATH, or Bun's posix_spawn +// doesn't fall back to PATH lookup the way `execvp` does — either way, +// spawn("bash", ...) ENOENTs. Probe absolute paths up front; the deep-reset +// script uses only POSIX features, so /bin/sh is a safe fallback if bash is +// absent. +function detectShell(): string | null { + for (const p of ["/bin/bash", "/usr/bin/bash", "/bin/sh", "/usr/bin/sh"]) { + if (existsSync(p)) return p + } + return null +} function carefulPass(baseCommit: string): string { return ( @@ -87,10 +100,15 @@ export function buildDeepResetCmd(baseCommit: string): string { export async function runDeepReset(workspaceRoot: string, baseCommit: string): Promise { if (!baseCommit) return + const shell = detectShell() + if (!shell) { + console.warn(`[bench] deep_reset skipped: no shell found at /bin/{bash,sh} or /usr/bin/{bash,sh}`) + return + } const cmd = buildDeepResetCmd(baseCommit) - console.log(`[bench] deep_reset workspace=${workspaceRoot} base=${baseCommit}`) + console.log(`[bench] deep_reset workspace=${workspaceRoot} base=${baseCommit} shell=${shell}`) await new Promise((resolve) => { - const child = spawn("bash", ["-c", cmd], { + const child = spawn(shell, ["-c", cmd], { cwd: workspaceRoot, stdio: ["ignore", "inherit", "inherit"], }) From cdfc4b76aa08939cf581b578e5c0308808b80140 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Mon, 11 May 2026 18:09:51 -0700 Subject: [PATCH 17/49] bench: drop spawn cwd, route chdir via shell / opencode --dir The previous fix moved to absolute binary paths but `/bin/bash` still ENOENTs through Bun's posix_spawn whenever the spawn passes a `cwd` option on minimal apptainer images (e.g., SWE-bench's astropy SIF). Root cause: libc lacks `posix_spawn_file_actions_addchdir_np`, so libuv's cwd-handling fallback fails the spawn outright instead of falling back to fork+chdir+exec. Workaround: don't pass `cwd` to spawn at all. - deep_reset.ts: prepend `cd && ` to the shell script itself (shellQuote handles paths with spaces / special chars). - cli.ts runOpencode: drop `cwd: workspaceRoot` from the bun spawn. Opencode's existing `--dir ` flag changes its own working directory, so spawn-level cwd was redundant anyway. - captureGitDiff already uses `git -C ` (no cwd). Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/cli.ts | 5 ++++- packages/opencode/src/bench/deep_reset.ts | 11 +++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 759bc753638b..8c9643b61b96 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -252,6 +252,10 @@ function runOpencode(args: { // and avoids PATH lookup quirks under Bun's posix_spawn. const bunPath = process.execPath return new Promise((resolve) => { + // Don't set spawn's `cwd` — Bun's posix_spawn on some minimal apptainer + // images ENOENTs whenever cwd is set (libc lacks addchdir_np). Opencode's + // own `--dir ` flag changes the working directory + // internally, so we don't need spawn-level cwd. const child = spawn( bunPath, [ @@ -269,7 +273,6 @@ function runOpencode(args: { args.workspaceRoot, ], { - cwd: args.workspaceRoot, env: args.env, stdio: ["ignore", "pipe", "pipe"], }, diff --git a/packages/opencode/src/bench/deep_reset.ts b/packages/opencode/src/bench/deep_reset.ts index 77167e1530c6..543815f454a5 100644 --- a/packages/opencode/src/bench/deep_reset.ts +++ b/packages/opencode/src/bench/deep_reset.ts @@ -98,6 +98,10 @@ export function buildDeepResetCmd(baseCommit: string): string { return `( ${carefulPass(baseCommit)} ) || ( ${nuclearPass(baseCommit)} ) || true` } +function shellQuote(s: string): string { + return `'${s.replace(/'/g, `'\\''`)}'` +} + export async function runDeepReset(workspaceRoot: string, baseCommit: string): Promise { if (!baseCommit) return const shell = detectShell() @@ -105,11 +109,14 @@ export async function runDeepReset(workspaceRoot: string, baseCommit: string): P console.warn(`[bench] deep_reset skipped: no shell found at /bin/{bash,sh} or /usr/bin/{bash,sh}`) return } - const cmd = buildDeepResetCmd(baseCommit) + // Bake `cd ` into the shell script instead of passing the `cwd` + // option to spawn(). On some minimal apptainer images Bun's posix_spawn + // ENOENTs whenever a `cwd` is set (libc lacks addchdir_np extension); routing + // the chdir through the shell sidesteps that entirely. + const cmd = `cd ${shellQuote(workspaceRoot)} && ` + buildDeepResetCmd(baseCommit) console.log(`[bench] deep_reset workspace=${workspaceRoot} base=${baseCommit} shell=${shell}`) await new Promise((resolve) => { const child = spawn(shell, ["-c", cmd], { - cwd: workspaceRoot, stdio: ["ignore", "inherit", "inherit"], }) child.on("close", (code) => { From b9602b100aefee5d8ba03bfdd57858e3955bc1a3 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Mon, 18 May 2026 14:17:35 -0700 Subject: [PATCH 18/49] feat: change nudge to user message --- packages/opencode/src/session/prompt.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index fef8c438366c..2e53938e04e9 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -1581,7 +1581,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the sessionID, parentSessionID: session.parentID, system, - messages: [...modelMsgs, ...(isLastStep ? [{ role: "assistant" as const, content: MAX_STEPS }] : [])], + messages: [...modelMsgs, ...(isLastStep ? [{ role: "user" as const, content: MAX_STEPS }] : [])], tools, model, toolChoice: format.type === "json_schema" ? "required" : undefined, From 52efa897091fcfa50dd56aa9d13930b7240a9e08 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Thu, 4 Jun 2026 12:57:07 -0700 Subject: [PATCH 19/49] bench: include untracked files in captured git diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plain `git -C diff` only reports changes to tracked files. Every new file the agent creates via the `write` tool stays untracked, so it was silently dropped from the captured patch and the rollout was recorded as `patch_exists=false` / reward 0 even though the agent ran to completion (`reason: "stop"`, bench `exit=0`). In a recent SWE-bench batch (swebench_results_1780596228320_bcd81ba7), 8 of the 21 zero-byte patches were exactly this case — 7/8 had `write` calls to brand-new paths and never staged. Example trajectory: denoland-deno-8408-agentic wrote a 6499-byte std/encoding/csv_stringify.ts from scratch, type-checked, ran a functional test, and stopped. git_patch came back "" because the file was untracked. Fix: mark untracked files as intent-to-add (`git add -AN`) so they appear in `git diff` without being committed. Worktree-style diff is preserved so the SWE-bench evaluator's `git apply` still works. Also pass `--binary` so any binary artifacts the agent produces aren't silently truncated by the textual diff path. The system-prompt addendum already tells the model not to commit or push, which is still correct after this fix — the harness now picks up unstaged new files on its own. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/cli.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 8c9643b61b96..d0dc383dcdd4 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -300,15 +300,21 @@ function runOpencode(args: { async function captureGitDiff(workspaceRoot: string): Promise { const gitPath = detectBin(["/usr/bin/git", "/bin/git", "/usr/local/bin/git"]) ?? "git" - return new Promise((resolve) => { - const child = spawn(gitPath, ["-C", workspaceRoot, "diff"], { - env: { ...process.env, GIT_PAGER: "cat" }, + const runGit = (args: string[], capture: boolean): Promise => + new Promise((resolve) => { + const child = spawn(gitPath, ["-C", workspaceRoot, ...args], { + env: { ...process.env, GIT_PAGER: "cat" }, + }) + let stdout = "" + if (capture) child.stdout?.on("data", (b) => (stdout += b.toString("utf8"))) + child.on("close", () => resolve(stdout)) + child.on("error", () => resolve("")) }) - let stdout = "" - child.stdout?.on("data", (b) => (stdout += b.toString("utf8"))) - child.on("close", () => resolve(stdout)) - child.on("error", () => resolve("")) - }) + // Mark untracked files as intent-to-add so newly-created files appear in + // `git diff` without being committed. Plain `git diff` only shows changes + // to tracked files, which silently drops new-file patches the agent wrote. + await runGit(["add", "-AN"], false) + return runGit(["diff", "--binary"], true) } interface OutputJsonl { From a8f6fc98d6196c998d3a03c3d96a6877e4c6c61e Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Wed, 10 Jun 2026 13:56:38 -0700 Subject: [PATCH 20/49] bench: bootstrap a git repo when the SIF ships no .git MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some dataset SIFs (notably `swe-bench-ext`, and certain SWE-rebench variants) copy the repo into the workspace as a flat source tree with no `.git` directory. Without one: * runDeepReset's first `git rev-parse` fails, the outer `|| true` swallows the error, and deep_reset is a silent no-op. * captureGitDiff runs `git -C diff` against a non-repo; git emits `fatal: not a git repository` to stderr and exits with empty stdout, so the function returns "". Net effect: every rollout for those SIFs lands as `patch=0 bytes` regardless of how many files the agent edited. Recent symptom — in swebench_results_1781123430109_686771d1 (swe-bench-ext, 37 instances), every patch was zero bytes despite trajectories with up to 30 successful `edit` calls on the workspace. `cd /workspace/repo && git status` in the same SIF returns `fatal: not a git repository`. Port of nv-OpenHands' run_infer.py:1142-1156 swe-bench-ext baseline: detect the missing `.git`, init a local repo, snapshot the pristine tree as `opencode_bench_baseline`, and skip deep_reset (the upstream base_commit SHA doesn't exist in the fresh repo, so deep_reset would just fall through to its nuclear pass and noisily no-op anyway). This composes with 52efa8970 (`git add -AN && git diff --binary`) — the prior commit handled `.git`-present-but-untracked-write-files; this commit handles `.git`-missing-entirely. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/bootstrap_repo.ts | 78 +++++++++++++++++++ packages/opencode/src/bench/cli.ts | 14 +++- 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 packages/opencode/src/bench/bootstrap_repo.ts diff --git a/packages/opencode/src/bench/bootstrap_repo.ts b/packages/opencode/src/bench/bootstrap_repo.ts new file mode 100644 index 000000000000..82c9c61a2353 --- /dev/null +++ b/packages/opencode/src/bench/bootstrap_repo.ts @@ -0,0 +1,78 @@ +/** + * Bootstrap a git repository inside the workspace when the SIF ships a flat + * source tree without a `.git` directory. + * + * Some dataset SIFs (notably `swe-bench-ext`, and certain SWE-rebench variants) + * copy the repo contents into `/workspace/repo` (or the dataset-specific path) + * without preserving git history. Without `.git`, `runDeepReset` is a silent + * no-op (its `git rev-parse` fails under the outer `|| true`) and + * `captureGitDiff` returns "" — every rollout is recorded as `patch=0 bytes` + * regardless of what the agent did. Port of nv-OpenHands' + * `evaluation/benchmarks/swe_bench/run_infer.py:1142-1156`. + * + * If `.git` already exists, this is a no-op. Otherwise a pristine baseline + * commit is created and tagged `opencode_bench_baseline`. Callers should skip + * `runDeepReset` when this returns `{ freshInit: true }` — the dataset's + * upstream `base_commit` SHA does not exist in the fresh repo, so deep_reset + * would just fail rev-parse and noisily fall through to its nuclear pass. + */ + +import { spawn } from "node:child_process" +import { existsSync } from "node:fs" +import path from "node:path" + +function detectShell(): string | null { + for (const p of ["/bin/bash", "/usr/bin/bash", "/bin/sh", "/usr/bin/sh"]) { + if (existsSync(p)) return p + } + return null +} + +function shellQuote(s: string): string { + return `'${s.replace(/'/g, `'\\''`)}'` +} + +function buildBootstrapCmd(workspaceRoot: string): string { + const q = shellQuote(workspaceRoot) + return ( + `cd ${q} && ` + + `echo "[bootstrap_repo] initializing git repo at ${workspaceRoot}" && ` + + `git config --global --add safe.directory ${q} && ` + + `git init -q && ` + + `git config user.email 'bench@opencode.local' && ` + + `git config user.name 'opencode bench' && ` + + `git add -A && ` + + `git commit -q --allow-empty -m 'opencode bench baseline' && ` + + `git tag -f opencode_bench_baseline HEAD && ` + + `echo "[bootstrap_repo] done; HEAD=$(git rev-parse --short HEAD)"` + ) +} + +export interface BootstrapResult { + freshInit: boolean +} + +export async function bootstrapRepoIfMissing(workspaceRoot: string): Promise { + if (existsSync(path.join(workspaceRoot, ".git"))) { + return { freshInit: false } + } + const shell = detectShell() + if (!shell) { + console.warn(`[bench] bootstrap_repo skipped: no shell found at /bin/{bash,sh} or /usr/bin/{bash,sh}`) + return { freshInit: false } + } + const cmd = buildBootstrapCmd(workspaceRoot) + console.log(`[bench] bootstrap_repo workspace=${workspaceRoot} shell=${shell}`) + const exitCode = await new Promise((resolve) => { + const child = spawn(shell, ["-c", cmd], { + stdio: ["ignore", "inherit", "inherit"], + }) + child.on("close", (code) => resolve(code ?? 0)) + child.on("error", (err) => { + console.warn(`[bench] bootstrap_repo spawn error: ${err}`) + resolve(1) + }) + }) + console.log(`[bench] bootstrap_repo exit=${exitCode}`) + return { freshInit: exitCode === 0 } +} diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index d0dc383dcdd4..e4ba969d7dde 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -24,6 +24,7 @@ import path from "node:path" import os from "node:os" import { spawn } from "node:child_process" import { runDeepReset } from "./deep_reset" +import { bootstrapRepoIfMissing } from "./bootstrap_repo" // opencode's built-in anthropic system prompt — Bun bundles .txt as a string. // Used as the default when no --system-prompt override is passed. import PROMPT_ANTHROPIC from "../session/prompt/anthropic.txt" @@ -402,8 +403,19 @@ async function main() { OPENCODE_DISABLE_ENV_PROMPT: "1", } + // Bootstrap a git repo if the SIF shipped a flat source tree (swe-bench-ext + // and some SWE-rebench variants). Without this, captureGitDiff returns "" + // and every patch is recorded as 0 bytes. + const { freshInit } = await bootstrapRepoIfMissing(workspaceRoot) + // Prune git history past base_commit so the agent can't reach future commits. - await runDeepReset(workspaceRoot, String(instance.base_commit ?? "")) + // Skip when we just freshly initialized: the dataset's upstream base_commit + // SHA doesn't exist in our local repo, so deep_reset would just fail + // rev-parse and fall through to its nuclear pass. The fresh `HEAD` is + // already the correct baseline (also tagged `opencode_bench_baseline`). + if (!freshInit) { + await runDeepReset(workspaceRoot, String(instance.base_commit ?? "")) + } const opencodeBin = detectOpencodeBin() const result = await runOpencode({ From 0c088fd18b5ef6ff9b2a949ac2a41204b1ac8046 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Wed, 10 Jun 2026 14:58:35 -0700 Subject: [PATCH 21/49] bench: exit 0 deterministically when bench wrote output.jsonl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Falling off the end of main() with no explicit process.exit() left Bun to drain the event loop on its own, which produced a flaky exit=1 even on successful runs that wrote a valid output.jsonl with `error=none`. The residual handles seem to come from the spawned opencode subprocess (piped stdio that hasn't been explicitly closed) and from opencode's sqlite migration code — both stay registered long enough that Bun treats the runtime drain as a non-clean exit. Symptom in DeepSeek-V4-Flash_opencode_0-0-12711152.out (SWE-rebench-V2 train shard): three instances logged [bench] wrote .../bench_run/output.jsonl (patch=N bytes, error=none) with N=2001, 9046, ... — `error=none` means `result.exitCode === 0`, so opencode itself exited cleanly and the patch was captured. But apptainer exited 1, gym's runner raised `RuntimeError("Command failed with return code 1")`, and the rollout was discarded as a failure even though the bench succeeded. Fix: explicitly `process.exit(result.exitCode === 0 ? 0 : 1)` after the [bench] wrote log line. Deterministic, mirrors opencode's exit code, and doesn't depend on the event-loop drain heuristic. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/cli.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index e4ba969d7dde..707c98c12a2a 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -444,7 +444,13 @@ async function main() { console.log(`[bench] wrote ${outPath} (patch=${patch.length} bytes, error=${error ?? "none"})`) - if (result.exitCode !== 0) process.exit(1) + // Mirror opencode's exit code explicitly. Falling off the end of main() and + // letting Bun drain the event loop produced a flaky exit=1 even when the + // bench wrote output.jsonl cleanly (sqlite migration handles, residual + // child-stdio pipes from the opencode subprocess). Gym's runner treats any + // non-zero apptainer exit as `Agent command failed` and discards the + // already-written patch, so we MUST exit 0 deterministically on success. + process.exit(result.exitCode === 0 ? 0 : 1) } main().catch((err) => { From 4b3796bb682b892bbebd3de7aea7db8b3333b039 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Tue, 30 Jun 2026 10:39:17 -0700 Subject: [PATCH 22/49] bench: drop SWE-bench addendum from default system prompt Use opencode's built-in anthropic system prompt as-is, without appending the SWE-bench harness context block. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/cli.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 707c98c12a2a..bed80a87b304 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -139,17 +139,7 @@ function loadGymConfig(configPath: string): Record { return JSON.parse(readFileSync(configPath, "utf8")) } -// Default is opencode's built-in anthropic system prompt + a short SWE-bench -// addendum (workspace is git-tracked, harness captures git diff as the patch, -// don't commit/format the diff yourself, don't modify the test files). -const SWE_BENCH_ADDENDUM = ` - -# SWE-bench harness context - -You are running inside a SWE-bench evaluation harness on a checked-out git repository. The harness will capture the final \`git diff\` of the workspace as your patch — do not commit, push, or format the diff yourself. Do NOT modify the test files unless the task explicitly says so. Stop calling tools once you are confident the issue is fully resolved. -` - -const DEFAULT_SYSTEM_PROMPT = PROMPT_ANTHROPIC + SWE_BENCH_ADDENDUM +const DEFAULT_SYSTEM_PROMPT = PROMPT_ANTHROPIC async function buildConfigDir(args: { instanceId: string From 0db3864b8ccf0121ee2f5c09c34ade7e9502519e Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Tue, 30 Jun 2026 14:03:58 -0700 Subject: [PATCH 23/49] bench: bump nemo-gym provider retries/timeout to 6 / 1200s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A transient streak of model-server timeouts otherwise tears down the entire agent session and produces an empty model_patch even after the agent has done meaningful exploration. Bumping retries (3 → 6) and per-call timeout (600s → 1200s) lets long-horizon DeNovoSWE runs survive the model server's occasional slow responses, recovering many "empty patch" failures that were not real agent give-ups. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/cli.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index bed80a87b304..22c6699cd329 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -166,6 +166,12 @@ async function buildConfigDir(args: { baseURL: args.baseURL, completionsDir: args.completionsDir, instanceId: args.instanceId, + // Bumped from provider defaults (3 retries / 600s) — a transient + // model-server timeout streak otherwise tears down the whole + // session and produces an empty model_patch even after the agent + // has done meaningful exploration. + retries: 6, + requestTimeoutMs: 1_200_000, }, models: { [args.modelName]: { From 0a37db46081993f987009c85a57d8359e2f283b4 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Tue, 30 Jun 2026 18:25:37 -0700 Subject: [PATCH 24/49] bench: remove nemo-gym provider retry/timeout caps Under high shard concurrency the vLLM fleet can stall a single request for tens of minutes (KV cache thrashing, request preemption). The finite retries=6 / requestTimeoutMs=1_200_000 setting still ran out of runway and tore down the whole session with an empty model_patch after ~30 min of useful work. Wait instead: retries=MAX_SAFE_INTEGER, requestTimeoutMs=0 (sentinel: no abort timer), backoff capped at 60s so an unlimited-retry loop can't stall on huge exponential sleeps. Signed-off-by: Sugam Devare --- packages/opencode/src/bench/cli.ts | 12 ++++++------ .../src/provider/sdk/nemo-gym/language-model.ts | 12 ++++++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 22c6699cd329..1cd7c4ba7911 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -166,12 +166,12 @@ async function buildConfigDir(args: { baseURL: args.baseURL, completionsDir: args.completionsDir, instanceId: args.instanceId, - // Bumped from provider defaults (3 retries / 600s) — a transient - // model-server timeout streak otherwise tears down the whole - // session and produces an empty model_patch even after the agent - // has done meaningful exploration. - retries: 6, - requestTimeoutMs: 1_200_000, + // Unlimited: model-server backpressure can stall a request for + // tens of minutes at high shard concurrency; we'd rather wait + // than tear down the session and produce an empty model_patch. + // requestTimeoutMs<=0 disables the abort timer in the provider. + retries: Number.MAX_SAFE_INTEGER, + requestTimeoutMs: 0, }, models: { [args.modelName]: { diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 6800fc568798..32b1e47802ee 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -421,10 +421,12 @@ export class NemoGymLanguageModel implements LanguageModelV3 { } const retries = this.cfg.retries ?? 3 + const timeoutMs = this.cfg.requestTimeoutMs ?? 0 let lastErr: unknown = null for (let attempt = 0; attempt < retries; attempt++) { const ac = new AbortController() - const timer = setTimeout(() => ac.abort(), this.cfg.requestTimeoutMs) + // timeoutMs<=0 means "no timeout" — don't install the abort timer. + const timer = timeoutMs > 0 ? setTimeout(() => ac.abort(), timeoutMs) : null try { const res = await fetch(url, { method: "POST", @@ -432,7 +434,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { body: JSON.stringify(params), signal: ac.signal, }) - clearTimeout(timer) + if (timer) clearTimeout(timer) if (!res.ok) { const text = await res.text().catch(() => "") throw new Error(`NeMoGym ${url} ${res.status}: ${text.slice(0, 500)}`) @@ -448,10 +450,12 @@ export class NemoGymLanguageModel implements LanguageModelV3 { const responseJson = (await res.json()) as ChatResponse return { responseJson } } catch (err) { - clearTimeout(timer) + if (timer) clearTimeout(timer) lastErr = err if (attempt === retries - 1) break - await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt)) + // Cap exponential backoff at 60s so unlimited-retries configs don't blow up the delay. + const backoffMs = Math.min(1000 * 2 ** attempt, 60_000) + await new Promise((r) => setTimeout(r, backoffMs)) } } throw new Error(`NeMoGym chat completions failed after ${retries} attempts: ${String(lastErr)}`) From 8e47cccfc31531dd3f25a3441faa0744fbef43f9 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Thu, 2 Jul 2026 13:25:25 -0700 Subject: [PATCH 25/49] bench: rewrite deep_reset to match nv-OpenHands simple in-place Port the improved _deep_reset_to_base_commit from nv-OpenHands. Same bash verbatim: git reset + clean, checkout BASE on the original branch, batch-delete refs/tags refs/remotes refs/stash refs/notes refs/replace refs/prefetch refs/pull with `option no-deref` (fixes transaction abort on symbolic refs like refs/remotes/NAME/HEAD), kill .git/packed-refs and transient op refs, expire reflog, `git repack -ad` + prune, set identity, verify HEAD == BASE with a __DEEP_RESET_OK__ sentinel. Behaviour changes vs the previous two-pass (careful + nuclear) port: - No per-ref merge-base --is-ancestor iteration (which timed out on monorepos with thousands of refs). - `-ad` instead of `-Ad`: on facebook/react (~200k unreachable objects) drops runtime from ~10 min to ~4 s by skipping the demote-to-loose intermediate. - Adds git clean -fd (removes stray untracked files) and git config user.email/name (so agent commits work when the container has no global identity). - Best-effort: failure logs `[bench] REBUILD_FAILED base_commit=... exit_code=...` and returns so the agent still runs; grep task logs to filter these instances from clean-run analysis. Preserves the SIF/apptainer machinery (detectShell/shellQuote/baked cwd). Validated end-to-end via `bun run` -> runDeepReset -> /bin/bash on 10 real repos (5.7 MB click to 1.6 GB kubernetes): all pass with HEAD == BASE, refs = 1, reflog empty, post-base cat-file returns "Not a valid object name". Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/deep_reset.ts | 157 +++++++++++----------- 1 file changed, 78 insertions(+), 79 deletions(-) diff --git a/packages/opencode/src/bench/deep_reset.ts b/packages/opencode/src/bench/deep_reset.ts index 543815f454a5..c4a8e78f7bb0 100644 --- a/packages/opencode/src/bench/deep_reset.ts +++ b/packages/opencode/src/bench/deep_reset.ts @@ -1,18 +1,32 @@ /** - * Strip git history past base_commit so the agent can't reach future commits. + * Pin HEAD at base_commit and drop every ref/reflog entry/object that reaches + * commits past base_commit. * - * Port of nv-OpenHands' `_deep_reset_to_base_commit` - * (evaluation/benchmarks/swe_bench/run_infer.py:774). Two-pass design: + * Ported from the improved nv-OpenHands version in + * `evaluation/benchmarks/swe_bench/run_infer.py` (`_deep_reset_to_base_commit`). + * See its docstring for design rationale. Summary: * - * - Careful pass: per-ref iteration with `git for-each-ref`. Preserves - * local branches that don't descend from base, resets branches that do, - * deletes tags/remote-tracking/stash/notes refs past base. - * - Nuclear fallback: batch-delete every tag/remote/stash/notes ref + every - * local branch in two `git update-ref --stdin` calls. Microseconds - * regardless of ref count — handles monorepos with thousands of refs - * where the careful pass times out. + * - In-place cleanup (no bundle, no .git swap). Each step is best-effort; + * only the final HEAD-at-BASE check decides success. + * - Batch-deletes every ref namespace that can reach post-base commits + * (`refs/tags`, `refs/remotes`, `refs/stash`, `refs/notes`, + * `refs/replace`, `refs/prefetch`, `refs/pull`) plus local branches + * other than the current one. Uses "option no-deref" per entry so + * symbolic refs (refs/remotes/NAME/HEAD) don't abort the transaction. + * - `git repack -ad` (lowercase -a) to drop unreachable objects from the + * pack in one step — avoids the demote-to-loose intermediate that made + * `-Ad` take ~10 minutes on repos like facebook/react. + * - Sets `user.email`/`user.name` and runs `git clean -fd` so downstream + * agent commands work even on containers with no global git identity + * or with leftover untracked files from a previous run. + * - On failure, logs `[bench] REBUILD_FAILED` and returns anyway so the + * agent still runs (grep task logs for `REBUILD_FAILED` to filter these + * instances from clean-run analysis). * - * `|| true` at the very end so a busted git state can't kill the agent run. + * Measured on 20 real repos ranging from 5.7 MB (pallets/click) to 997 MB + * (facebook/react): all pass with HEAD=BASE, refs=1, reflog=0, post-base + * `cat-file` returns "Not a valid object name". Slowest: cpython at 12.5s. + * facebook/react: 4.1s (previous `-Ad`-based version: 662s). */ import { spawn } from "node:child_process" @@ -30,76 +44,46 @@ function detectShell(): string | null { return null } -function carefulPass(baseCommit: string): string { - return ( - `echo "[deep_reset:careful] start" && ` + - `BASE=$(git rev-parse --verify ${baseCommit}^{commit}) && ` + - `ORIG_BRANCH=$(git symbolic-ref --short -q HEAD || echo main) && ` + - `echo "[deep_reset:careful] base=$BASE orig_branch=$ORIG_BRANCH" && ` + - `git checkout --detach "$BASE" && ` + - `echo "[deep_reset:careful] resetting local branches descending from base..." && ` + - `git for-each-ref --format="%(refname)" refs/heads | while read -r ref; do ` + - ` tip=$(git rev-parse -q --verify "$ref^{commit}" 2>/dev/null || true); ` + - ` [ -z "$tip" ] && continue; ` + - ` if [ "$tip" != "$BASE" ] && git merge-base --is-ancestor "$BASE" "$tip"; then ` + - ` echo "[deep_reset:careful] reset $ref -> $BASE"; ` + - ` git update-ref "$ref" "$BASE"; ` + - ` fi; ` + - `done && ` + - `echo "[deep_reset:careful] deleting tags/remotes/stash/notes past base..." && ` + - `git for-each-ref --format="%(refname)" refs | while read -r ref; do ` + - ` case "$ref" in refs/heads/*) continue ;; esac; ` + - ` if git symbolic-ref -q "$ref" >/dev/null 2>&1; then continue; fi; ` + - ` tip=$(git rev-parse -q --verify "$ref^{commit}" 2>/dev/null || true); ` + - ` [ -z "$tip" ] && continue; ` + - ` if [ "$tip" != "$BASE" ] && git merge-base --is-ancestor "$BASE" "$tip"; then ` + - ` echo "[deep_reset:careful] delete $ref"; ` + - ` git update-ref -d "$ref"; ` + - ` fi; ` + - `done && ` + - `echo "[deep_reset:careful] removing remotes + transient refs..." && ` + - `for r in $(git remote); do echo "[deep_reset:careful] rm remote $r"; git remote remove "$r"; done; ` + - `gd=$(git rev-parse --git-dir) && ` + - `rm -f "$gd"/FETCH_HEAD "$gd"/ORIG_HEAD "$gd"/MERGE_HEAD "$gd"/CHERRY_PICK_HEAD ` + - `"$gd"/REVERT_HEAD "$gd"/BISECT_HEAD "$gd"/AUTO_MERGE && ` + - `echo "[deep_reset:careful] expiring reflog + gc..." && ` + - `git reflog expire --expire=now --expire-unreachable=now --all && ` + - `git repack -ad && git prune --expire=now && git gc --prune=now && ` + - `git checkout -B "$ORIG_BRANCH" "$BASE" && ` + - `echo "[deep_reset:careful] done; HEAD=$ORIG_BRANCH at $BASE"` - ) -} - -function nuclearPass(baseCommit: string): string { - return ( - `echo "[deep_reset:nuclear] careful pass failed; running batch-delete fallback" && ` + - `BASE=$(git rev-parse --verify ${baseCommit}^{commit}) && ` + - `ORIG_BRANCH=$(git symbolic-ref --short -q HEAD || echo main) && ` + - `echo "[deep_reset:nuclear] base=$BASE orig_branch=$ORIG_BRANCH" && ` + - `git checkout --detach "$BASE" && ` + - `for r in $(git remote); do echo "[deep_reset:nuclear] rm remote $r"; git remote remove "$r"; done; ` + - `echo "[deep_reset:nuclear] batch-delete tags/remotes/stash/notes..." && ` + - `git for-each-ref --format="delete %(refname)" refs/tags refs/remotes refs/stash refs/notes 2>/dev/null ` + - `| git update-ref --stdin; ` + - `echo "[deep_reset:nuclear] batch-delete local branches..." && ` + - `git for-each-ref --format="delete %(refname)" refs/heads | git update-ref --stdin; ` + - `gd=$(git rev-parse --git-dir) && ` + - `rm -f "$gd"/FETCH_HEAD "$gd"/ORIG_HEAD "$gd"/MERGE_HEAD "$gd"/CHERRY_PICK_HEAD ` + - `"$gd"/REVERT_HEAD "$gd"/BISECT_HEAD "$gd"/AUTO_MERGE && ` + - `echo "[deep_reset:nuclear] expiring reflog + gc..." && ` + - `git reflog expire --expire=now --expire-unreachable=now --all && ` + - `git repack -ad && git prune --expire=now && git gc --prune=now && ` + - `git checkout -B "$ORIG_BRANCH" "$BASE" && ` + - `echo "[deep_reset:nuclear] done; HEAD=$ORIG_BRANCH at $BASE"` - ) +function shellQuote(s: string): string { + return `'${s.replace(/'/g, `'\\''`)}'` } +/** + * The bash body that does the actual reset. `set +e` at the top — + * intermediate step failures don't abort; only the terminal HEAD-check + * controls the exit code. + * + * Note on template-literal escaping: this string contains bash `$VAR` and + * `$(...)` references. Only `${...}` would be interpolated by JS — none of + * those exist in the shell body, so the `$` characters pass through verbatim. + * The single `${baseCommit}` interpolation is the caller-supplied SHA. + */ export function buildDeepResetCmd(baseCommit: string): string { - return `( ${carefulPass(baseCommit)} ) || ( ${nuclearPass(baseCommit)} ) || true` -} - -function shellQuote(s: string): string { - return `'${s.replace(/'/g, `'\\''`)}'` + return `set +e +BASE=$(git rev-parse --verify ${baseCommit}^{commit}) || { echo "[deep_reset] BAD_BASE ${baseCommit}"; exit 1; } +BRANCH=$(git symbolic-ref --short -q HEAD || echo main) +echo "[deep_reset] BASE=$BASE BRANCH=$BRANCH" +git reset --hard --quiet +git clean -fd --quiet +git checkout "$BASE" -f --quiet +git checkout -B "$BRANCH" "$BASE" --quiet +git for-each-ref --format='option no-deref%0Adelete %(refname)' refs/tags refs/remotes refs/stash refs/notes refs/replace refs/prefetch refs/pull 2>/dev/null | git update-ref --stdin 2>/dev/null +git for-each-ref --format='%(refname)' refs/heads 2>/dev/null | grep -vFx "refs/heads/$BRANCH" | while read r; do git update-ref -d "$r" --no-deref 2>/dev/null; done +rm -f .git/packed-refs +rm -f .git/FETCH_HEAD .git/ORIG_HEAD .git/MERGE_HEAD .git/CHERRY_PICK_HEAD .git/REVERT_HEAD .git/BISECT_HEAD .git/AUTO_MERGE +git reflog expire --expire=now --expire-unreachable=now --all 2>/dev/null +git repack -ad --quiet 2>/dev/null +git prune --expire=now 2>/dev/null +git config user.email 'opencode-bench@localhost' 2>/dev/null +git config user.name 'opencode bench' 2>/dev/null +HEAD_SHA=$(git rev-parse --verify HEAD^{commit} 2>/dev/null) +if [ "$HEAD_SHA" = "$BASE" ]; then + echo "__DEEP_RESET_OK__ HEAD=$HEAD_SHA" + exit 0 +else + echo "__DEEP_RESET_MISMATCH__ HEAD=$HEAD_SHA BASE=$BASE" + exit 1 +fi` } export async function runDeepReset(workspaceRoot: string, baseCommit: string): Promise { @@ -120,7 +104,22 @@ export async function runDeepReset(workspaceRoot: string, baseCommit: string): P stdio: ["ignore", "inherit", "inherit"], }) child.on("close", (code) => { - console.log(`[bench] deep_reset exit=${code ?? 0}`) + const rc = code ?? 0 + if (rc === 0) { + console.log(`[bench] deep_reset OK`) + } else { + // Best-effort: log LOUDLY so downstream analysis can grep for + // REBUILD_FAILED and filter these instances from clean-run + // aggregates, but do not throw — running the task on a + // partially-reset workspace is more useful than failing outright. + console.warn( + `[bench] REBUILD_FAILED base_commit=${baseCommit} exit_code=${rc}. ` + + `The workspace .git may be partially reset and may still ` + + `reference commits past base_commit. The agent will proceed ` + + `but may be able to inspect the solution via git. Filter this ` + + `task from clean-run analysis.`, + ) + } resolve() }) child.on("error", (err) => { From 46388623b058377a76a0d3ca301ccd928ef9905e Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Thu, 2 Jul 2026 13:26:40 -0700 Subject: [PATCH 26/49] bench: cap deep_reset at 10 min via spawn timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `timeout: 600_000, killSignal: "SIGKILL"` to the spawn options so a hung git command is force-killed instead of blocking the eval indefinitely. Matches the Python port's 10-min budget. Real repos (5.7 MB click → 1.6 GB kubernetes) finish well under 20s in our measurements. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/deep_reset.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/opencode/src/bench/deep_reset.ts b/packages/opencode/src/bench/deep_reset.ts index c4a8e78f7bb0..f9249842a277 100644 --- a/packages/opencode/src/bench/deep_reset.ts +++ b/packages/opencode/src/bench/deep_reset.ts @@ -100,8 +100,12 @@ export async function runDeepReset(workspaceRoot: string, baseCommit: string): P const cmd = `cd ${shellQuote(workspaceRoot)} && ` + buildDeepResetCmd(baseCommit) console.log(`[bench] deep_reset workspace=${workspaceRoot} base=${baseCommit} shell=${shell}`) await new Promise((resolve) => { + // 10 min hard cap. In-place reset on real repos (5.7 MB → 3.5 GB) + // finishes well under 20s; anything past 10 min is a hang. const child = spawn(shell, ["-c", cmd], { stdio: ["ignore", "inherit", "inherit"], + timeout: 600_000, + killSignal: "SIGKILL", }) child.on("close", (code) => { const rc = code ?? 0 From 68c02b1783536f18bffc30a676fa002190b9abb2 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Thu, 2 Jul 2026 14:41:12 -0700 Subject: [PATCH 27/49] revert: restore old two-pass deep_reset.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the in-place port from 8e47cccfc and its spawn-timeout follow-up from 46388623b. Training runs surfaced failures with the new version — restoring the pre-change (careful + nuclear) implementation from cdfc4b76a while diagnosis continues. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Sugam Devare --- packages/opencode/src/bench/deep_reset.ts | 161 +++++++++++----------- 1 file changed, 79 insertions(+), 82 deletions(-) diff --git a/packages/opencode/src/bench/deep_reset.ts b/packages/opencode/src/bench/deep_reset.ts index f9249842a277..543815f454a5 100644 --- a/packages/opencode/src/bench/deep_reset.ts +++ b/packages/opencode/src/bench/deep_reset.ts @@ -1,32 +1,18 @@ /** - * Pin HEAD at base_commit and drop every ref/reflog entry/object that reaches - * commits past base_commit. + * Strip git history past base_commit so the agent can't reach future commits. * - * Ported from the improved nv-OpenHands version in - * `evaluation/benchmarks/swe_bench/run_infer.py` (`_deep_reset_to_base_commit`). - * See its docstring for design rationale. Summary: + * Port of nv-OpenHands' `_deep_reset_to_base_commit` + * (evaluation/benchmarks/swe_bench/run_infer.py:774). Two-pass design: * - * - In-place cleanup (no bundle, no .git swap). Each step is best-effort; - * only the final HEAD-at-BASE check decides success. - * - Batch-deletes every ref namespace that can reach post-base commits - * (`refs/tags`, `refs/remotes`, `refs/stash`, `refs/notes`, - * `refs/replace`, `refs/prefetch`, `refs/pull`) plus local branches - * other than the current one. Uses "option no-deref" per entry so - * symbolic refs (refs/remotes/NAME/HEAD) don't abort the transaction. - * - `git repack -ad` (lowercase -a) to drop unreachable objects from the - * pack in one step — avoids the demote-to-loose intermediate that made - * `-Ad` take ~10 minutes on repos like facebook/react. - * - Sets `user.email`/`user.name` and runs `git clean -fd` so downstream - * agent commands work even on containers with no global git identity - * or with leftover untracked files from a previous run. - * - On failure, logs `[bench] REBUILD_FAILED` and returns anyway so the - * agent still runs (grep task logs for `REBUILD_FAILED` to filter these - * instances from clean-run analysis). + * - Careful pass: per-ref iteration with `git for-each-ref`. Preserves + * local branches that don't descend from base, resets branches that do, + * deletes tags/remote-tracking/stash/notes refs past base. + * - Nuclear fallback: batch-delete every tag/remote/stash/notes ref + every + * local branch in two `git update-ref --stdin` calls. Microseconds + * regardless of ref count — handles monorepos with thousands of refs + * where the careful pass times out. * - * Measured on 20 real repos ranging from 5.7 MB (pallets/click) to 997 MB - * (facebook/react): all pass with HEAD=BASE, refs=1, reflog=0, post-base - * `cat-file` returns "Not a valid object name". Slowest: cpython at 12.5s. - * facebook/react: 4.1s (previous `-Ad`-based version: 662s). + * `|| true` at the very end so a busted git state can't kill the agent run. */ import { spawn } from "node:child_process" @@ -44,46 +30,76 @@ function detectShell(): string | null { return null } -function shellQuote(s: string): string { - return `'${s.replace(/'/g, `'\\''`)}'` +function carefulPass(baseCommit: string): string { + return ( + `echo "[deep_reset:careful] start" && ` + + `BASE=$(git rev-parse --verify ${baseCommit}^{commit}) && ` + + `ORIG_BRANCH=$(git symbolic-ref --short -q HEAD || echo main) && ` + + `echo "[deep_reset:careful] base=$BASE orig_branch=$ORIG_BRANCH" && ` + + `git checkout --detach "$BASE" && ` + + `echo "[deep_reset:careful] resetting local branches descending from base..." && ` + + `git for-each-ref --format="%(refname)" refs/heads | while read -r ref; do ` + + ` tip=$(git rev-parse -q --verify "$ref^{commit}" 2>/dev/null || true); ` + + ` [ -z "$tip" ] && continue; ` + + ` if [ "$tip" != "$BASE" ] && git merge-base --is-ancestor "$BASE" "$tip"; then ` + + ` echo "[deep_reset:careful] reset $ref -> $BASE"; ` + + ` git update-ref "$ref" "$BASE"; ` + + ` fi; ` + + `done && ` + + `echo "[deep_reset:careful] deleting tags/remotes/stash/notes past base..." && ` + + `git for-each-ref --format="%(refname)" refs | while read -r ref; do ` + + ` case "$ref" in refs/heads/*) continue ;; esac; ` + + ` if git symbolic-ref -q "$ref" >/dev/null 2>&1; then continue; fi; ` + + ` tip=$(git rev-parse -q --verify "$ref^{commit}" 2>/dev/null || true); ` + + ` [ -z "$tip" ] && continue; ` + + ` if [ "$tip" != "$BASE" ] && git merge-base --is-ancestor "$BASE" "$tip"; then ` + + ` echo "[deep_reset:careful] delete $ref"; ` + + ` git update-ref -d "$ref"; ` + + ` fi; ` + + `done && ` + + `echo "[deep_reset:careful] removing remotes + transient refs..." && ` + + `for r in $(git remote); do echo "[deep_reset:careful] rm remote $r"; git remote remove "$r"; done; ` + + `gd=$(git rev-parse --git-dir) && ` + + `rm -f "$gd"/FETCH_HEAD "$gd"/ORIG_HEAD "$gd"/MERGE_HEAD "$gd"/CHERRY_PICK_HEAD ` + + `"$gd"/REVERT_HEAD "$gd"/BISECT_HEAD "$gd"/AUTO_MERGE && ` + + `echo "[deep_reset:careful] expiring reflog + gc..." && ` + + `git reflog expire --expire=now --expire-unreachable=now --all && ` + + `git repack -ad && git prune --expire=now && git gc --prune=now && ` + + `git checkout -B "$ORIG_BRANCH" "$BASE" && ` + + `echo "[deep_reset:careful] done; HEAD=$ORIG_BRANCH at $BASE"` + ) +} + +function nuclearPass(baseCommit: string): string { + return ( + `echo "[deep_reset:nuclear] careful pass failed; running batch-delete fallback" && ` + + `BASE=$(git rev-parse --verify ${baseCommit}^{commit}) && ` + + `ORIG_BRANCH=$(git symbolic-ref --short -q HEAD || echo main) && ` + + `echo "[deep_reset:nuclear] base=$BASE orig_branch=$ORIG_BRANCH" && ` + + `git checkout --detach "$BASE" && ` + + `for r in $(git remote); do echo "[deep_reset:nuclear] rm remote $r"; git remote remove "$r"; done; ` + + `echo "[deep_reset:nuclear] batch-delete tags/remotes/stash/notes..." && ` + + `git for-each-ref --format="delete %(refname)" refs/tags refs/remotes refs/stash refs/notes 2>/dev/null ` + + `| git update-ref --stdin; ` + + `echo "[deep_reset:nuclear] batch-delete local branches..." && ` + + `git for-each-ref --format="delete %(refname)" refs/heads | git update-ref --stdin; ` + + `gd=$(git rev-parse --git-dir) && ` + + `rm -f "$gd"/FETCH_HEAD "$gd"/ORIG_HEAD "$gd"/MERGE_HEAD "$gd"/CHERRY_PICK_HEAD ` + + `"$gd"/REVERT_HEAD "$gd"/BISECT_HEAD "$gd"/AUTO_MERGE && ` + + `echo "[deep_reset:nuclear] expiring reflog + gc..." && ` + + `git reflog expire --expire=now --expire-unreachable=now --all && ` + + `git repack -ad && git prune --expire=now && git gc --prune=now && ` + + `git checkout -B "$ORIG_BRANCH" "$BASE" && ` + + `echo "[deep_reset:nuclear] done; HEAD=$ORIG_BRANCH at $BASE"` + ) } -/** - * The bash body that does the actual reset. `set +e` at the top — - * intermediate step failures don't abort; only the terminal HEAD-check - * controls the exit code. - * - * Note on template-literal escaping: this string contains bash `$VAR` and - * `$(...)` references. Only `${...}` would be interpolated by JS — none of - * those exist in the shell body, so the `$` characters pass through verbatim. - * The single `${baseCommit}` interpolation is the caller-supplied SHA. - */ export function buildDeepResetCmd(baseCommit: string): string { - return `set +e -BASE=$(git rev-parse --verify ${baseCommit}^{commit}) || { echo "[deep_reset] BAD_BASE ${baseCommit}"; exit 1; } -BRANCH=$(git symbolic-ref --short -q HEAD || echo main) -echo "[deep_reset] BASE=$BASE BRANCH=$BRANCH" -git reset --hard --quiet -git clean -fd --quiet -git checkout "$BASE" -f --quiet -git checkout -B "$BRANCH" "$BASE" --quiet -git for-each-ref --format='option no-deref%0Adelete %(refname)' refs/tags refs/remotes refs/stash refs/notes refs/replace refs/prefetch refs/pull 2>/dev/null | git update-ref --stdin 2>/dev/null -git for-each-ref --format='%(refname)' refs/heads 2>/dev/null | grep -vFx "refs/heads/$BRANCH" | while read r; do git update-ref -d "$r" --no-deref 2>/dev/null; done -rm -f .git/packed-refs -rm -f .git/FETCH_HEAD .git/ORIG_HEAD .git/MERGE_HEAD .git/CHERRY_PICK_HEAD .git/REVERT_HEAD .git/BISECT_HEAD .git/AUTO_MERGE -git reflog expire --expire=now --expire-unreachable=now --all 2>/dev/null -git repack -ad --quiet 2>/dev/null -git prune --expire=now 2>/dev/null -git config user.email 'opencode-bench@localhost' 2>/dev/null -git config user.name 'opencode bench' 2>/dev/null -HEAD_SHA=$(git rev-parse --verify HEAD^{commit} 2>/dev/null) -if [ "$HEAD_SHA" = "$BASE" ]; then - echo "__DEEP_RESET_OK__ HEAD=$HEAD_SHA" - exit 0 -else - echo "__DEEP_RESET_MISMATCH__ HEAD=$HEAD_SHA BASE=$BASE" - exit 1 -fi` + return `( ${carefulPass(baseCommit)} ) || ( ${nuclearPass(baseCommit)} ) || true` +} + +function shellQuote(s: string): string { + return `'${s.replace(/'/g, `'\\''`)}'` } export async function runDeepReset(workspaceRoot: string, baseCommit: string): Promise { @@ -100,30 +116,11 @@ export async function runDeepReset(workspaceRoot: string, baseCommit: string): P const cmd = `cd ${shellQuote(workspaceRoot)} && ` + buildDeepResetCmd(baseCommit) console.log(`[bench] deep_reset workspace=${workspaceRoot} base=${baseCommit} shell=${shell}`) await new Promise((resolve) => { - // 10 min hard cap. In-place reset on real repos (5.7 MB → 3.5 GB) - // finishes well under 20s; anything past 10 min is a hang. const child = spawn(shell, ["-c", cmd], { stdio: ["ignore", "inherit", "inherit"], - timeout: 600_000, - killSignal: "SIGKILL", }) child.on("close", (code) => { - const rc = code ?? 0 - if (rc === 0) { - console.log(`[bench] deep_reset OK`) - } else { - // Best-effort: log LOUDLY so downstream analysis can grep for - // REBUILD_FAILED and filter these instances from clean-run - // aggregates, but do not throw — running the task on a - // partially-reset workspace is more useful than failing outright. - console.warn( - `[bench] REBUILD_FAILED base_commit=${baseCommit} exit_code=${rc}. ` + - `The workspace .git may be partially reset and may still ` + - `reference commits past base_commit. The agent will proceed ` + - `but may be able to inspect the solution via git. Filter this ` + - `task from clean-run analysis.`, - ) - } + console.log(`[bench] deep_reset exit=${code ?? 0}`) resolve() }) child.on("error", (err) => { From 7fbf78532c3a4f32dc30a695c019513d911c0674 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sat, 4 Jul 2026 12:01:45 -0700 Subject: [PATCH 28/49] bench: force on-policy temperature/top_p from gym config NeMo-RL's async vLLM worker asserts every /v1/chat/completions request carries exactly the training generation config's temperature and top_p (on-policy requirement). The gym now passes them via the llm.model config block; cli.ts forwards them as provider options and the nemo-gym provider forces them on every request (all sessions, including subagents), overriding any session/agent-level sampling choice. Co-Authored-By: Claude Fable 5 --- packages/opencode/src/bench/cli.ts | 13 +++++++++++++ .../opencode/src/provider/sdk/nemo-gym/index.ts | 9 +++++++++ .../src/provider/sdk/nemo-gym/language-model.ts | 15 +++++++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 1cd7c4ba7911..0838f48569b5 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -149,6 +149,9 @@ async function buildConfigDir(args: { maxTurns: number systemPromptPath?: string enableSubagents: boolean + /** Forced sampling params (RL on-policy requirement); from gym llm.model config. */ + temperature?: number + topP?: number }): Promise<{ tmpRoot: string; configFile: string }> { const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), `bench-${args.instanceId}-`)) await fs.mkdir(tmpRoot, { recursive: true }) @@ -172,6 +175,11 @@ async function buildConfigDir(args: { // requestTimeoutMs<=0 disables the abort timer in the provider. retries: Number.MAX_SAFE_INTEGER, requestTimeoutMs: 0, + // Forced sampling params: NeMo-RL's vLLM worker asserts every + // request's temperature/top_p match the training generation config + // exactly (on-policy). Passed by gym via the llm.model config block. + ...(args.temperature !== undefined ? { temperature: args.temperature } : {}), + ...(args.topP !== undefined ? { topP: args.topP } : {}), }, models: { [args.modelName]: { @@ -364,6 +372,9 @@ async function main() { unknown > const modelName = String(llmModelCfg.model ?? "unknown-model") + // Forced sampling params from gym (RL training on-policy requirement). + const forcedTemperature = typeof llmModelCfg.temperature === "number" ? llmModelCfg.temperature : undefined + const forcedTopP = typeof llmModelCfg.top_p === "number" ? llmModelCfg.top_p : undefined const baseURL = process.env.NEMO_GYM_MODEL_SERVER_BASE_URL if (!baseURL) throw new Error("NEMO_GYM_MODEL_SERVER_BASE_URL not set in env (gym harness sets this).") @@ -382,6 +393,8 @@ async function main() { maxTurns: args.maxTurns, systemPromptPath: args.systemPromptPath, enableSubagents: args.enableSubagents, + temperature: forcedTemperature, + topP: forcedTopP, }) const startedAt = Date.now() diff --git a/packages/opencode/src/provider/sdk/nemo-gym/index.ts b/packages/opencode/src/provider/sdk/nemo-gym/index.ts index 0d7001b8dc28..244f8399057a 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/index.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/index.ts @@ -29,6 +29,13 @@ export interface CreateNemoGymOptions { requestTimeoutMs?: number /** HTTP retry count on transient errors. */ retries?: number + /** + * Forced sampling params for RL training (on-policy requirement): when set, + * every request carries exactly these values regardless of what the + * session/agent layer picked. Wired from the gym config's llm.model block. + */ + temperature?: number + topP?: number /** Optional turn counter shared across all model calls in a session. */ turnCounter?: { next(): number } /** Optional callback invoked after each successful chat-completion. */ @@ -54,6 +61,8 @@ export function createNemoGym(opts: CreateNemoGymOptions): NemoGymProvider { instanceId: opts.instanceId, requestTimeoutMs: opts.requestTimeoutMs, retries: opts.retries, + temperature: opts.temperature, + topP: opts.topP, turnCounter: opts.turnCounter, onCompletion: opts.onCompletion, }) diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 32b1e47802ee..99a9d0a356d0 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -104,6 +104,15 @@ export interface NemoGymLanguageModelConfig { requestTimeoutMs?: number /** Number of HTTP retry attempts on transient errors. */ retries?: number + /** + * Forced sampling params for RL training. NeMo-RL's vLLM worker asserts + * that every request's temperature/top_p exactly match the training + * generation config (on-policy requirement) — when set, these override + * whatever the session/agent layer picked, for ALL sessions including + * subagents. + */ + temperature?: number + topP?: number /** * Where per-call llm_completions JSONs land. The bench harness builds this * path; it must match what gym's host-side glob expects. If unset, no @@ -368,8 +377,10 @@ export class NemoGymLanguageModel implements LanguageModelV3 { const requestParams: Record = { messages, max_tokens: options.maxOutputTokens, - temperature: options.temperature, - top_p: options.topP, + // Forced training params (cfg) win over session/agent-level choices: + // NeMo-RL asserts exact temperature/top_p equality on every request. + temperature: this.cfg.temperature ?? options.temperature, + top_p: this.cfg.topP ?? options.topP, stop: options.stopSequences, seed: options.seed, } From c41d867d32a15b742bcf6bb6fd6b7dda2072622c Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sat, 4 Jul 2026 13:54:50 -0700 Subject: [PATCH 29/49] bench: omit max_tokens by default (unlimited output) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit opencode's session layer caps every request at OUTPUT_TOKEN_MAX (32k), which both truncated long turns and shrank the usable input window (vLLM rejects input+max_tokens > context, so 196k context effectively became 164k input). The nemo-gym provider now sends NO max_tokens unless the gym config forces one via llm.model.max_tokens — vLLM then generates up to the remaining context. Co-Authored-By: Claude Fable 5 --- packages/opencode/src/bench/cli.ts | 5 +++++ .../opencode/src/provider/sdk/nemo-gym/index.ts | 3 +++ .../src/provider/sdk/nemo-gym/language-model.ts | 13 ++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 0838f48569b5..c557f5849c58 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -152,6 +152,7 @@ async function buildConfigDir(args: { /** Forced sampling params (RL on-policy requirement); from gym llm.model config. */ temperature?: number topP?: number + maxTokens?: number }): Promise<{ tmpRoot: string; configFile: string }> { const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), `bench-${args.instanceId}-`)) await fs.mkdir(tmpRoot, { recursive: true }) @@ -180,6 +181,7 @@ async function buildConfigDir(args: { // exactly (on-policy). Passed by gym via the llm.model config block. ...(args.temperature !== undefined ? { temperature: args.temperature } : {}), ...(args.topP !== undefined ? { topP: args.topP } : {}), + ...(args.maxTokens !== undefined ? { maxTokens: args.maxTokens } : {}), }, models: { [args.modelName]: { @@ -375,6 +377,8 @@ async function main() { // Forced sampling params from gym (RL training on-policy requirement). const forcedTemperature = typeof llmModelCfg.temperature === "number" ? llmModelCfg.temperature : undefined const forcedTopP = typeof llmModelCfg.top_p === "number" ? llmModelCfg.top_p : undefined + // Optional: force a max_tokens cap; when absent, requests carry none (unlimited). + const forcedMaxTokens = typeof llmModelCfg.max_tokens === "number" ? llmModelCfg.max_tokens : undefined const baseURL = process.env.NEMO_GYM_MODEL_SERVER_BASE_URL if (!baseURL) throw new Error("NEMO_GYM_MODEL_SERVER_BASE_URL not set in env (gym harness sets this).") @@ -395,6 +399,7 @@ async function main() { enableSubagents: args.enableSubagents, temperature: forcedTemperature, topP: forcedTopP, + maxTokens: forcedMaxTokens, }) const startedAt = Date.now() diff --git a/packages/opencode/src/provider/sdk/nemo-gym/index.ts b/packages/opencode/src/provider/sdk/nemo-gym/index.ts index 244f8399057a..9f7460d1450e 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/index.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/index.ts @@ -36,6 +36,8 @@ export interface CreateNemoGymOptions { */ temperature?: number topP?: number + /** Optional forced max_tokens; unset = no cap (vLLM generates to remaining context). */ + maxTokens?: number /** Optional turn counter shared across all model calls in a session. */ turnCounter?: { next(): number } /** Optional callback invoked after each successful chat-completion. */ @@ -63,6 +65,7 @@ export function createNemoGym(opts: CreateNemoGymOptions): NemoGymProvider { retries: opts.retries, temperature: opts.temperature, topP: opts.topP, + maxTokens: opts.maxTokens, turnCounter: opts.turnCounter, onCompletion: opts.onCompletion, }) diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 99a9d0a356d0..59e48d62be8a 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -113,6 +113,12 @@ export interface NemoGymLanguageModelConfig { */ temperature?: number topP?: number + /** + * Optional forced max_tokens. When unset (the default), requests carry NO + * max_tokens and vLLM generates up to the remaining context — opencode's + * session-level output cap is deliberately ignored. + */ + maxTokens?: number /** * Where per-call llm_completions JSONs land. The bench harness builds this * path; it must match what gym's host-side glob expects. If unset, no @@ -376,7 +382,12 @@ export class NemoGymLanguageModel implements LanguageModelV3 { const requestParams: Record = { messages, - max_tokens: options.maxOutputTokens, + // max_tokens is intentionally OMITTED unless the gym config forces one: + // without it vLLM generates up to the remaining context + // (max_model_len - prompt), i.e. "unlimited" output. Sending opencode's + // session-level cap (OUTPUT_TOKEN_MAX=32k) both truncated long turns and + // shrank the usable input window (vLLM rejects input+max_tokens>context). + ...(this.cfg.maxTokens ? { max_tokens: this.cfg.maxTokens } : {}), // Forced training params (cfg) win over session/agent-level choices: // NeMo-RL asserts exact temperature/top_p equality on every request. temperature: this.cfg.temperature ?? options.temperature, From c5cb2febc897e140fc90c2ea49dbcb2b884c1384 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sat, 4 Jul 2026 14:26:31 -0700 Subject: [PATCH 30/49] bench: round-trip token IDs to the most recent assistant message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RL training requires the most recent assistant message on each request to carry prompt/generation token IDs (the vllm proxy verifies continuity and the final turn's prompt_token_ids embed the whole episode's exact token stream for NeMo-RL reconstruction — mirrors OpenHands' nemo_gym_client.py). opencode's session layer dropped these custom fields when rebuilding messages, so NO turn carried them: training silently reconstructed only the final generation, and episodes ending in an empty over-context completion had zero generation data, crashing the trajectory postprocess (group-worker retries -> step deadlock at buffer 11/32). Carrier: emit token IDs as part-level providerMetadata on text-end / reasoning-end (persisted as part.metadata, replayed as part.providerOptions['nemo-gym']), then re-attach them to the most recent assistant wire message in _buildRequestParams. Co-Authored-By: Claude Fable 5 --- .../provider/sdk/nemo-gym/language-model.ts | 54 +++++++++++++++---- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 59e48d62be8a..347ea6df835b 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -199,9 +199,13 @@ export class NemoGymLanguageModel implements LanguageModelV3 { if (!choice) throw new Error("nemo-gym: empty choices in response") const msg: ChatResponseChoice["message"] = choice.message ?? ({ role: "assistant" } as ChatResponseChoice["message"]) + const providerSpecificFields = this._extractProviderFields(msg) + const providerMetadata = this._buildProviderMetadata(providerSpecificFields) + const content: LanguageModelV3Content[] = [] - if (msg.content) content.push({ type: "text", text: msg.content }) - if (msg.reasoning_text) content.push({ type: "reasoning", text: msg.reasoning_text }) + // Part-level providerMetadata round-trips per-turn token IDs (see doStream). + if (msg.content) content.push({ type: "text", text: msg.content, providerMetadata }) + if (msg.reasoning_text) content.push({ type: "reasoning", text: msg.reasoning_text, providerMetadata }) if (msg.tool_calls) { for (const tc of msg.tool_calls) { content.push({ @@ -213,9 +217,6 @@ export class NemoGymLanguageModel implements LanguageModelV3 { } } - const providerSpecificFields = this._extractProviderFields(msg) - const providerMetadata = this._buildProviderMetadata(providerSpecificFields) - await this._dumpAndNotify({ messages, response: responseJson, @@ -266,18 +267,22 @@ export class NemoGymLanguageModel implements LanguageModelV3 { timestamp: responseJson.created ? new Date(responseJson.created * 1000) : undefined, }) - // Reasoning content. + // Reasoning content. providerMetadata on the *-end events is + // persisted by opencode's processor as part.metadata and replayed + // on the next request as part.providerOptions["nemo-gym"] — this is + // how per-turn token IDs round-trip so EVERY assistant turn (not + // just the last) carries them for RL training reconstruction. if (msg.reasoning_text) { controller.enqueue({ type: "reasoning-start", id: "reasoning-0" }) controller.enqueue({ type: "reasoning-delta", id: "reasoning-0", delta: msg.reasoning_text }) - controller.enqueue({ type: "reasoning-end", id: "reasoning-0" }) + controller.enqueue({ type: "reasoning-end", id: "reasoning-0", providerMetadata }) } // Text content. if (msg.content) { controller.enqueue({ type: "text-start", id: "txt-0" }) controller.enqueue({ type: "text-delta", id: "txt-0", delta: msg.content }) - controller.enqueue({ type: "text-end", id: "txt-0" }) + controller.enqueue({ type: "text-end", id: "txt-0", providerMetadata }) } // Tool calls. @@ -358,15 +363,42 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // tool-call / multi-content shapes map identically to the rest of opencode. const messages = convertToOpenAICompatibleChatMessages(options.prompt) as unknown as ChatRequestMessage[] + // Attach token IDs to the MOST RECENT assistant message only, mirroring + // OpenHands' nemo_gym_client.py: the last turn's prompt_token_ids embed + // the exact token stream of the whole conversation, which is (a) how the + // vllm proxy verifies continuity and avoids retokenization drift, and + // (b) all NeMo-RL needs to reconstruct the full episode for training. + // The IDs round-trip through opencode's part metadata: doStream emits + // providerMetadata on text-end / reasoning-end -> processor stores + // part.metadata -> replay attaches part.providerOptions["nemo-gym"]. + // (The generic converter drops the fields, so we restore them here.) + { + const promptAssistants = options.prompt.filter((m) => m.role === "assistant") + const wireAssistants = (messages as Array>).filter((m) => m["role"] === "assistant") + const n = Math.min(promptAssistants.length, wireAssistants.length) + outer: for (let i = n - 1; i >= 0; i--) { + const parts = promptAssistants[i].content + if (!Array.isArray(parts)) continue + for (const part of parts as Array<{ providerOptions?: Record> }>) { + const md = part.providerOptions?.[this.provider] + if (md && TOKEN_ID_FIELDS.every((f) => f in md)) { + for (const f of TOKEN_ID_FIELDS) wireAssistants[i][f] = md[f] + break outer + } + } + } + } + const { tools, toolChoice, toolWarnings } = prepareTools({ tools: options.tools, toolChoice: options.toolChoice, }) warnings.push(...toolWarnings) - // Strip token-ID fields from all assistant messages EXCEPT the most recent. - // Mirrors nemo_gym_client.py:85-97. Wire-payload dedup; the most recent - // message keeps its IDs so the server can verify continuity. + // Safeguard: strip token-ID fields from all assistant messages EXCEPT the + // most recent (mirrors nemo_gym_client.py:85-97). With the single-message + // attach above this is normally a no-op, but it keeps the wire contract + // if an upstream change ever attaches more. { let lastSeen = false for (let i = messages.length - 1; i >= 0; i--) { From 15db4d2a29c2ef8cd15a404743368cabba182b62 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sat, 4 Jul 2026 20:22:07 -0700 Subject: [PATCH 31/49] bench: scrub token-ID arrays from echoed event stream, bound stdout capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The token-ID round-trip metadata rides opencode's JSON event stream; the bench cli echoed it verbatim, so every turn re-emitted that turn's full-context prompt_token_ids — O(n^2) log growth (22GB driver log, multi-MB agent logs in minutes at 1024-way training concurrency), which also amplified Lustre quota exhaustion. Strip the arrays from echoed lines (the llm_completions dumps keep them) and cap the in-memory stdout/stderr tails at 256KB. Co-Authored-By: Claude Fable 5 --- packages/opencode/src/bench/cli.ts | 37 ++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index c557f5849c58..2b47ec452def 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -286,15 +286,42 @@ function runOpencode(args: { ) let stdout = "" let stderr = "" + // Strip bulky token-ID metadata from echoed event lines. The IDs already + // live in the llm_completions dumps; leaving them in the event stream + // makes each turn re-echo that turn's full-context prompt_token_ids -> + // O(n^2) log growth (observed: 22GB driver logs, multi-MB agent logs + // within minutes at 1024-way training concurrency). + const TOKEN_FIELDS = ["prompt_token_ids", "generation_token_ids", "generation_log_probs"] + const scrub = (line: string): string => { + if (!(line.includes('"nemo-gym"') && line.includes('"prompt_token_ids"'))) return line + try { + const evt = JSON.parse(line) + const md = evt?.part?.metadata?.["nemo-gym"] + if (md) { + for (const k of TOKEN_FIELDS) { + if (Array.isArray(md[k])) md[k] = `<${md[k].length} stripped>` + } + return JSON.stringify(evt) + } + } catch {} + return line + } + const MAX_KEEP = 256 * 1024 // keep only a bounded tail for error reporting + let lineBuf = "" child.stdout?.on("data", (b) => { - const chunk = b.toString("utf8") - stdout += chunk - // Forward to our stdout so the gym log captures the event stream. - process.stdout.write(chunk) + lineBuf += b.toString("utf8") + let idx: number + while ((idx = lineBuf.indexOf("\n")) >= 0) { + const line = scrub(lineBuf.slice(0, idx)) + lineBuf = lineBuf.slice(idx + 1) + // Forward to our stdout so the gym log captures the event stream. + process.stdout.write(line + "\n") + stdout = (stdout + line + "\n").slice(-MAX_KEEP) + } }) child.stderr?.on("data", (b) => { const chunk = b.toString("utf8") - stderr += chunk + stderr = (stderr + chunk).slice(-MAX_KEEP) process.stderr.write(chunk) }) child.on("close", (code) => resolve({ exitCode: code ?? 0, stdout, stderr })) From 4fa94a2533270aa8f9d60d595a533ecd90ebeb77 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sun, 5 Jul 2026 10:32:06 -0700 Subject: [PATCH 32/49] bench: flatten multi-part message content to plain string for gym vllm proxy The AI-SDK converter emits an array of content parts for multi-part user turns (e.g. the synthetic 'Attached image(s) from tool result:' message injected when a tool result contains an image). The gym's NeMoGymEasyInputMessage validator only accepts plain-string content on those messages, so such requests 500'd and burned prompt-group retries. Flatten arrays to newline-joined text; stub non-text parts. --- .../src/provider/sdk/nemo-gym/language-model.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 347ea6df835b..207291b696f9 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -363,6 +363,23 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // tool-call / multi-content shapes map identically to the rest of opencode. const messages = convertToOpenAICompatibleChatMessages(options.prompt) as unknown as ChatRequestMessage[] + // The gym's vllm proxy (NeMoGymEasyInputMessage) only accepts plain-string + // content on non-assistant messages; the converter emits an ARRAY of parts + // for multi-part user turns (e.g. the synthetic "Attached image(s) from + // tool result:" message) which fails validation server-side with a 500. + // The policy model is text-only anyway, so flatten arrays to a single + // string and stub out non-text parts. + for (const m of messages as Array>) { + const content = m["content"] + if (Array.isArray(content)) { + m["content"] = (content as Array>) + .map((p) => + p?.["type"] === "text" ? String(p["text"] ?? "") : `[${String(p?.["type"] ?? "unknown")} part omitted]`, + ) + .join("\n") + } + } + // Attach token IDs to the MOST RECENT assistant message only, mirroring // OpenHands' nemo_gym_client.py: the last turn's prompt_token_ids embed // the exact token stream of the whole conversation, which is (a) how the From 306684253a1af9a5616d2e2e7eae070936028e13 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sun, 5 Jul 2026 10:42:41 -0700 Subject: [PATCH 33/49] bench: keep per-turn token IDs in the logged trajectory, last-turn-only on the wire Matches nemo_gym_client.py exactly: the wire request strips token-ID fields from all but the most recent assistant message, but the llm_completions/*.json dump keeps every assistant turn's prompt_token_ids / generation_token_ids / generation_log_probs. swe_agents app.py materializes the training episode from the logged messages, and NeMo-RL needs per-turn generation IDs/log-probs to build the loss mask over all model-generated spans, not just the final turn. Implemented by shallow-cloning the converted messages into a separate loggedMessages array before wire-side attach/strip; _dumpAndNotify now records the logged copy while requestParams keeps the wire copy. --- .../provider/sdk/nemo-gym/language-model.ts | 45 +++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 207291b696f9..0a0f9e56c386 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -191,7 +191,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // The streamText path in `session/llm.ts` only calls doStream. We still // implement doGenerate for completeness / future direct-use. async doGenerate(options: LanguageModelV3CallOptions) { - const { warnings, messages, requestParams } = await this._buildRequestParams(options) + const { warnings, loggedMessages, requestParams } = await this._buildRequestParams(options) const session = this._sessionFromHeaders(options.headers) const { responseJson } = await this._postChat(requestParams) @@ -218,7 +218,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { } await this._dumpAndNotify({ - messages, + messages: loggedMessages, response: responseJson, providerSpecificFields, requestParams, @@ -237,7 +237,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { } async doStream(options: LanguageModelV3CallOptions) { - const { warnings, messages, requestParams } = await this._buildRequestParams(options) + const { warnings, loggedMessages, requestParams } = await this._buildRequestParams(options) const session = this._sessionFromHeaders(options.headers) // Fire the HTTP call eagerly so any error surfaces synchronously when the @@ -312,7 +312,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // Persist trajectory BEFORE finishing so a downstream tool crash // cannot lose this turn's token IDs. await self._dumpAndNotify({ - messages, + messages: loggedMessages, response: responseJson, providerSpecificFields, requestParams, @@ -354,6 +354,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { private async _buildRequestParams(options: LanguageModelV3CallOptions): Promise<{ warnings: SharedV3Warning[] messages: ChatRequestMessage[] + loggedMessages: ChatRequestMessage[] tools: unknown toolChoice: unknown requestParams: Record @@ -380,27 +381,45 @@ export class NemoGymLanguageModel implements LanguageModelV3 { } } - // Attach token IDs to the MOST RECENT assistant message only, mirroring - // OpenHands' nemo_gym_client.py: the last turn's prompt_token_ids embed - // the exact token stream of the whole conversation, which is (a) how the - // vllm proxy verifies continuity and avoids retokenization drift, and - // (b) all NeMo-RL needs to reconstruct the full episode for training. + // Token-ID handling, mirroring OpenHands' nemo_gym_client.py exactly: + // - WIRE request: token IDs on the MOST RECENT assistant message only + // (the last turn's prompt_token_ids embed the exact token stream of + // the whole conversation; the vllm proxy uses it to verify continuity + // and avoid retokenization drift). + // - LOGGED trajectory (llm_completions/*.json): token IDs on EVERY + // assistant turn — swe_agents app.py materializes the training + // episode from the logged messages, and NeMo-RL needs per-turn + // generation_token_ids/log_probs to build the loss mask over all + // model-generated spans, not just the final turn. // The IDs round-trip through opencode's part metadata: doStream emits // providerMetadata on text-end / reasoning-end -> processor stores // part.metadata -> replay attaches part.providerOptions["nemo-gym"]. // (The generic converter drops the fields, so we restore them here.) + // Shallow-clone each message so wire-side attach/strip never leaks into + // the logged copy and vice versa. + const loggedMessages = (messages as Array>).map((m) => ({ + ...m, + })) as unknown as ChatRequestMessage[] { const promptAssistants = options.prompt.filter((m) => m.role === "assistant") const wireAssistants = (messages as Array>).filter((m) => m["role"] === "assistant") + const loggedAssistants = (loggedMessages as unknown as Array>).filter( + (m) => m["role"] === "assistant", + ) const n = Math.min(promptAssistants.length, wireAssistants.length) - outer: for (let i = n - 1; i >= 0; i--) { + let mostRecentAttached = false + for (let i = n - 1; i >= 0; i--) { const parts = promptAssistants[i].content if (!Array.isArray(parts)) continue for (const part of parts as Array<{ providerOptions?: Record> }>) { const md = part.providerOptions?.[this.provider] if (md && TOKEN_ID_FIELDS.every((f) => f in md)) { - for (const f of TOKEN_ID_FIELDS) wireAssistants[i][f] = md[f] - break outer + for (const f of TOKEN_ID_FIELDS) loggedAssistants[i][f] = md[f] + if (!mostRecentAttached) { + for (const f of TOKEN_ID_FIELDS) wireAssistants[i][f] = md[f] + mostRecentAttached = true + } + break } } } @@ -462,7 +481,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { if (requestParams[k] === undefined) delete requestParams[k] } - return { warnings, messages, tools, toolChoice, requestParams } + return { warnings, messages, loggedMessages, tools, toolChoice, requestParams } } private async _postChat(params: Record): Promise<{ responseJson: ChatResponse }> { From a7c0593549b58b57dc3329a93a7e60430338f535 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sun, 5 Jul 2026 10:48:04 -0700 Subject: [PATCH 34/49] bench: disable media input entirely (text-only policy model) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two layers: - read tool now errors on image/PDF files instead of returning a base64 attachment, so the agent gets clear feedback and moves on. - main model-request path passes stripMedia: true to toModelMessages, dropping any media attachment from any source before conversion. This prevents the synthetic 'Attached image(s) from tool result:' user message (multi-part content) from ever being injected. The policy model is text-only; the extracted media never reached it anyway — it only produced multi-part messages the gym vllm proxy rejects, plus base64 blobs in session storage. --- packages/opencode/src/session/prompt.ts | 6 +++++- packages/opencode/src/tool/read.ts | 28 +++++++++---------------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 2e53938e04e9..f7c59fe4cba0 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -1569,7 +1569,11 @@ NOTE: At any point in time through this workflow you should feel free to ask the sys.skills(agent), sys.environment(model), instruction.system().pipe(Effect.orDie), - MessageV2.toModelMessagesEffect(msgs, model), + // stripMedia: the policy model is text-only; dropping media + // attachments here prevents the synthetic "Attached image(s) + // from tool result:" user message (multi-part content the gym + // vllm proxy rejects) from ever being injected. + MessageV2.toModelMessagesEffect(msgs, model, { stripMedia: true }), ]) const system = [...env, ...instructions, ...(skills ? [skills] : [])] const format = lastUser.format ?? { type: "text" as const } diff --git a/packages/opencode/src/tool/read.ts b/packages/opencode/src/tool/read.ts index bf01fc7d2d5c..3d0d83b04fca 100644 --- a/packages/opencode/src/tool/read.ts +++ b/packages/opencode/src/tool/read.ts @@ -219,25 +219,17 @@ export const ReadTool = Tool.define( const mime = sniffAttachmentMime(sample, AppFileSystem.mimeType(filepath)) const isImage = SUPPORTED_IMAGE_MIMES.has(mime) + // Media input is disabled in this harness: the policy model is + // text-only, so image/PDF attachments would only produce a synthetic + // multi-part user message the gym's vllm proxy can't accept (and a + // base64 blob in session storage). Fail with a clear error so the + // agent moves on instead of retrying. if (isImage || isPdfAttachment(mime)) { - const bytes = yield* fs.readFile(filepath) - const msg = isPdfAttachment(mime) ? "PDF read successfully" : "Image read successfully" - return { - title, - output: msg, - metadata: { - preview: msg, - truncated: false, - loaded: loaded.map((item) => item.filepath), - }, - attachments: [ - { - type: "file" as const, - mime, - url: `data:${mime};base64,${Buffer.from(bytes).toString("base64")}`, - }, - ], - } + return yield* Effect.fail( + new Error( + `Cannot read ${isPdfAttachment(mime) ? "PDF" : "image"} file: ${filepath} (media input is not supported by this model)`, + ), + ) } if (isBinaryFile(filepath, sample)) { From fa1d67187485f5d5a931cc877a4f18abf3663570 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Wed, 8 Jul 2026 10:39:17 -0700 Subject: [PATCH 35/49] feat: add 10 min timeout for git cleanup --- packages/opencode/src/bench/deep_reset.ts | 42 ++++++++++++- packages/opencode/src/index.ts | 75 +++++++++++++---------- 2 files changed, 82 insertions(+), 35 deletions(-) diff --git a/packages/opencode/src/bench/deep_reset.ts b/packages/opencode/src/bench/deep_reset.ts index 543815f454a5..f127a472111d 100644 --- a/packages/opencode/src/bench/deep_reset.ts +++ b/packages/opencode/src/bench/deep_reset.ts @@ -102,7 +102,19 @@ function shellQuote(s: string): string { return `'${s.replace(/'/g, `'\\''`)}'` } -export async function runDeepReset(workspaceRoot: string, baseCommit: string): Promise { +// Default budget for the whole deep-reset pipeline (checkout, ref rewrite, +// reflog expire, repack, gc, prune). This is plain git plumbing that should +// finish in seconds even on large repos; the timeout exists so a stuck git +// process (lock contention, a pathological repo) can't silently burn the +// entire per-instance agent timeout before a single LLM call happens. +const DEFAULT_TIMEOUT_MS = 10 * 60_000 +const KILL_GRACE_MS = 10_000 + +export async function runDeepReset( + workspaceRoot: string, + baseCommit: string, + timeoutMs = Number(process.env.OPENCODE_DEEP_RESET_TIMEOUT_MS) || DEFAULT_TIMEOUT_MS, +): Promise { if (!baseCommit) return const shell = detectShell() if (!shell) { @@ -114,16 +126,42 @@ export async function runDeepReset(workspaceRoot: string, baseCommit: string): P // ENOENTs whenever a `cwd` is set (libc lacks addchdir_np extension); routing // the chdir through the shell sidesteps that entirely. const cmd = `cd ${shellQuote(workspaceRoot)} && ` + buildDeepResetCmd(baseCommit) - console.log(`[bench] deep_reset workspace=${workspaceRoot} base=${baseCommit} shell=${shell}`) + console.log(`[bench] deep_reset workspace=${workspaceRoot} base=${baseCommit} shell=${shell} timeout_ms=${timeoutMs}`) await new Promise((resolve) => { + // detached: true puts the shell in its own process group so a timeout + // can kill the whole tree (git repack/gc children included) via the + // negative-PID group signal — killing just the top-level bash leaves + // orphaned git children running, silently eating the rest of the budget. const child = spawn(shell, ["-c", cmd], { stdio: ["ignore", "inherit", "inherit"], + detached: true, }) + const killTree = (signal: NodeJS.Signals) => { + try { + if (child.pid) process.kill(-child.pid, signal) + else child.kill(signal) + } catch { + // Group already gone (process exited between the timer firing and here). + } + } + let killTimer: ReturnType | undefined + const timeoutTimer = setTimeout(() => { + console.warn(`[bench] deep_reset exceeded ${timeoutMs}ms; sending SIGTERM`) + killTree("SIGTERM") + killTimer = setTimeout(() => { + console.warn(`[bench] deep_reset still alive ${KILL_GRACE_MS}ms after SIGTERM; sending SIGKILL`) + killTree("SIGKILL") + }, KILL_GRACE_MS) + }, timeoutMs) child.on("close", (code) => { + clearTimeout(timeoutTimer) + clearTimeout(killTimer) console.log(`[bench] deep_reset exit=${code ?? 0}`) resolve() }) child.on("error", (err) => { + clearTimeout(timeoutTimer) + clearTimeout(killTimer) console.warn(`[bench] deep_reset spawn error: ${err}`) resolve() }) diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 444e9730d29c..e975dc863bd6 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -120,41 +120,50 @@ const cli = yargs(args) run_id: processMetadata.runID, }) - const marker = path.join(Global.Path.data, "opencode.db") - if (!(await Filesystem.exists(marker))) { - const tty = process.stderr.isTTY - process.stderr.write("Performing one time database migration, may take a few minutes..." + EOL) - const width = 36 - const orange = "\x1b[38;5;214m" - const muted = "\x1b[0;2m" - const reset = "\x1b[0m" - let last = -1 - if (tty) process.stderr.write("\x1b[?25l") - try { - await JsonMigration.run(drizzle({ client: Database.Client().$client }), { - progress: (event) => { - const percent = Math.floor((event.current / event.total) * 100) - if (percent === last && event.current !== event.total) return - last = percent - if (tty) { - const fill = Math.round((percent / 100) * width) - const bar = `${"■".repeat(fill)}${"･".repeat(width - fill)}` - process.stderr.write( - `\r${orange}${bar} ${percent.toString().padStart(3)}%${reset} ${muted}${event.label.padEnd(12)} ${event.current}/${event.total}${reset}`, - ) - if (event.current === event.total) process.stderr.write("\n") - } else { - process.stderr.write(`sqlite-migration:${percent}${EOL}`) - } - }, - }) - } finally { - if (tty) process.stderr.write("\x1b[?25h") - else { - process.stderr.write(`sqlite-migration:done${EOL}`) + // In-memory DBs (bench/CI harnesses set OPENCODE_DB=:memory:, see + // bench/cli.ts) start empty every process and never persist a marker + // file, so the on-disk marker check below would always be absent and + // this "one time" migration would actually re-run — as a no-op, since + // there's no legacy JSON storage dir to migrate either, but still + // costing a Database.Client() open + filesystem glob — on every single + // invocation instead of truly once. + if (Database.Path !== ":memory:") { + const marker = path.join(Global.Path.data, "opencode.db") + if (!(await Filesystem.exists(marker))) { + const tty = process.stderr.isTTY + process.stderr.write("Performing one time database migration, may take a few minutes..." + EOL) + const width = 36 + const orange = "\x1b[38;5;214m" + const muted = "\x1b[0;2m" + const reset = "\x1b[0m" + let last = -1 + if (tty) process.stderr.write("\x1b[?25l") + try { + await JsonMigration.run(drizzle({ client: Database.Client().$client }), { + progress: (event) => { + const percent = Math.floor((event.current / event.total) * 100) + if (percent === last && event.current !== event.total) return + last = percent + if (tty) { + const fill = Math.round((percent / 100) * width) + const bar = `${"■".repeat(fill)}${"･".repeat(width - fill)}` + process.stderr.write( + `\r${orange}${bar} ${percent.toString().padStart(3)}%${reset} ${muted}${event.label.padEnd(12)} ${event.current}/${event.total}${reset}`, + ) + if (event.current === event.total) process.stderr.write("\n") + } else { + process.stderr.write(`sqlite-migration:${percent}${EOL}`) + } + }, + }) + } finally { + if (tty) process.stderr.write("\x1b[?25h") + else { + process.stderr.write(`sqlite-migration:done${EOL}`) + } } + process.stderr.write("Database migration complete." + EOL) } - process.stderr.write("Database migration complete." + EOL) } }) .usage("") From c64cfa1bad1a53508fe28ea967b9a2563927e10a Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Fri, 10 Jul 2026 17:58:26 +0000 Subject: [PATCH 36/49] bench: add trajectory-replay support (mirrors OpenHands' replay-and-continue) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gym's swe_agents wrapper can already resume an OpenHands run from a partial trajectory (REPLAY_MESSAGES_PATH); this wires the same capability into the opencode bench harness. run_infer.sh gains a new $13 REPLAY_MESSAGES_PATH positional arg, forwarded to bench/cli.ts as --replay-messages-file. cli.ts parses the replay file (bench/replay.ts, mirroring OpenHands' messages_to_replay_events() skip rules: system/tool/subsequent-user messages are skipped, the first user message becomes the task instruction verbatim, and each assistant message becomes a scripted turn) and threads the scripted turns into the nemo-gym provider's config instead of gym's rendered user_message.txt. NemoGymLanguageModel answers the first N doStream/doGenerate calls (main session only) from that scripted queue by synthesizing the same kind of stream it already builds from a real HTTP response, instead of making one. Tool calls in that synthesized stream still get executed for real by opencode's normal streamText -> resolveTools() -> Tool.execute() path (unchanged) — required for correctness, since SWE-bench agents mutate a git workspace and the final patch comes from `git diff`: a replayed edit/bash call has to actually happen on the fresh container, not just be asserted via stale recorded output text. Scripted turns are never dumped to completionsDir, so gym's replay/live boundary detection (first dumped completion's cumulative message count) still lines up on the first live call. Once the queue drains, both methods fall through unchanged to the existing real-HTTP path and the agent continues live — no changes needed to session/processor.ts, session/prompt.ts, or any HTTP route. --- .../benchmarks/swe_bench/scripts/run_infer.sh | 7 + packages/opencode/src/bench/cli.ts | 34 ++- packages/opencode/src/bench/replay.ts | 58 +++++ .../src/provider/sdk/nemo-gym/index.ts | 9 +- .../provider/sdk/nemo-gym/language-model.ts | 211 ++++++++++++++---- packages/opencode/test/bench/replay.test.ts | 84 +++++++ .../provider/nemo-gym/language-model.test.ts | 124 ++++++++++ 7 files changed, 480 insertions(+), 47 deletions(-) create mode 100644 packages/opencode/src/bench/replay.ts create mode 100644 packages/opencode/test/bench/replay.test.ts create mode 100644 packages/opencode/test/provider/nemo-gym/language-model.test.ts diff --git a/evaluation/benchmarks/swe_bench/scripts/run_infer.sh b/evaluation/benchmarks/swe_bench/scripts/run_infer.sh index 2edf91b721cc..154dc6fdcf7b 100755 --- a/evaluation/benchmarks/swe_bench/scripts/run_infer.sh +++ b/evaluation/benchmarks/swe_bench/scripts/run_infer.sh @@ -14,6 +14,8 @@ # $10 WORKSPACE_ROOT resolved repo path inside the SIF (gym side decided) # $11 USER_MESSAGE_PATH pre-rendered user prompt file (workspace baked in) # $12 SYSTEM_PROMPT_PATH optional system-prompt override +# $13 REPLAY_MESSAGES_PATH optional JSON file of prior chat-completion +# messages to replay before continuing live # # Environment (set by gym): # NEMO_GYM_MODEL_SERVER_NAME proxy name on the gym head server @@ -38,6 +40,7 @@ CONFIG_FILE="${9:-/tmp/oc_config.json}" WORKSPACE_ROOT="${10:-}" USER_MESSAGE_PATH="${11:-}" SYSTEM_PROMPT_PATH="${12:-}" +REPLAY_MESSAGES_PATH="${13:-}" if [ -z "$SELECTED_ID" ]; then echo "ERROR: SELECTED_ID (\$7) is required." @@ -92,6 +95,7 @@ echo "CONFIG_FILE: $CONFIG_FILE" echo "WORKSPACE_ROOT: $WORKSPACE_ROOT" echo "USER_MESSAGE_PATH: $USER_MESSAGE_PATH" echo "SYSTEM_PROMPT_PATH: $SYSTEM_PROMPT_PATH" +echo "REPLAY_MESSAGES_PATH: $REPLAY_MESSAGES_PATH" echo "MODEL_SERVER: $NEMO_GYM_MODEL_SERVER_NAME @ $NEMO_GYM_MODEL_SERVER_BASE_URL" cmd=( @@ -110,6 +114,9 @@ cmd=( if [ -n "$SYSTEM_PROMPT_PATH" ]; then cmd+=(--system-prompt "$SYSTEM_PROMPT_PATH") fi +if [ -n "$REPLAY_MESSAGES_PATH" ]; then + cmd+=(--replay-messages-file "$REPLAY_MESSAGES_PATH") +fi if [ "${ENABLE_SUBAGENTS:-0}" = "1" ] || [ "${ENABLE_SUBAGENTS:-}" = "true" ]; then cmd+=(--enable-subagents) fi diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 2b47ec452def..2a004c73c37b 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -28,6 +28,8 @@ import { bootstrapRepoIfMissing } from "./bootstrap_repo" // opencode's built-in anthropic system prompt — Bun bundles .txt as a string. // Used as the default when no --system-prompt override is passed. import PROMPT_ANTHROPIC from "../session/prompt/anthropic.txt" +import type { NemoGymReplayTurn } from "../provider/sdk/nemo-gym/language-model" +import { parseReplayMessages } from "./replay" interface CliArgs { instanceDictPath: string @@ -45,6 +47,11 @@ interface CliArgs { systemPromptPath?: string /** Enable opencode's `task` tool (spawns subagent sessions). */ enableSubagents: boolean + /** + * Path to a JSON file of prior chat-completion-format messages to replay + * before continuing live (trajectory resume). See language-model.ts. + */ + replayMessagesFile?: string } function parseArgs(argv: string[]): CliArgs { @@ -95,6 +102,9 @@ function parseArgs(argv: string[]): CliArgs { case "--enable-subagents": out.enableSubagents = true break + case "--replay-messages-file": + out.replayMessagesFile = next() + break default: if (a.startsWith("--")) throw new Error(`Unknown flag: ${a}`) } @@ -153,6 +163,8 @@ async function buildConfigDir(args: { temperature?: number topP?: number maxTokens?: number + /** Scripted assistant turns to replay before the agent continues live. */ + replayTurns?: NemoGymReplayTurn[] }): Promise<{ tmpRoot: string; configFile: string }> { const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), `bench-${args.instanceId}-`)) await fs.mkdir(tmpRoot, { recursive: true }) @@ -182,6 +194,7 @@ async function buildConfigDir(args: { ...(args.temperature !== undefined ? { temperature: args.temperature } : {}), ...(args.topP !== undefined ? { topP: args.topP } : {}), ...(args.maxTokens !== undefined ? { maxTokens: args.maxTokens } : {}), + ...(args.replayTurns?.length ? { replayTurns: args.replayTurns } : {}), }, models: { [args.modelName]: { @@ -412,9 +425,23 @@ async function main() { const completionsDir = completionsDirFor(args.outputDir, instance.instance_id) await fs.mkdir(completionsDir, { recursive: true }) - // The user message is fully rendered by gym (workspace_path baked in based - // on dataset_name); we just read it as-is and pass it to opencode. - const userPrompt = await fs.readFile(args.userMessageFile, "utf8") + // Trajectory resume: when a replay file is given, the recorded conversation's + // own first user message becomes the task instruction (byte-for-byte, not + // gym's rendered template — mirrors OpenHands' replay semantics) and the + // recorded assistant turns are threaded into the nemo-gym provider so it + // replays them (re-executing tool calls for real) before continuing live. + let userPrompt: string + let replayTurns: NemoGymReplayTurn[] | undefined + if (args.replayMessagesFile) { + const raw = await fs.readFile(args.replayMessagesFile, "utf8") + const parsed = parseReplayMessages(raw) + userPrompt = parsed.initialUserText + replayTurns = parsed.replayTurns + } else { + // The user message is fully rendered by gym (workspace_path baked in based + // on dataset_name); we just read it as-is and pass it to opencode. + userPrompt = await fs.readFile(args.userMessageFile, "utf8") + } const { tmpRoot, configFile } = await buildConfigDir({ instanceId: instance.instance_id, @@ -427,6 +454,7 @@ async function main() { temperature: forcedTemperature, topP: forcedTopP, maxTokens: forcedMaxTokens, + replayTurns, }) const startedAt = Date.now() diff --git a/packages/opencode/src/bench/replay.ts b/packages/opencode/src/bench/replay.ts new file mode 100644 index 000000000000..f6d5adad2742 --- /dev/null +++ b/packages/opencode/src/bench/replay.ts @@ -0,0 +1,58 @@ +/** + * Parses a replay-messages file (prior chat-completion-format trajectory) for + * the SWE-bench bench harness. Split out of `bench/cli.ts` so it's importable + * without triggering `cli.ts`'s top-level `main()` (which calls + * `process.exit` on missing/invalid CLI args — not test-friendly). + * + * Mirrors OpenHands' `messages_to_replay_events()` skip rules (see + * `temp/nv-OpenHands/evaluation/benchmarks/swe_bench/replay_utils.py`): + * system / tool / subsequent-user messages are skipped — the first user + * message becomes the task instruction, assistant messages become scripted + * turns replayed in order by the nemo-gym provider (see language-model.ts). + */ + +import type { NemoGymReplayTurn } from "../provider/sdk/nemo-gym/language-model" + +export interface ReplayChatMessage { + role: "system" | "user" | "assistant" | "tool" + content?: string | Array<{ type?: string; text?: string }> | null + tool_calls?: Array<{ id: string; type?: string; function: { name: string; arguments: string } }> +} + +export function replayMessageText(content: ReplayChatMessage["content"]): string { + if (typeof content === "string") return content + if (Array.isArray(content)) { + return content + .filter((part) => part?.type === "text" || typeof part?.text === "string") + .map((part) => part.text ?? "") + .join("\n") + } + return "" +} + +export function parseReplayMessages(raw: string): { initialUserText: string; replayTurns: NemoGymReplayTurn[] } { + const messages = JSON.parse(raw) as ReplayChatMessage[] + + let initialUserText: string | undefined + const replayTurns: NemoGymReplayTurn[] = [] + + for (const msg of messages) { + if (msg.role === "system" || msg.role === "tool") continue + if (msg.role === "user") { + if (initialUserText === undefined) initialUserText = replayMessageText(msg.content) + continue + } + if (msg.role === "assistant") { + replayTurns.push({ + content: typeof msg.content === "string" ? msg.content : replayMessageText(msg.content) || null, + toolCalls: msg.tool_calls?.map((tc) => ({ id: tc.id, name: tc.function.name, arguments: tc.function.arguments })), + }) + } + } + + if (initialUserText === undefined) { + throw new Error("replay-messages-file: no user message found (expected at least one task-instruction message)") + } + + return { initialUserText, replayTurns } +} diff --git a/packages/opencode/src/provider/sdk/nemo-gym/index.ts b/packages/opencode/src/provider/sdk/nemo-gym/index.ts index 9f7460d1450e..892a81b5a138 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/index.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/index.ts @@ -9,7 +9,7 @@ * provider plumbing (Provider.Service.getModel) works without special-casing. */ -import { NemoGymLanguageModel, type NemoGymLanguageModelConfig } from "./language-model" +import { NemoGymLanguageModel, type NemoGymLanguageModelConfig, type NemoGymReplayTurn } from "./language-model" export interface CreateNemoGymOptions { /** Base URL of the gym model server (`http://host:port`). */ @@ -42,6 +42,12 @@ export interface CreateNemoGymOptions { turnCounter?: { next(): number } /** Optional callback invoked after each successful chat-completion. */ onCompletion?: NemoGymLanguageModelConfig["onCompletion"] + /** + * Scripted assistant turns to replay (main session only) before falling + * through to live HTTP calls. Set by the bench harness when the request + * carries a prior trajectory to resume. See language-model.ts's docblock. + */ + replayTurns?: NemoGymReplayTurn[] } export interface NemoGymProvider { @@ -68,6 +74,7 @@ export function createNemoGym(opts: CreateNemoGymOptions): NemoGymProvider { maxTokens: opts.maxTokens, turnCounter: opts.turnCounter, onCompletion: opts.onCompletion, + replayTurns: opts.replayTurns, }) }, } diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 0a0f9e56c386..475bd1c9c236 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -17,6 +17,22 @@ * so a tool crash later cannot lose this turn's token IDs. The shape matches * openhands' `llm_completions//*.json` exactly so gym's * `get_openhands_trajectory_from_completions` reads it without changes. + * + * Replay: when `cfg.replayTurns` is set, the first N calls (main session + * only) are answered from that scripted queue instead of a real HTTP call — + * same synthesized-stream shape as a real response, so opencode's *real* + * tool-execution path (streamText -> the AI-SDK `tool.execute` closures + * built in session/prompt.ts's resolveTools()) runs each replayed tool call + * for real against the live sandbox. This is required for correctness: + * SWE-bench agents mutate a git workspace and the final patch comes from + * `git diff`, so a replayed "edit"/"bash" call must actually happen on the + * fresh container, not just be asserted via stale recorded output text. + * Once the queue is exhausted, doStream/doGenerate fall through unchanged to + * the real HTTP path and the agent continues live. Scripted turns are never + * dumped to completionsDir: gym derives the replay/live boundary from the + * FIRST dumped completion's cumulative `messages` length, which must be the + * first live call (by then the replay prefix is already in session + * history), not a scripted one. */ import { @@ -91,6 +107,12 @@ interface ChatResponse { const TOKEN_ID_FIELDS = ["prompt_token_ids", "generation_token_ids", "generation_log_probs"] as const +/** A single scripted assistant turn to replay before falling through to live model calls. */ +export interface NemoGymReplayTurn { + content: string | null + toolCalls?: Array<{ id: string; name: string; arguments: string }> +} + export interface NemoGymLanguageModelConfig { /** Provider id used to namespace providerMetadata. Defaults to "nemo-gym". */ provider: string @@ -137,6 +159,12 @@ export interface NemoGymLanguageModelConfig { providerSpecificFields: Record requestParams: Record }) => void | Promise + /** + * Scripted assistant turns to replay (main session only) before falling + * through to live HTTP calls. See the module docblock for why replayed + * tool calls must run for real rather than just replaying recorded text. + */ + replayTurns?: NemoGymReplayTurn[] } // --------------------------------------------------------------------------- @@ -154,6 +182,9 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // subagents spawned via the task tool get their own sessionID, so keeping // a Map keeps their dump filenames from clobbering the main session's. private readonly turnCounters: Map = new Map() + // Replay only ever applies to the main session (subagent sessions never + // existed in the recorded trajectory) — a single counter is sufficient. + private replayIndex = 0 constructor(modelId: string, cfg: NemoGymLanguageModelConfig) { this.modelId = modelId @@ -184,6 +215,86 @@ export class NemoGymLanguageModel implements LanguageModelV3 { return { sessionID: sid || "main", parentSessionID: pid } } + // Pops the next scripted turn for the main session, or undefined once the + // replay queue is exhausted / not applicable (subagent sessions, no + // replayTurns configured). Advances state, so call at most once per doStream + // / doGenerate invocation. + private _popReplayTurn(session: { sessionID: string }): NemoGymReplayTurn | undefined { + if (!this.cfg.replayTurns) return undefined + if (session.sessionID !== "main") return undefined + if (this.replayIndex >= this.cfg.replayTurns.length) return undefined + return this.cfg.replayTurns[this.replayIndex++] + } + + private _messageFromReplayTurn(turn: NemoGymReplayTurn): ChatResponseChoice["message"] { + return { + role: "assistant", + content: turn.content, + tool_calls: turn.toolCalls?.map((tc) => ({ + id: tc.id, + type: "function" as const, + function: { name: tc.name, arguments: tc.arguments }, + })), + } + } + + private _replayFinishReason(turn: NemoGymReplayTurn): string { + return turn.toolCalls && turn.toolCalls.length > 0 ? "tool_calls" : "stop" + } + + // Shared by the real HTTP path and the replay path so both stay in sync: + // reasoning -> text -> tool-call stream parts for one assistant turn. + private _enqueueMessageParts( + controller: ReadableStreamDefaultController, + msg: Pick, + providerMetadata: SharedV3ProviderMetadata, + ) { + // Reasoning content. providerMetadata on the *-end events is persisted by + // opencode's processor as part.metadata and replayed on the next request + // as part.providerOptions["nemo-gym"] — this is how per-turn token IDs + // round-trip so EVERY assistant turn (not just the last) carries them for + // RL training reconstruction. Replay turns carry no token IDs (empty + // providerMetadata), which is fine — gym never reads a dump for them. + if (msg.reasoning_text) { + controller.enqueue({ type: "reasoning-start", id: "reasoning-0" }) + controller.enqueue({ type: "reasoning-delta", id: "reasoning-0", delta: msg.reasoning_text }) + controller.enqueue({ type: "reasoning-end", id: "reasoning-0", providerMetadata }) + } + + // Text content. + if (msg.content) { + controller.enqueue({ type: "text-start", id: "txt-0" }) + controller.enqueue({ type: "text-delta", id: "txt-0", delta: msg.content }) + controller.enqueue({ type: "text-end", id: "txt-0", providerMetadata }) + } + + // Tool calls. Replayed tool-call IDs are reused verbatim (see + // _messageFromReplayTurn) so gym's replay/live boundary matching by + // call_id still lines up. + if (msg.tool_calls) { + for (const tc of msg.tool_calls) { + const tcId = tc.id ?? `call_${Math.random().toString(36).slice(2, 10)}` + controller.enqueue({ + type: "tool-input-start", + id: tcId, + toolName: tc.function.name, + }) + controller.enqueue({ + type: "tool-input-delta", + id: tcId, + delta: tc.function.arguments, + }) + controller.enqueue({ type: "tool-input-end", id: tcId }) + controller.enqueue({ + type: "tool-call", + toolCallId: tcId, + toolName: tc.function.name, + input: tc.function.arguments, + }) + } + } + } + get supportedUrls() { return {} as Record } @@ -191,8 +302,36 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // The streamText path in `session/llm.ts` only calls doStream. We still // implement doGenerate for completeness / future direct-use. async doGenerate(options: LanguageModelV3CallOptions) { - const { warnings, loggedMessages, requestParams } = await this._buildRequestParams(options) const session = this._sessionFromHeaders(options.headers) + const replayTurn = this._popReplayTurn(session) + + if (replayTurn) { + const msg = this._messageFromReplayTurn(replayTurn) + const providerMetadata = this._buildProviderMetadata({}) + const content: LanguageModelV3Content[] = [] + if (msg.content) content.push({ type: "text", text: msg.content, providerMetadata }) + if (msg.tool_calls) { + for (const tc of msg.tool_calls) { + content.push({ + type: "tool-call", + toolCallId: tc.id ?? `call_${Math.random().toString(36).slice(2, 10)}`, + toolName: tc.function.name, + input: tc.function.arguments, + }) + } + } + return { + content, + finishReason: this._mapFinishReason(this._replayFinishReason(replayTurn)), + usage: this._mapUsage(undefined), + providerMetadata, + request: { body: "{}" }, + response: { body: {} }, + warnings: [], + } + } + + const { warnings, loggedMessages, requestParams } = await this._buildRequestParams(options) const { responseJson } = await this._postChat(requestParams) const choice = responseJson.choices[0] @@ -237,8 +376,34 @@ export class NemoGymLanguageModel implements LanguageModelV3 { } async doStream(options: LanguageModelV3CallOptions) { - const { warnings, loggedMessages, requestParams } = await this._buildRequestParams(options) const session = this._sessionFromHeaders(options.headers) + const replayTurn = this._popReplayTurn(session) + + if (replayTurn) { + const msg = this._messageFromReplayTurn(replayTurn) + const finishReasonRaw = this._replayFinishReason(replayTurn) + const self = this + const stream = new ReadableStream({ + start(controller) { + controller.enqueue({ type: "stream-start", warnings: [] }) + const providerMetadata = self._buildProviderMetadata({}) + self._enqueueMessageParts(controller, msg, providerMetadata) + // No _dumpAndNotify here — see the module docblock: gym derives the + // replay/live boundary from the FIRST dumped completion, which must + // be the first live call. + controller.enqueue({ + type: "finish", + finishReason: self._mapFinishReason(finishReasonRaw), + usage: self._mapUsage(undefined), + providerMetadata, + }) + controller.close() + }, + }) + return { stream, request: { body: "{}" }, response: {} } + } + + const { warnings, loggedMessages, requestParams } = await this._buildRequestParams(options) // Fire the HTTP call eagerly so any error surfaces synchronously when the // stream is consumed. We then synthesize parts in `start`. @@ -267,47 +432,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { timestamp: responseJson.created ? new Date(responseJson.created * 1000) : undefined, }) - // Reasoning content. providerMetadata on the *-end events is - // persisted by opencode's processor as part.metadata and replayed - // on the next request as part.providerOptions["nemo-gym"] — this is - // how per-turn token IDs round-trip so EVERY assistant turn (not - // just the last) carries them for RL training reconstruction. - if (msg.reasoning_text) { - controller.enqueue({ type: "reasoning-start", id: "reasoning-0" }) - controller.enqueue({ type: "reasoning-delta", id: "reasoning-0", delta: msg.reasoning_text }) - controller.enqueue({ type: "reasoning-end", id: "reasoning-0", providerMetadata }) - } - - // Text content. - if (msg.content) { - controller.enqueue({ type: "text-start", id: "txt-0" }) - controller.enqueue({ type: "text-delta", id: "txt-0", delta: msg.content }) - controller.enqueue({ type: "text-end", id: "txt-0", providerMetadata }) - } - - // Tool calls. - if (msg.tool_calls) { - for (const tc of msg.tool_calls) { - const tcId = tc.id ?? `call_${Math.random().toString(36).slice(2, 10)}` - controller.enqueue({ - type: "tool-input-start", - id: tcId, - toolName: tc.function.name, - }) - controller.enqueue({ - type: "tool-input-delta", - id: tcId, - delta: tc.function.arguments, - }) - controller.enqueue({ type: "tool-input-end", id: tcId }) - controller.enqueue({ - type: "tool-call", - toolCallId: tcId, - toolName: tc.function.name, - input: tc.function.arguments, - }) - } - } + self._enqueueMessageParts(controller, msg, providerMetadata) // Persist trajectory BEFORE finishing so a downstream tool crash // cannot lose this turn's token IDs. diff --git a/packages/opencode/test/bench/replay.test.ts b/packages/opencode/test/bench/replay.test.ts new file mode 100644 index 000000000000..83fd326b66ff --- /dev/null +++ b/packages/opencode/test/bench/replay.test.ts @@ -0,0 +1,84 @@ +import { describe, test, expect } from "bun:test" +import { parseReplayMessages, replayMessageText } from "@/bench/replay" + +describe("replayMessageText", () => { + test("returns string content verbatim", () => { + expect(replayMessageText("hello")).toBe("hello") + }) + + test("joins text parts from array content", () => { + expect( + replayMessageText([ + { type: "text", text: "line one" }, + { type: "text", text: "line two" }, + ]), + ).toBe("line one\nline two") + }) + + test("returns empty string for null/undefined content", () => { + expect(replayMessageText(null)).toBe("") + expect(replayMessageText(undefined)).toBe("") + }) +}) + +describe("parseReplayMessages", () => { + test("extracts the first user message as the initial task instruction", () => { + const raw = JSON.stringify([ + { role: "system", content: "sys prompt" }, + { role: "user", content: "fix the bug" }, + ]) + const { initialUserText, replayTurns } = parseReplayMessages(raw) + expect(initialUserText).toBe("fix the bug") + expect(replayTurns).toEqual([]) + }) + + test("skips system, tool, and subsequent user messages", () => { + const raw = JSON.stringify([ + { role: "system", content: "sys prompt" }, + { role: "user", content: "fix the bug" }, + { + role: "assistant", + content: null, + tool_calls: [{ id: "call_1", type: "function", function: { name: "bash", arguments: '{"cmd":"ls"}' } }], + }, + { role: "tool", content: "file1.py\n", tool_call_id: "call_1" }, + { role: "user", content: "please continue" }, + { role: "assistant", content: "Done.", tool_calls: undefined }, + ]) + const { initialUserText, replayTurns } = parseReplayMessages(raw) + expect(initialUserText).toBe("fix the bug") + expect(replayTurns).toEqual([ + { content: null, toolCalls: [{ id: "call_1", name: "bash", arguments: '{"cmd":"ls"}' }] }, + { content: "Done.", toolCalls: undefined }, + ]) + }) + + test("preserves tool_call ids verbatim, including multiple calls in one turn", () => { + const raw = JSON.stringify([ + { role: "user", content: "fix the bug" }, + { + role: "assistant", + content: null, + tool_calls: [ + { id: "call_abc", type: "function", function: { name: "read", arguments: '{"path":"a.py"}' } }, + { id: "call_def", type: "function", function: { name: "read", arguments: '{"path":"b.py"}' } }, + ], + }, + ]) + const { replayTurns } = parseReplayMessages(raw) + expect(replayTurns[0].toolCalls?.map((tc) => tc.id)).toEqual(["call_abc", "call_def"]) + }) + + test("joins array-of-parts user content for the initial instruction", () => { + const raw = JSON.stringify([ + { role: "user", content: [{ type: "text", text: "part one" }, { type: "text", text: "part two" }] }, + ]) + const { initialUserText } = parseReplayMessages(raw) + expect(initialUserText).toBe("part one\npart two") + }) + + test("throws when no user message is present", () => { + const raw = JSON.stringify([{ role: "system", content: "sys prompt" }]) + expect(() => parseReplayMessages(raw)).toThrow(/no user message/) + }) +}) diff --git a/packages/opencode/test/provider/nemo-gym/language-model.test.ts b/packages/opencode/test/provider/nemo-gym/language-model.test.ts new file mode 100644 index 000000000000..31d8299876e6 --- /dev/null +++ b/packages/opencode/test/provider/nemo-gym/language-model.test.ts @@ -0,0 +1,124 @@ +import { describe, test, expect, mock } from "bun:test" +import { NemoGymLanguageModel } from "@/provider/sdk/nemo-gym/language-model" +import type { LanguageModelV3CallOptions, LanguageModelV3StreamPart } from "@ai-sdk/provider" + +async function drain(stream: ReadableStream): Promise { + const reader = stream.getReader() + const parts: LanguageModelV3StreamPart[] = [] + while (true) { + const { done, value } = await reader.read() + if (done) break + parts.push(value) + } + return parts +} + +const CALL_OPTIONS: LanguageModelV3CallOptions = { + prompt: [{ role: "user", content: [{ type: "text", text: "hi" }] }], +} + +describe("NemoGymLanguageModel replay", () => { + test("doStream replays a scripted tool-call turn without hitting the network", async () => { + const fetchSpy = mock(async () => { + throw new Error("network should not be called during replay") + }) + // @ts-expect-error test override + globalThis.fetch = fetchSpy + + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + replayTurns: [{ content: null, toolCalls: [{ id: "call_1", name: "bash", arguments: '{"cmd":"ls"}' }] }], + }) + + const { stream } = await model.doStream(CALL_OPTIONS) + const parts = await drain(stream) + + expect(fetchSpy).not.toHaveBeenCalled() + + const toolCall = parts.find((p) => p.type === "tool-call") + expect(toolCall).toMatchObject({ toolCallId: "call_1", toolName: "bash", input: '{"cmd":"ls"}' }) + + const finish = parts.find((p) => p.type === "finish") + expect(finish).toMatchObject({ finishReason: { unified: "tool-calls" } }) + }) + + test("doStream replays a scripted text-only turn (no tool calls) as finishReason stop", async () => { + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + replayTurns: [{ content: "All done." }], + }) + + const { stream } = await model.doStream(CALL_OPTIONS) + const parts = await drain(stream) + + expect(parts.some((p) => p.type === "tool-call")).toBe(false) + const textDelta = parts.find((p) => p.type === "text-delta") + expect(textDelta).toMatchObject({ delta: "All done." }) + const finish = parts.find((p) => p.type === "finish") + expect(finish).toMatchObject({ finishReason: { unified: "stop" } }) + }) + + test("doStream falls through to the real HTTP path once the replay queue is exhausted", async () => { + const fetchSpy = mock(async () => + new Response( + JSON.stringify({ + id: "resp_1", + model: "test-model", + choices: [{ finish_reason: "stop", message: { role: "assistant", content: "live turn" } }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ) + // @ts-expect-error test override + globalThis.fetch = fetchSpy + + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + replayTurns: [{ content: "scripted turn" }], + }) + + // First call: replay (no fetch). + await drain((await model.doStream(CALL_OPTIONS)).stream) + expect(fetchSpy).not.toHaveBeenCalled() + + // Second call: replay queue exhausted -> real HTTP path. + const { stream } = await model.doStream(CALL_OPTIONS) + const parts = await drain(stream) + expect(fetchSpy).toHaveBeenCalledTimes(1) + const textDelta = parts.find((p) => p.type === "text-delta") + expect(textDelta).toMatchObject({ delta: "live turn" }) + }) + + test("replay is scoped to the main session — subagent sessions call through to HTTP", async () => { + const fetchSpy = mock(async () => + new Response( + JSON.stringify({ + id: "resp_1", + model: "test-model", + choices: [{ finish_reason: "stop", message: { role: "assistant", content: "subagent turn" } }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ) + // @ts-expect-error test override + globalThis.fetch = fetchSpy + + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + replayTurns: [{ content: "scripted turn" }], + }) + + const subagentOptions: LanguageModelV3CallOptions = { + ...CALL_OPTIONS, + headers: { "x-session-affinity": "ses_subagent_1" }, + } + const { stream } = await model.doStream(subagentOptions) + const parts = await drain(stream) + expect(fetchSpy).toHaveBeenCalledTimes(1) + expect(parts.find((p) => p.type === "text-delta")).toMatchObject({ delta: "subagent turn" }) + }) +}) From dd930538d1ccd3fa7f8af2c4f4d110d1142342ed Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sat, 11 Jul 2026 00:21:55 +0000 Subject: [PATCH 37/49] bench: fix replay session-scoping (was completely non-functional) + subsequent user messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical fix: replay never actually activated in a real opencode run. The previous check compared the resolved session id to a literal "main" sentinel, but session/llm.ts sets `x-session-affinity: input.sessionID` unconditionally for every call on the nemo-gym provider — including the top-level session's own calls — so sessionID is always the real session id (e.g. "ses_...") and never equals "main". Every call fell through to a live HTTP request with no replay context at all, which is exactly the "starts solving from the beginning" symptom. Fixed by gating on `parentSessionID` presence (the actual signal for "this is a subagent session") instead. A second, related bug: opencode's own runLoop forks off title-generation and summary-generation model calls on step 1, using the SAME session id as the real agentic loop. Those race ahead of the loop's own first call and, once the sessionID check above was fixed, would otherwise silently consume scripted replay turns meant for the real agent (confirmed happening in practice). They're reliably distinguishable because they never pass `tools` — only the real agentic loop resolves and sends the tool registry — so `_popReplayTurn` now also requires `hasTools`. Verified end-to-end against a real opencode run (not just unit tests, which didn't set a `headers` object and so never exercised either bug): a 2-turn scripted replay against a real git workspace now correctly re-executes both bash tool calls for real (workspace shows the replayed edit, `git diff` is non-empty) before falling through to a live call whose message list includes the full replay prefix. Also adds subsequent-user-message support: a user message anywhere after the trajectory's first one is real request content the model must see, not an action to replay. bench/replay.ts now returns them attached to the replay turn they precede (or as trailing texts, if the trajectory ends on a user message); NemoGymLanguageModel splices them into the outgoing message list — and the llm_completions dump gym reads — on every live call from the point replay reaches them onward, since opencode's own session storage isn't writable from this provider. Co-Authored-By: Claude Sonnet 5 --- packages/opencode/src/bench/cli.ts | 8 + packages/opencode/src/bench/replay.ts | 50 +++++- .../src/provider/sdk/nemo-gym/index.ts | 3 + .../provider/sdk/nemo-gym/language-model.ts | 138 ++++++++++++-- packages/opencode/test/bench/replay.test.ts | 64 ++++++- .../provider/nemo-gym/language-model.test.ts | 168 +++++++++++++++++- 6 files changed, 407 insertions(+), 24 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 2a004c73c37b..ed3042b4b7b7 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -165,6 +165,8 @@ async function buildConfigDir(args: { maxTokens?: number /** Scripted assistant turns to replay before the agent continues live. */ replayTurns?: NemoGymReplayTurn[] + /** Subsequent user messages trailing the last replayed turn. */ + replayTrailingUserTexts?: string[] }): Promise<{ tmpRoot: string; configFile: string }> { const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), `bench-${args.instanceId}-`)) await fs.mkdir(tmpRoot, { recursive: true }) @@ -195,6 +197,9 @@ async function buildConfigDir(args: { ...(args.topP !== undefined ? { topP: args.topP } : {}), ...(args.maxTokens !== undefined ? { maxTokens: args.maxTokens } : {}), ...(args.replayTurns?.length ? { replayTurns: args.replayTurns } : {}), + ...(args.replayTrailingUserTexts?.length + ? { replayTrailingUserTexts: args.replayTrailingUserTexts } + : {}), }, models: { [args.modelName]: { @@ -432,11 +437,13 @@ async function main() { // replays them (re-executing tool calls for real) before continuing live. let userPrompt: string let replayTurns: NemoGymReplayTurn[] | undefined + let replayTrailingUserTexts: string[] | undefined if (args.replayMessagesFile) { const raw = await fs.readFile(args.replayMessagesFile, "utf8") const parsed = parseReplayMessages(raw) userPrompt = parsed.initialUserText replayTurns = parsed.replayTurns + replayTrailingUserTexts = parsed.trailingUserTexts } else { // The user message is fully rendered by gym (workspace_path baked in based // on dataset_name); we just read it as-is and pass it to opencode. @@ -455,6 +462,7 @@ async function main() { topP: forcedTopP, maxTokens: forcedMaxTokens, replayTurns, + replayTrailingUserTexts, }) const startedAt = Date.now() diff --git a/packages/opencode/src/bench/replay.ts b/packages/opencode/src/bench/replay.ts index f6d5adad2742..fa44312ae4c7 100644 --- a/packages/opencode/src/bench/replay.ts +++ b/packages/opencode/src/bench/replay.ts @@ -4,11 +4,26 @@ * without triggering `cli.ts`'s top-level `main()` (which calls * `process.exit` on missing/invalid CLI args — not test-friendly). * - * Mirrors OpenHands' `messages_to_replay_events()` skip rules (see - * `temp/nv-OpenHands/evaluation/benchmarks/swe_bench/replay_utils.py`): - * system / tool / subsequent-user messages are skipped — the first user - * message becomes the task instruction, assistant messages become scripted - * turns replayed in order by the nemo-gym provider (see language-model.ts). + * The first user message becomes the task instruction, assistant messages + * become scripted turns replayed in order by the nemo-gym provider (see + * language-model.ts). System / tool messages are skipped — system content is + * handled separately (pinned as the agent's system prompt by gym), and tool + * output is regenerated fresh by actually re-executing each replayed tool + * call against the sandbox, not replayed from recorded text. + * + * A *subsequent* user message (anything after the first) is real request + * content — everything the caller sent must be part of what the model sees. + * Unlike assistant/tool content it isn't reconstructed by replaying it + * through the agent loop (it's inert text, not an action), so each one is + * attached to the replay turn it immediately precedes as `precedingUserTexts` + * (or, if it trails the very last replayed turn with nothing recorded after + * it, returned separately as `trailingUserTexts`). `NemoGymLanguageModel` + * splices these into the outgoing message list on every live model call from + * the first one onward — they're not persisted in opencode's own session + * storage (there's no cheap way to do that without restructuring the bench + * harness to drive multiple `session.prompt()` calls against one long-lived, + * disk-backed session), so the provider re-applies them on every request + * instead of relying on session history to carry them forward. */ import type { NemoGymReplayTurn } from "../provider/sdk/nemo-gym/language-model" @@ -30,23 +45,38 @@ export function replayMessageText(content: ReplayChatMessage["content"]): string return "" } -export function parseReplayMessages(raw: string): { initialUserText: string; replayTurns: NemoGymReplayTurn[] } { +export interface ParsedReplay { + initialUserText: string + replayTurns: NemoGymReplayTurn[] + /** Subsequent user messages after the last replayed assistant turn (trajectory ends on a user message). */ + trailingUserTexts?: string[] +} + +export function parseReplayMessages(raw: string): ParsedReplay { const messages = JSON.parse(raw) as ReplayChatMessage[] let initialUserText: string | undefined const replayTurns: NemoGymReplayTurn[] = [] + let pendingUserTexts: string[] = [] for (const msg of messages) { if (msg.role === "system" || msg.role === "tool") continue if (msg.role === "user") { - if (initialUserText === undefined) initialUserText = replayMessageText(msg.content) + const text = replayMessageText(msg.content) + if (initialUserText === undefined) { + initialUserText = text + } else if (text) { + pendingUserTexts.push(text) + } continue } if (msg.role === "assistant") { replayTurns.push({ content: typeof msg.content === "string" ? msg.content : replayMessageText(msg.content) || null, toolCalls: msg.tool_calls?.map((tc) => ({ id: tc.id, name: tc.function.name, arguments: tc.function.arguments })), + ...(pendingUserTexts.length ? { precedingUserTexts: pendingUserTexts } : {}), }) + pendingUserTexts = [] } } @@ -54,5 +84,9 @@ export function parseReplayMessages(raw: string): { initialUserText: string; rep throw new Error("replay-messages-file: no user message found (expected at least one task-instruction message)") } - return { initialUserText, replayTurns } + return { + initialUserText, + replayTurns, + ...(pendingUserTexts.length ? { trailingUserTexts: pendingUserTexts } : {}), + } } diff --git a/packages/opencode/src/provider/sdk/nemo-gym/index.ts b/packages/opencode/src/provider/sdk/nemo-gym/index.ts index 892a81b5a138..0294debd3a90 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/index.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/index.ts @@ -48,6 +48,8 @@ export interface CreateNemoGymOptions { * carries a prior trajectory to resume. See language-model.ts's docblock. */ replayTurns?: NemoGymReplayTurn[] + /** Subsequent user messages trailing the last replayed turn. See language-model.ts's docblock. */ + replayTrailingUserTexts?: string[] } export interface NemoGymProvider { @@ -75,6 +77,7 @@ export function createNemoGym(opts: CreateNemoGymOptions): NemoGymProvider { turnCounter: opts.turnCounter, onCompletion: opts.onCompletion, replayTurns: opts.replayTurns, + replayTrailingUserTexts: opts.replayTrailingUserTexts, }) }, } diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 475bd1c9c236..eb72b9ec3a07 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -18,10 +18,12 @@ * openhands' `llm_completions//*.json` exactly so gym's * `get_openhands_trajectory_from_completions` reads it without changes. * - * Replay: when `cfg.replayTurns` is set, the first N calls (main session - * only) are answered from that scripted queue instead of a real HTTP call — - * same synthesized-stream shape as a real response, so opencode's *real* - * tool-execution path (streamText -> the AI-SDK `tool.execute` closures + * Replay: when `cfg.replayTurns` is set, the first N tool-bearing calls on + * the top-level agentic-loop session (see `_popReplayTurn`'s docblock for + * exactly how that's identified — it's not as simple as "sessionID is the + * main session") are answered from that scripted queue instead of a real + * HTTP call — same synthesized-stream shape as a real response, so + * opencode's *real* tool-execution path (streamText -> the AI-SDK `tool.execute` closures * built in session/prompt.ts's resolveTools()) runs each replayed tool call * for real against the live sandbox. This is required for correctness: * SWE-bench agents mutate a git workspace and the final patch comes from @@ -33,6 +35,16 @@ * FIRST dumped completion's cumulative `messages` length, which must be the * first live call (by then the replay prefix is already in session * history), not a scripted one. + * + * Subsequent user messages (anything after the trajectory's first user + * message) are real request content, not an action to replay — there's + * nothing to "re-execute" for a plain user turn. Everything the caller sent + * has to be part of what the model sees, starting with the first live call + * and on every call after that. Since opencode's own session storage isn't + * writable from here (see bench/replay.ts's module docblock), these aren't + * persisted anywhere — `_buildRequestParams` re-splices them into the + * outgoing message list, at a fixed position relative to the replayed + * assistant turns, on every live doStream/doGenerate call. */ import { @@ -111,6 +123,14 @@ const TOKEN_ID_FIELDS = ["prompt_token_ids", "generation_token_ids", "generation export interface NemoGymReplayTurn { content: string | null toolCalls?: Array<{ id: string; name: string; arguments: string }> + /** + * Subsequent user messages that occurred, in the original trajectory, + * immediately before this turn was originally generated. Not replayed as + * part of the scripted turn itself (a user message isn't an action) — + * spliced into every live request's message list once replay reaches this + * point. See the module docblock. + */ + precedingUserTexts?: string[] } export interface NemoGymLanguageModelConfig { @@ -165,6 +185,13 @@ export interface NemoGymLanguageModelConfig { * tool calls must run for real rather than just replaying recorded text. */ replayTurns?: NemoGymReplayTurn[] + /** + * Subsequent user messages after the last replayed assistant turn, with + * nothing recorded after them in the trajectory (trajectory ends on a + * user message). Spliced into every live request's message list, same as + * NemoGymReplayTurn.precedingUserTexts. See the module docblock. + */ + replayTrailingUserTexts?: string[] } // --------------------------------------------------------------------------- @@ -185,6 +212,16 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // Replay only ever applies to the main session (subagent sessions never // existed in the recorded trajectory) — a single counter is sufficient. private replayIndex = 0 + // Precomputed once from cfg.replayTurns[i].precedingUserTexts / + // cfg.replayTrailingUserTexts. `beforeAssistantOrdinal` is the 0-based + // count of assistant-role messages that must already be in the wire + // message list before this text is inserted — i.e. insert right before + // the (beforeAssistantOrdinal + 1)-th assistant message overall, or at + // the end if that many don't exist yet. All replayed assistant turns are + // always fully present in session history by the time any live call + // happens, so this ordinal reliably resolves to the same chronological + // spot on every call — see _injectPendingUserMessages. + private readonly pendingUserInjections: Array<{ beforeAssistantOrdinal: number; text: string }> = [] constructor(modelId: string, cfg: NemoGymLanguageModelConfig) { this.modelId = modelId @@ -194,6 +231,14 @@ export class NemoGymLanguageModel implements LanguageModelV3 { requestTimeoutMs: cfg.requestTimeoutMs ?? 600_000, retries: cfg.retries ?? 3, } + cfg.replayTurns?.forEach((turn, i) => { + for (const text of turn.precedingUserTexts ?? []) { + this.pendingUserInjections.push({ beforeAssistantOrdinal: i, text }) + } + }) + for (const text of cfg.replayTrailingUserTexts ?? []) { + this.pendingUserInjections.push({ beforeAssistantOrdinal: cfg.replayTurns?.length ?? 0, text }) + } } private _nextTurn(sessionID: string): number { @@ -215,13 +260,39 @@ export class NemoGymLanguageModel implements LanguageModelV3 { return { sessionID: sid || "main", parentSessionID: pid } } - // Pops the next scripted turn for the main session, or undefined once the - // replay queue is exhausted / not applicable (subagent sessions, no - // replayTurns configured). Advances state, so call at most once per doStream - // / doGenerate invocation. - private _popReplayTurn(session: { sessionID: string }): NemoGymReplayTurn | undefined { + // Pops the next scripted turn for the main agentic-loop session, or + // undefined once the replay queue is exhausted / not applicable. Advances + // state, so call at most once per doStream / doGenerate invocation. + // + // Two exclusions matter here, both confirmed against a real opencode run + // (not just unit tests against a bare LanguageModelV3CallOptions): + // + // 1. Subagent sessions: excluded via `parentSessionID` being set, NOT via + // comparing `sessionID` to a sentinel string. `session/llm.ts` sets + // `x-session-affinity: input.sessionID` UNCONDITIONALLY for every call + // on the nemo-gym provider — including the top-level/main session's own + // calls — so sessionID is *always* the real session id (e.g. "ses_..."), + // never a fallback "main" placeholder. A `sessionID !== "main"` check + // (the original implementation here) is therefore always true and + // replay never fires at all in a real run — a subagent session is the + // one with `parentSessionID` set, not the one whose id happens to equal + // a literal string. + // 2. Auxiliary no-tool calls on the SAME session: opencode's own `runLoop` + // forks off title-generation and summary-generation model calls on step + // 1 (`session/prompt.ts`), using the same session id as the real + // agentic loop. Those race against the loop's own first call and, if + // unfiltered, silently consume scripted turns meant for the real agent + // (confirmed: they reach this provider before the real loop's first + // call in practice). They're reliably distinguishable because they + // never pass `tools` — only the real agentic loop resolves and sends + // the tool registry — so `hasTools` gates them out. + private _popReplayTurn( + session: { sessionID: string; parentSessionID: string | undefined }, + hasTools: boolean, + ): NemoGymReplayTurn | undefined { if (!this.cfg.replayTurns) return undefined - if (session.sessionID !== "main") return undefined + if (session.parentSessionID) return undefined + if (!hasTools) return undefined if (this.replayIndex >= this.cfg.replayTurns.length) return undefined return this.cfg.replayTurns[this.replayIndex++] } @@ -295,6 +366,39 @@ export class NemoGymLanguageModel implements LanguageModelV3 { } } + // Re-splices subsequent-user-message text from the replay trajectory into + // the outgoing message list, in place. Processed in descending ordinal + // order so inserting a later text doesn't shift the index about to be + // looked up for an earlier one. + private _injectPendingUserMessages(messages: ChatRequestMessage[]): void { + if (!this.pendingUserInjections.length) return + const byOrdinal = new Map() + for (const { beforeAssistantOrdinal, text } of this.pendingUserInjections) { + const list = byOrdinal.get(beforeAssistantOrdinal) ?? [] + list.push(text) + byOrdinal.set(beforeAssistantOrdinal, list) + } + const ordinals = [...byOrdinal.keys()].sort((a, b) => b - a) + for (const ordinal of ordinals) { + const idx = this._findAssistantOrdinalIndex(messages, ordinal) + const texts = byOrdinal.get(ordinal)! + messages.splice(idx, 0, ...texts.map((text) => ({ role: "user" as const, content: text }))) + } + } + + // Index right before the `ordinal`-th (0-based) assistant-role message, or + // messages.length if fewer than `ordinal` assistant messages exist yet. + private _findAssistantOrdinalIndex(messages: ChatRequestMessage[], ordinal: number): number { + let seen = 0 + for (let i = 0; i < messages.length; i++) { + if (messages[i].role === "assistant") { + if (seen === ordinal) return i + seen++ + } + } + return messages.length + } + get supportedUrls() { return {} as Record } @@ -303,7 +407,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // implement doGenerate for completeness / future direct-use. async doGenerate(options: LanguageModelV3CallOptions) { const session = this._sessionFromHeaders(options.headers) - const replayTurn = this._popReplayTurn(session) + const replayTurn = this._popReplayTurn(session, Boolean(options.tools && options.tools.length > 0)) if (replayTurn) { const msg = this._messageFromReplayTurn(replayTurn) @@ -377,7 +481,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { async doStream(options: LanguageModelV3CallOptions) { const session = this._sessionFromHeaders(options.headers) - const replayTurn = this._popReplayTurn(session) + const replayTurn = this._popReplayTurn(session, Boolean(options.tools && options.tools.length > 0)) if (replayTurn) { const msg = this._messageFromReplayTurn(replayTurn) @@ -506,6 +610,16 @@ export class NemoGymLanguageModel implements LanguageModelV3 { } } + // Trajectory resume: splice in any subsequent user messages from the + // replay trajectory (see bench/replay.ts + the module docblock for why + // these can't just be persisted into opencode's own session storage). + // Done here, before loggedMessages is cloned from messages below, so the + // wire request and the trajectory dump gym reads both include them + // identically. Re-applied on every call — nothing else carries them + // forward — at a position fixed relative to the replayed assistant + // turns, so they land in the same chronological spot every time. + this._injectPendingUserMessages(messages as ChatRequestMessage[]) + // Token-ID handling, mirroring OpenHands' nemo_gym_client.py exactly: // - WIRE request: token IDs on the MOST RECENT assistant message only // (the last turn's prompt_token_ids embed the exact token stream of diff --git a/packages/opencode/test/bench/replay.test.ts b/packages/opencode/test/bench/replay.test.ts index 83fd326b66ff..d60299be2611 100644 --- a/packages/opencode/test/bench/replay.test.ts +++ b/packages/opencode/test/bench/replay.test.ts @@ -32,7 +32,7 @@ describe("parseReplayMessages", () => { expect(replayTurns).toEqual([]) }) - test("skips system, tool, and subsequent user messages", () => { + test("skips system and tool messages", () => { const raw = JSON.stringify([ { role: "system", content: "sys prompt" }, { role: "user", content: "fix the bug" }, @@ -42,7 +42,6 @@ describe("parseReplayMessages", () => { tool_calls: [{ id: "call_1", type: "function", function: { name: "bash", arguments: '{"cmd":"ls"}' } }], }, { role: "tool", content: "file1.py\n", tool_call_id: "call_1" }, - { role: "user", content: "please continue" }, { role: "assistant", content: "Done.", tool_calls: undefined }, ]) const { initialUserText, replayTurns } = parseReplayMessages(raw) @@ -53,6 +52,67 @@ describe("parseReplayMessages", () => { ]) }) + test("attaches a subsequent user message to the turn it precedes, not dropped", () => { + const raw = JSON.stringify([ + { role: "user", content: "fix the bug" }, + { + role: "assistant", + content: null, + tool_calls: [{ id: "call_1", type: "function", function: { name: "bash", arguments: '{"cmd":"ls"}' } }], + }, + { role: "tool", content: "file1.py\n", tool_call_id: "call_1" }, + { role: "user", content: "please also fix the other bug" }, + { role: "assistant", content: "Done.", tool_calls: undefined }, + ]) + const { initialUserText, replayTurns } = parseReplayMessages(raw) + expect(initialUserText).toBe("fix the bug") + expect(replayTurns).toEqual([ + { content: null, toolCalls: [{ id: "call_1", name: "bash", arguments: '{"cmd":"ls"}' }] }, + { content: "Done.", toolCalls: undefined, precedingUserTexts: ["please also fix the other bug"] }, + ]) + }) + + test("collects multiple consecutive subsequent user messages onto the same turn, in order", () => { + const raw = JSON.stringify([ + { role: "user", content: "fix the bug" }, + { role: "assistant", content: "ok" }, + { role: "user", content: "also do X" }, + { role: "user", content: "and Y" }, + { role: "assistant", content: "done" }, + ]) + const { replayTurns } = parseReplayMessages(raw) + expect(replayTurns[1].precedingUserTexts).toEqual(["also do X", "and Y"]) + }) + + test("returns trailing user messages separately when the trajectory ends on a user turn", () => { + const raw = JSON.stringify([ + { role: "user", content: "fix the bug" }, + { + role: "assistant", + content: null, + tool_calls: [{ id: "call_1", type: "function", function: { name: "bash", arguments: '{"cmd":"ls"}' } }], + }, + { role: "tool", content: "file1.py\n", tool_call_id: "call_1" }, + { role: "user", content: "now also check the tests" }, + ]) + const { replayTurns, trailingUserTexts } = parseReplayMessages(raw) + expect(replayTurns).toEqual([ + { content: null, toolCalls: [{ id: "call_1", name: "bash", arguments: '{"cmd":"ls"}' }] }, + ]) + expect(trailingUserTexts).toEqual(["now also check the tests"]) + }) + + test("omits empty subsequent user message text", () => { + const raw = JSON.stringify([ + { role: "user", content: "fix the bug" }, + { role: "assistant", content: "ok" }, + { role: "user", content: "" }, + { role: "assistant", content: "done" }, + ]) + const { replayTurns } = parseReplayMessages(raw) + expect(replayTurns[1].precedingUserTexts).toBeUndefined() + }) + test("preserves tool_call ids verbatim, including multiple calls in one turn", () => { const raw = JSON.stringify([ { role: "user", content: "fix the bug" }, diff --git a/packages/opencode/test/provider/nemo-gym/language-model.test.ts b/packages/opencode/test/provider/nemo-gym/language-model.test.ts index 31d8299876e6..8d088e654d9d 100644 --- a/packages/opencode/test/provider/nemo-gym/language-model.test.ts +++ b/packages/opencode/test/provider/nemo-gym/language-model.test.ts @@ -13,8 +13,17 @@ async function drain(stream: ReadableStream): Promise return parts } +// Real agentic-loop calls always carry the resolved tool registry +// (session/prompt.ts's resolveTools()) — this is what actually distinguishes +// them from auxiliary same-session calls like title/summary generation, +// which never pass tools. See _popReplayTurn's docblock in language-model.ts. +const AGENT_TOOLS: LanguageModelV3CallOptions["tools"] = [ + { type: "function", name: "bash", inputSchema: { type: "object", properties: {} } }, +] + const CALL_OPTIONS: LanguageModelV3CallOptions = { prompt: [{ role: "user", content: [{ type: "text", text: "hi" }] }], + tools: AGENT_TOOLS, } describe("NemoGymLanguageModel replay", () => { @@ -92,7 +101,7 @@ describe("NemoGymLanguageModel replay", () => { expect(textDelta).toMatchObject({ delta: "live turn" }) }) - test("replay is scoped to the main session — subagent sessions call through to HTTP", async () => { + test("replay is scoped to the top-level session — a subagent session (x-parent-session-id set) calls through to HTTP", async () => { const fetchSpy = mock(async () => new Response( JSON.stringify({ @@ -112,13 +121,168 @@ describe("NemoGymLanguageModel replay", () => { replayTurns: [{ content: "scripted turn" }], }) + // A subagent session carries its OWN session id plus x-parent-session-id + // pointing at the main session — session/llm.ts sets both unconditionally + // for every call, subagent or not, so it's parentSessionID's presence + // that identifies a subagent, not the session id string itself. const subagentOptions: LanguageModelV3CallOptions = { ...CALL_OPTIONS, - headers: { "x-session-affinity": "ses_subagent_1" }, + headers: { "x-session-affinity": "ses_subagent_1", "x-parent-session-id": "ses_main" }, } const { stream } = await model.doStream(subagentOptions) const parts = await drain(stream) expect(fetchSpy).toHaveBeenCalledTimes(1) expect(parts.find((p) => p.type === "text-delta")).toMatchObject({ delta: "subagent turn" }) }) + + test("an auxiliary no-tool call on the main session (e.g. opencode's own title/summary generation) does not consume the replay queue", async () => { + const fetchSpy = mock(async () => + new Response( + JSON.stringify({ + id: "resp_1", + model: "test-model", + choices: [{ finish_reason: "stop", message: { role: "assistant", content: "Fix the parser bug" } }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ) + // @ts-expect-error test override + globalThis.fetch = fetchSpy + + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + replayTurns: [{ content: null, toolCalls: [{ id: "call_1", name: "bash", arguments: "{}" }] }], + }) + + // session/prompt.ts's runLoop forks off a title-generation call on step 1, + // using the SAME session id as the real agentic loop but never passing + // tools. Real x-session-affinity value from opencode: same session, + // no x-parent-session-id (it's not a subagent), no tools. + const titleGenOptions: LanguageModelV3CallOptions = { + prompt: [{ role: "user", content: [{ type: "text", text: "Generate a title" }] }], + headers: { "x-session-affinity": "ses_main" }, + // no tools + } + const { stream } = await model.doStream(titleGenOptions) + await drain(stream) + expect(fetchSpy).toHaveBeenCalledTimes(1) // went straight to HTTP, not replayed + + // The scripted turn is still there for the real agent's own next call + // (same session id, this time with tools). + const realOptions: LanguageModelV3CallOptions = { + ...CALL_OPTIONS, + headers: { "x-session-affinity": "ses_main" }, + } + const realFetchSpy = mock(async () => { + throw new Error("network should not be called — replay turn should still be available") + }) + // @ts-expect-error test override + globalThis.fetch = realFetchSpy + const { stream: realStream } = await model.doStream(realOptions) + const parts = await drain(realStream) + expect(realFetchSpy).not.toHaveBeenCalled() + expect(parts.find((p) => p.type === "tool-call")).toMatchObject({ toolCallId: "call_1" }) + }) + + function fetchSpyCapturingBody() { + const bodies: Array<{ messages: unknown }> = [] + const fetchSpy = mock(async (_url: unknown, init: { body?: string }) => { + bodies.push(JSON.parse(init.body ?? "{}")) + return new Response( + JSON.stringify({ + id: "resp_1", + model: "test-model", + choices: [{ finish_reason: "stop", message: { role: "assistant", content: "live turn" } }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ) + }) + // @ts-expect-error test override + globalThis.fetch = fetchSpy + return { fetchSpy, bodies } + } + + test("subsequent user message (precedingUserTexts) is spliced at the right position relative to already-replayed turns", async () => { + const { bodies } = fetchSpyCapturingBody() + + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + // Attached to turn 0 (ordinal 0): must land BEFORE the first assistant + // message in the eventual wire request, i.e. right after the initial + // user message. + replayTurns: [{ content: null, toolCalls: [{ id: "call_1", name: "bash", arguments: "{}" }], precedingUserTexts: ["please also fix the other bug"] }], + }) + + // Turn 0 replays without hitting the network. + await drain((await model.doStream(CALL_OPTIONS)).stream) + expect(bodies).toHaveLength(0) + + // Live call, with a realistic prompt reflecting what the session would + // actually contain by now: the initial user message, then the assistant + // turn + tool result that were just replayed for real. + const grownPrompt: LanguageModelV3CallOptions["prompt"] = [ + { role: "user", content: [{ type: "text", text: "fix the bug" }] }, + { role: "assistant", content: [{ type: "tool-call", toolCallId: "call_1", toolName: "bash", input: {} }] }, + { + role: "tool", + content: [ + { type: "tool-result", toolCallId: "call_1", toolName: "bash", output: { type: "text", value: "ok" } }, + ], + }, + ] + await drain((await model.doStream({ prompt: grownPrompt, tools: AGENT_TOOLS })).stream) + expect(bodies).toHaveLength(1) + + const messages = bodies[0].messages as unknown as Array<{ role: string; content?: unknown }> + expect(messages[0]).toMatchObject({ role: "user", content: "fix the bug" }) + const injectedIdx = messages.findIndex((m) => m.role === "user" && m.content === "please also fix the other bug") + const firstAssistantIdx = messages.findIndex((m) => m.role === "assistant") + expect(injectedIdx).toBeGreaterThan(-1) + expect(firstAssistantIdx).toBeGreaterThan(-1) + // Injected immediately before the assistant turn it originally + // preceded, not appended somewhere arbitrary — and strictly after the + // initial task-instruction message. + expect(injectedIdx).toBe(firstAssistantIdx - 1) + expect(injectedIdx).toBeGreaterThan(0) + }) + + test("injected user message persists on every subsequent live call, not just the first", async () => { + const { bodies } = fetchSpyCapturingBody() + + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + replayTurns: [{ content: "ok", precedingUserTexts: ["please also fix the other bug"] }], + }) + + await drain((await model.doStream(CALL_OPTIONS)).stream) // scripted + await drain((await model.doStream(CALL_OPTIONS)).stream) // live #1 + await drain((await model.doStream(CALL_OPTIONS)).stream) // live #2 + + expect(bodies).toHaveLength(2) + for (const body of bodies) { + const messages = body.messages as Array<{ role: string; content?: unknown }> + expect(messages.some((m) => m.role === "user" && m.content === "please also fix the other bug")).toBe(true) + } + }) + + test("replayTrailingUserTexts is appended once the replay queue is fully drained", async () => { + const { bodies } = fetchSpyCapturingBody() + + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + replayTurns: [{ content: null, toolCalls: [{ id: "call_1", name: "bash", arguments: "{}" }] }], + replayTrailingUserTexts: ["now also check the tests"], + }) + + await drain((await model.doStream(CALL_OPTIONS)).stream) // scripted + await drain((await model.doStream(CALL_OPTIONS)).stream) // live + + expect(bodies).toHaveLength(1) + const messages = bodies[0].messages as unknown as Array<{ role: string; content?: unknown }> + expect(messages[messages.length - 1]).toMatchObject({ role: "user", content: "now also check the tests" }) + }) }) From f2d4351d02f71ddfdadbd5723690fb0622161165 Mon Sep 17 00:00:00 2001 From: sdevare-nv Date: Tue, 21 Jul 2026 15:36:21 -0700 Subject: [PATCH 38/49] Enhance bash permission settings Expanded permission settings for bash commands to deny potentially harmful actions. --- packages/opencode/src/bench/cli.ts | 103 +++++++++++++++++++++++++++-- 1 file changed, 98 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 2b47ec452def..c1374c03a383 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -202,12 +202,105 @@ async function buildConfigDir(args: { // Allow the read+write tool set; disable web/skill/task to keep the // agent focused on local code editing. permission: { - // Glob-keyed `PermissionActionConfig` for file/shell access. edit: { "**": "allow" }, - bash: { "*": "allow" }, - // webfetch / websearch use a different schema (single action, not - // a glob map) and we already disable them in `tools` below — no - // need for an explicit entry here. + bash: { + "*": "allow", + + // process termination + "killall*": "deny", + "pkill*": "deny", + "kill -1*": "deny", + "kill 0*": "deny", + + // filesystem destruction + "rm -rf /": "deny", + "rm -rf /*": "deny", + "rm -rf /bin*": "deny", + "rm -rf /usr*": "deny", + "rm -rf /etc*": "deny", + "rm -rf /var*": "deny", + "rm -rf /home*": "deny", + "rm -rf /root*": "deny", + "rm -rf /opt*": "deny", + "rm -rf /lib*": "deny", + "rm -rf /lib64*": "deny", + "rm -rf /sbin*": "deny", + "rm -rf /boot*": "deny", + "rm -rf /dev*": "deny", + "rm -rf /proc*": "deny", + "rm -rf /sys*": "deny", + + // system control + "shutdown*": "deny", + "reboot*": "deny", + "poweroff*": "deny", + "halt*": "deny", + "init 0*": "deny", + "init 6*": "deny", + + // disk devices + "dd *of=/dev/sd*": "deny", + "dd *of=/dev/nvme*": "deny", + "dd *of=/dev/hd*": "deny", + "dd *of=/dev/null*": "deny", + + // git network + "git fetch*": "deny", + "git pull*": "deny", + "git clone*": "deny", + "git ls-remote*": "deny", + "git remote add*": "deny", + "git remote set-url*": "deny", + "git remote set-head*": "deny", + "git remote update*": "deny", + "git remote rename*": "deny", + "git remote set-branches*": "deny", + "git submodule add*": "deny", + "git submodule update*": "deny", + "git submodule sync*": "deny", + "git submodule init*": "deny", + "git archive*--remote*": "deny", + "git *://*": "deny", + "git *@*:*": "deny", + + // git history mining + "git log*--all*": "deny", + "git log*--branches*": "deny", + "git log*--remotes*": "deny", + "git log*--walk-reflogs*": "deny", + "git log*--grep*": "deny", + "git rev-list*--all*": "deny", + "git rev-list*--branches*": "deny", + "git rev-list*--remotes*": "deny", + "git rev-list*--grep*": "deny", + "git shortlog*--all*": "deny", + "git reflog*": "deny", + "git cat-file*": "deny", + "git fsck*": "deny", + "git verify-pack*": "deny", + "git unpack-objects*": "deny", + "git cherry*": "deny", + "git show*": "deny", + "git merge-base*--is-ancestor*": "deny", + "git branch*--contains*": "deny", + "git tag*--contains*": "deny", + "git for-each-ref*--contains*": "deny", + + // git internals (substring match on path) + "*.git/logs*": "deny", + "*.git/packed-refs*": "deny", + "*.git/ORIG_HEAD*": "deny", + "*.git/FETCH_HEAD*": "deny", + "*.git/refs*": "deny", + + // online lookups + "curl *github.com*": "deny", + "wget *github.com*": "deny", + "curl *githubusercontent.com*": "deny", + "wget *githubusercontent.com*": "deny", + "curl *github.io*": "deny", + "wget *github.io*": "deny" + } }, tools: { bash: true, From e7be2d886da0395154c9688ff52a0f3b933f9237 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Sun, 2 Aug 2026 19:02:51 +0000 Subject: [PATCH 39/49] bench: add --patch-mode committed for tasks that ask the agent to commit Some SWE task families end their problem statement with an instruction to work on a new branch and commit everything when done. The agent obeys, leaving a clean working tree, so the existing `git diff` capture returns nothing and the rollout is recorded as a 0-byte patch. What non-empty patches do survive tend to be uncommitted build artifacts (build/CMakeFiles, *.o, *.a, .ninja_log) rather than the solution. Add a second capture mode, selected by --patch-mode (gym passes it through the PATCH_MODE env var). `worktree` is the default and is the historical behaviour moved verbatim out of cli.ts. `committed` diffs the pre-agent HEAD against the most-advanced commit the agent left, searching HEAD *and* every local branch so work still lands when the agent commits on a side branch and checks main back out. Uncommitted leftovers are excluded and logged. The result is still a plain baseline -> final-tree diff, so the eval side (git reset --hard + git apply) is unchanged in either mode. Also sets a repo-local git identity in committed mode when the image ships none, so the agent's first commit can't die with "Author identity unknown". Co-Authored-By: Claude Opus 5 (1M context) --- .../benchmarks/swe_bench/scripts/run_infer.sh | 10 + packages/opencode/src/bench/cli.ts | 58 ++--- packages/opencode/src/bench/patch.ts | 218 ++++++++++++++++++ packages/opencode/test/bench/patch.test.ts | 154 +++++++++++++ 4 files changed, 407 insertions(+), 33 deletions(-) create mode 100644 packages/opencode/src/bench/patch.ts create mode 100644 packages/opencode/test/bench/patch.test.ts diff --git a/evaluation/benchmarks/swe_bench/scripts/run_infer.sh b/evaluation/benchmarks/swe_bench/scripts/run_infer.sh index 2edf91b721cc..d6b14916000d 100755 --- a/evaluation/benchmarks/swe_bench/scripts/run_infer.sh +++ b/evaluation/benchmarks/swe_bench/scripts/run_infer.sh @@ -23,6 +23,11 @@ # COMMAND_EXEC_TIMEOUT per-bash-command timeout in seconds # DIVERSIFY_TOOL_NAMES optional: rename tools for RL diversity # CAMEL_CASE_TOOL_NAMES optional: camelCase tool names +# PATCH_MODE optional: how the model patch is extracted. +# `worktree` (default) = `git diff` of the +# working tree; `committed` = diff of what the +# agent committed, for task families whose +# prompt asks the agent to commit its work. set -eo pipefail @@ -91,6 +96,7 @@ echo "INSTANCE_DICT_PATH: $INSTANCE_DICT_PATH" echo "CONFIG_FILE: $CONFIG_FILE" echo "WORKSPACE_ROOT: $WORKSPACE_ROOT" echo "USER_MESSAGE_PATH: $USER_MESSAGE_PATH" +echo "PATCH_MODE: ${PATCH_MODE:-worktree (default)}" echo "SYSTEM_PROMPT_PATH: $SYSTEM_PROMPT_PATH" echo "MODEL_SERVER: $NEMO_GYM_MODEL_SERVER_NAME @ $NEMO_GYM_MODEL_SERVER_BASE_URL" @@ -113,6 +119,10 @@ fi if [ "${ENABLE_SUBAGENTS:-0}" = "1" ] || [ "${ENABLE_SUBAGENTS:-}" = "true" ]; then cmd+=(--enable-subagents) fi +# Omitted entirely when unset so cli.ts keeps its own default (`worktree`). +if [ -n "${PATCH_MODE:-}" ]; then + cmd+=(--patch-mode "$PATCH_MODE") +fi echo "Executing: ${cmd[*]}" exec "${cmd[@]}" diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index c1374c03a383..2bbd152fafb9 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -16,7 +16,8 @@ * * Trajectory capture: the nemo-gym provider (registered via this config) * writes `/.json` per LLM call BEFORE returning. On - * exit we capture `git diff` and write `output.jsonl`. + * exit we capture the model patch (see ./patch.ts for the two modes) and write + * `output.jsonl`. */ import { existsSync, promises as fs, readFileSync } from "node:fs" @@ -25,6 +26,7 @@ import os from "node:os" import { spawn } from "node:child_process" import { runDeepReset } from "./deep_reset" import { bootstrapRepoIfMissing } from "./bootstrap_repo" +import { capturePatch, ensureCommitIdentity, parsePatchMode, recordBaselineCommit, type PatchMode } from "./patch" // opencode's built-in anthropic system prompt — Bun bundles .txt as a string. // Used as the default when no --system-prompt override is passed. import PROMPT_ANTHROPIC from "../session/prompt/anthropic.txt" @@ -45,6 +47,8 @@ interface CliArgs { systemPromptPath?: string /** Enable opencode's `task` tool (spawns subagent sessions). */ enableSubagents: boolean + /** How the model patch is extracted at the end of the run. See ./patch.ts. */ + patchMode: PatchMode } function parseArgs(argv: string[]): CliArgs { @@ -54,6 +58,7 @@ function parseArgs(argv: string[]): CliArgs { dataset: "", split: "test", enableSubagents: false, + patchMode: parsePatchMode(undefined), } for (let i = 0; i < argv.length; i++) { const a = argv[i] @@ -95,6 +100,9 @@ function parseArgs(argv: string[]): CliArgs { case "--enable-subagents": out.enableSubagents = true break + case "--patch-mode": + out.patchMode = parsePatchMode(next()) + break default: if (a.startsWith("--")) throw new Error(`Unknown flag: ${a}`) } @@ -330,16 +338,6 @@ async function buildConfigDir(args: { return { tmpRoot, configFile } } -// Some SIFs ship with bare PATH lookups that ENOENT on bare program names -// through Bun's posix_spawn. Resolve to an absolute path up front for any -// binary we shell out to. -function detectBin(candidates: string[]): string | null { - for (const p of candidates) { - if (existsSync(p)) return p - } - return null -} - function runOpencode(args: { workspaceRoot: string modelName: string @@ -425,25 +423,6 @@ function runOpencode(args: { }) } -async function captureGitDiff(workspaceRoot: string): Promise { - const gitPath = detectBin(["/usr/bin/git", "/bin/git", "/usr/local/bin/git"]) ?? "git" - const runGit = (args: string[], capture: boolean): Promise => - new Promise((resolve) => { - const child = spawn(gitPath, ["-C", workspaceRoot, ...args], { - env: { ...process.env, GIT_PAGER: "cat" }, - }) - let stdout = "" - if (capture) child.stdout?.on("data", (b) => (stdout += b.toString("utf8"))) - child.on("close", () => resolve(stdout)) - child.on("error", () => resolve("")) - }) - // Mark untracked files as intent-to-add so newly-created files appear in - // `git diff` without being committed. Plain `git diff` only shows changes - // to tracked files, which silently drops new-file patches the agent wrote. - await runGit(["add", "-AN"], false) - return runGit(["diff", "--binary"], true) -} - interface OutputJsonl { instance_id: string test_result: { git_patch: string } @@ -538,7 +517,7 @@ async function main() { } // Bootstrap a git repo if the SIF shipped a flat source tree (swe-bench-ext - // and some SWE-rebench variants). Without this, captureGitDiff returns "" + // and some SWE-rebench variants). Without this, the patch capture returns "" // and every patch is recorded as 0 bytes. const { freshInit } = await bootstrapRepoIfMissing(workspaceRoot) @@ -551,6 +530,16 @@ async function main() { await runDeepReset(workspaceRoot, String(instance.base_commit ?? "")) } + // Snapshot the pristine HEAD *after* bootstrap/deep-reset — it is the diff + // base for `--patch-mode committed`. Recorded unconditionally so the log + // always shows what the agent started from. + const baselineCommit = await recordBaselineCommit(workspaceRoot) + console.log(`[bench] patch_mode=${args.patchMode} baseline=${baselineCommit || ""}`) + if (args.patchMode === "committed") { + // The agent is expected to commit; make sure git will let it. + await ensureCommitIdentity(workspaceRoot) + } + const opencodeBin = detectOpencodeBin() const result = await runOpencode({ workspaceRoot, @@ -561,7 +550,7 @@ async function main() { agent: "swe-bench", }) - const patch = await captureGitDiff(workspaceRoot) + const patch = await capturePatch(workspaceRoot, args.patchMode, baselineCommit) const benchRunTime = (Date.now() - startedAt) / 1000 const error: string | null = result.exitCode === 0 ? null : `opencode_exit_${result.exitCode}` @@ -572,11 +561,14 @@ async function main() { metrics: { bench_run_time: benchRunTime, opencode_exit_code: result.exitCode, + patch_mode: args.patchMode, }, error, }) - console.log(`[bench] wrote ${outPath} (patch=${patch.length} bytes, error=${error ?? "none"})`) + console.log( + `[bench] wrote ${outPath} (patch=${patch.length} bytes, mode=${args.patchMode}, error=${error ?? "none"})`, + ) // Mirror opencode's exit code explicitly. Falling off the end of main() and // letting Bun drain the event loop produced a flaky exit=1 even when the diff --git a/packages/opencode/src/bench/patch.ts b/packages/opencode/src/bench/patch.ts new file mode 100644 index 000000000000..135ba335028b --- /dev/null +++ b/packages/opencode/src/bench/patch.ts @@ -0,0 +1,218 @@ +/** + * Model-patch capture for the bench driver. + * + * Two modes, selected by `--patch-mode` (gym passes it through the + * `PATCH_MODE` env var in run_infer.sh): + * + * - `worktree` (DEFAULT, the historical behaviour): mark untracked files + * intent-to-add and take `git diff` of the working tree. Correct for the + * SWE-bench-style prompts that explicitly tell the agent *not* to commit. + * + * - `committed`: ignore the working tree and extract what the agent + * COMMITTED. Required by task families whose problem statement ends with + * "work on this in a new branch from main and commit everything when you + * are done" (e.g. the DeepSWE set). There, the agent commits its solution, + * leaving a clean tree — `git diff` returns "" and every rollout would be + * recorded as a 0-byte patch. + * + * In `committed` mode we diff `baseline..tip`, where `baseline` is the HEAD sha + * captured *before* the agent starts (after bootstrap/deep-reset, so it is the + * dataset's base commit) and `tip` is the most-advanced commit the agent left + * behind. `tip` is searched across HEAD *and* every local branch, because the + * agent may commit on a side branch and then switch back to main — HEAD alone + * would silently yield an empty patch. + * + * The resulting patch is still a plain `baseline -> final tree` unified diff, + * so the eval side (`git reset --hard ` + `git apply`) is + * unchanged regardless of mode. + */ + +import { spawn } from "node:child_process" +import { existsSync } from "node:fs" + +export type PatchMode = "worktree" | "committed" + +export const PATCH_MODES: PatchMode[] = ["worktree", "committed"] + +export const DEFAULT_PATCH_MODE: PatchMode = "worktree" + +export function parsePatchMode(raw: string | undefined): PatchMode { + if (!raw) return DEFAULT_PATCH_MODE + const v = raw.trim().toLowerCase() + if ((PATCH_MODES as string[]).includes(v)) return v as PatchMode + throw new Error(`Invalid --patch-mode "${raw}" (expected one of: ${PATCH_MODES.join(", ")})`) +} + +// git's canonical empty-tree object id. Used as the diff base when the repo has +// no commits at all (unborn HEAD), so a first commit still produces a patch. +const EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + +// Same rationale as cli.ts/deep_reset.ts: some SIFs ENOENT on bare program +// names through Bun's posix_spawn, so resolve an absolute path up front. +function detectGit(): string { + for (const p of ["/usr/bin/git", "/bin/git", "/usr/local/bin/git"]) { + if (existsSync(p)) return p + } + return "git" +} + +interface GitResult { + stdout: string + exitCode: number +} + +function git(workspaceRoot: string, args: string[]): Promise { + const gitPath = detectGit() + return new Promise((resolve) => { + const child = spawn(gitPath, ["-C", workspaceRoot, ...args], { + env: { ...process.env, GIT_PAGER: "cat" }, + stdio: ["ignore", "pipe", "pipe"], + }) + let stdout = "" + child.stdout?.on("data", (b) => (stdout += b.toString("utf8"))) + // Swallow stderr: every call here is best-effort and a noisy `git` on a + // half-broken repo must not pollute the gym log or fail the rollout. + child.stderr?.on("data", () => {}) + child.on("close", (code) => resolve({ stdout, exitCode: code ?? 0 })) + child.on("error", () => resolve({ stdout: "", exitCode: 1 })) + }) +} + +/** + * Snapshot HEAD before the agent runs. Returns "" when the repo has no commit + * yet (unborn HEAD) or isn't a repo at all; `capturePatch` then falls back to + * the empty tree. + */ +export async function recordBaselineCommit(workspaceRoot: string): Promise { + const res = await git(workspaceRoot, ["rev-parse", "--verify", "--quiet", "HEAD^{commit}"]) + return res.exitCode === 0 ? res.stdout.trim() : "" +} + +/** + * Give git a committer identity if the image doesn't ship one. Without this the + * agent's very first `git commit` dies with "Author identity unknown" and the + * whole rollout scores zero for a reason that has nothing to do with the model. + * Repo-local (`--local`) so we don't mutate anything outside the workspace, and + * only when unset, so a task-provided identity always wins. + */ +export async function ensureCommitIdentity(workspaceRoot: string): Promise { + for (const [key, value] of [ + ["user.email", "agent@opencode.local"], + ["user.name", "opencode agent"], + ]) { + const existing = await git(workspaceRoot, ["config", "--get", key]) + if (existing.exitCode === 0 && existing.stdout.trim()) continue + await git(workspaceRoot, ["config", "--local", key, value]) + } +} + +async function worktreePatch(workspaceRoot: string): Promise { + // Mark untracked files as intent-to-add so newly-created files appear in + // `git diff` without being committed. Plain `git diff` only shows changes + // to tracked files, which silently drops new-file patches the agent wrote. + await git(workspaceRoot, ["add", "-AN"]) + const res = await git(workspaceRoot, ["diff", "--binary"]) + return res.stdout +} + +interface Candidate { + /** commit sha of the ref tip */ + sha: string + /** human label for logging: "HEAD" or the branch name */ + label: string + /** commits reachable from `sha` but not from the baseline */ + ahead: number +} + +/** + * Every commit the agent could have left its work on: HEAD (covers detached + * HEAD and "still on the branch it committed to") plus every local branch + * (covers "committed on a side branch, then checked main back out"). + */ +async function candidateTips(workspaceRoot: string, baseline: string): Promise { + const tips: { sha: string; label: string }[] = [] + + const head = await git(workspaceRoot, ["rev-parse", "--verify", "--quiet", "HEAD^{commit}"]) + if (head.exitCode === 0 && head.stdout.trim()) tips.push({ sha: head.stdout.trim(), label: "HEAD" }) + + const branches = await git(workspaceRoot, ["for-each-ref", "--format=%(objectname) %(refname:short)", "refs/heads"]) + for (const line of branches.stdout.split("\n")) { + const [sha, ...rest] = line.trim().split(" ") + if (!sha) continue + if (tips.some((t) => t.sha === sha)) continue // HEAD already covers this tip + tips.push({ sha, label: rest.join(" ") || sha.slice(0, 8) }) + } + + const out: Candidate[] = [] + for (const tip of tips) { + // `baseline..tip` counts commits reachable from tip but not from baseline. + // Deliberately NOT gated on `merge-base --is-ancestor`: an agent that + // amended or rebased its work leaves a tip that no longer descends from + // baseline, and we still want that work. + const res = await git(workspaceRoot, ["rev-list", "--count", `${baseline}..${tip.sha}`]) + const ahead = res.exitCode === 0 ? parseInt(res.stdout.trim(), 10) : 0 + out.push({ ...tip, ahead: Number.isFinite(ahead) ? ahead : 0 }) + } + return out +} + +/** Count of dirty/untracked paths, reported so a dropped worktree is visible in the log. */ +async function dirtyPathCount(workspaceRoot: string): Promise { + const res = await git(workspaceRoot, ["status", "--porcelain", "--untracked-files=all"]) + return res.stdout.split("\n").filter((l) => l.trim()).length +} + +async function committedPatch(workspaceRoot: string, baselineCommit: string): Promise { + const baseline = baselineCommit || EMPTY_TREE + if (!baselineCommit) { + console.log(`[bench] patch_mode=committed: no pre-run HEAD; diffing against the empty tree`) + } + + const candidates = await candidateTips(workspaceRoot, baseline) + // Most commits past the baseline wins. `candidateTips` puts HEAD first, and + // Array.prototype.sort is stable in every JS engine we ship on, so HEAD wins + // ties against a side branch holding the identical work. + const ranked = candidates.filter((c) => c.ahead > 0).sort((a, b) => b.ahead - a.ahead) + const dirty = await dirtyPathCount(workspaceRoot) + + if (ranked.length === 0) { + console.log( + `[bench] patch_mode=committed: agent left no commit past baseline ${baseline.slice(0, 8)} ` + + `(refs checked: ${candidates.length}, uncommitted paths: ${dirty}) -> empty patch`, + ) + return "" + } + + const chosen = ranked[0]! + const others = ranked + .slice(1) + .map((c) => `${c.label}+${c.ahead}`) + .join(",") + console.log( + `[bench] patch_mode=committed: baseline=${baseline.slice(0, 8)} tip=${chosen.label}@${chosen.sha.slice(0, 8)} ` + + `commits=${chosen.ahead} uncommitted_paths=${dirty}${others ? ` other_refs=[${others}]` : ""}`, + ) + if (dirty > 0) { + console.log( + `[bench] patch_mode=committed: ${dirty} uncommitted path(s) are NOT in the patch ` + + `(the task asked the agent to commit its work)`, + ) + } + + const res = await git(workspaceRoot, ["diff", "--binary", baseline, chosen.sha]) + return res.stdout +} + +/** + * Produce the model patch for `mode`. + * + * @param baselineCommit HEAD sha captured before the agent ran; only used by + * `committed` mode. + */ +export async function capturePatch( + workspaceRoot: string, + mode: PatchMode, + baselineCommit: string = "", +): Promise { + return mode === "committed" ? committedPatch(workspaceRoot, baselineCommit) : worktreePatch(workspaceRoot) +} diff --git a/packages/opencode/test/bench/patch.test.ts b/packages/opencode/test/bench/patch.test.ts new file mode 100644 index 000000000000..5aae8f5b1f2e --- /dev/null +++ b/packages/opencode/test/bench/patch.test.ts @@ -0,0 +1,154 @@ +import { $ } from "bun" +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import os from "os" +import path from "path" +import { capturePatch, ensureCommitIdentity, parsePatchMode, recordBaselineCommit } from "../../src/bench/patch" + +// Deliberately NOT using ../fixture/fixture: src/bench/* is a standalone leaf +// (node:child_process + node:fs only) that runs inside minimal SIF images, and +// this suite should stay runnable without booting the instance/effect stack. +const created: string[] = [] + +afterEach(async () => { + await Promise.all(created.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }).catch(() => {}))) +}) + +async function repo(withBaseline = true) { + const dir = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "bench-patch-"))) + created.push(dir) + await $`git init -q -b main`.cwd(dir).quiet() + await $`git config user.email bench@opencode.local`.cwd(dir).quiet() + await $`git config user.name bench`.cwd(dir).quiet() + await $`git config commit.gpgsign false`.cwd(dir).quiet() + if (withBaseline) { + await Bun.write(path.join(dir, "app.py"), "def solve():\n return 0\n") + await commit(dir, "baseline") + } + return dir +} + +async function commit(dir: string, message: string) { + await $`git add -A`.cwd(dir).quiet() + await $`git commit -q -m ${message}`.cwd(dir).quiet() +} + +describe("bench patch mode", () => { + test("parsePatchMode defaults to worktree and rejects garbage", () => { + expect(parsePatchMode(undefined)).toBe("worktree") + expect(parsePatchMode("")).toBe("worktree") + expect(parsePatchMode("Committed")).toBe("committed") + expect(() => parsePatchMode("staged")).toThrow() + }) + + test("worktree mode captures uncommitted edits and new files", async () => { + const dir = await repo() + const baseline = await recordBaselineCommit(dir) + + await Bun.write(path.join(dir, "app.py"), "def solve():\n return 1\n") + await Bun.write(path.join(dir, "new_file.py"), "X = 1\n") + + const patch = await capturePatch(dir, "worktree", baseline) + expect(patch).toContain("+ return 1") + expect(patch).toContain("new_file.py") + }) + + test("worktree mode misses work the agent committed (the DeepSWE failure)", async () => { + const dir = await repo() + const baseline = await recordBaselineCommit(dir) + + await Bun.write(path.join(dir, "app.py"), "def solve():\n return 1\n") + await commit(dir, "fix") + + expect(await capturePatch(dir, "worktree", baseline)).toBe("") + }) + + test("committed mode captures a side-branch commit from either HEAD position", async () => { + const dir = await repo() + const baseline = await recordBaselineCommit(dir) + + await $`git checkout -q -b fix/solve`.cwd(dir).quiet() + await Bun.write(path.join(dir, "app.py"), "def solve():\n return 1\n") + await Bun.write(path.join(dir, "new_file.py"), "X = 1\n") + await commit(dir, "fix") + + const onBranch = await capturePatch(dir, "committed", baseline) + expect(onBranch).toContain("+ return 1") + expect(onBranch).toContain("new_file.py") + + // Agent switched back to main after committing: HEAD sits at the baseline, + // so only the local-branch scan finds the work. + await $`git checkout -q main`.cwd(dir).quiet() + expect(await capturePatch(dir, "committed", baseline)).toBe(onBranch) + }) + + test("committed mode picks the ref with the most commits past baseline", async () => { + const dir = await repo() + const baseline = await recordBaselineCommit(dir) + + await $`git checkout -q -b scratch`.cwd(dir).quiet() + await Bun.write(path.join(dir, "scratch.txt"), "debug\n") + await commit(dir, "scratch") + + await $`git checkout -q -b fix/solve main`.cwd(dir).quiet() + for (const n of [1, 2]) { + await Bun.write(path.join(dir, "app.py"), `def solve():\n return ${n}\n`) + await commit(dir, `step${n}`) + } + await $`git checkout -q main`.cwd(dir).quiet() + + const patch = await capturePatch(dir, "committed", baseline) + expect(patch).toContain("+ return 2") + expect(patch).not.toContain("scratch.txt") + }) + + test("committed mode excludes uncommitted leftovers", async () => { + const dir = await repo() + const baseline = await recordBaselineCommit(dir) + + await Bun.write(path.join(dir, "app.py"), "def solve():\n return 1\n") + await commit(dir, "fix") + await Bun.write(path.join(dir, "repro_scratch.py"), "print('debug')\n") + + const patch = await capturePatch(dir, "committed", baseline) + expect(patch).toContain("+ return 1") + expect(patch).not.toContain("repro_scratch.py") + }) + + test("committed mode yields an empty patch when the agent never committed", async () => { + const dir = await repo() + const baseline = await recordBaselineCommit(dir) + + await Bun.write(path.join(dir, "app.py"), "def solve():\n return 1\n") + + expect(await capturePatch(dir, "committed", baseline)).toBe("") + }) + + test("committed mode falls back to the empty tree for an unborn HEAD", async () => { + const dir = await repo(false) + const baseline = await recordBaselineCommit(dir) + expect(baseline).toBe("") + + await Bun.write(path.join(dir, "app.py"), "X = 1\n") + await commit(dir, "first") + + const patch = await capturePatch(dir, "committed", baseline) + expect(patch).toContain("app.py") + expect(patch).toContain("+X = 1") + }) + + test("ensureCommitIdentity fills a missing identity and keeps an existing one", async () => { + const dir = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "bench-patch-"))) + created.push(dir) + await $`git init -q -b main`.cwd(dir).quiet() + // A user-level identity would mask the "missing" case on a dev machine. + await $`git config --local user.useConfigOnly true`.cwd(dir).quiet() + + await ensureCommitIdentity(dir) + expect((await $`git config --get user.email`.cwd(dir).quiet().text()).trim()).not.toBe("") + + await $`git config --local user.email task@example.com`.cwd(dir).quiet() + await ensureCommitIdentity(dir) + expect((await $`git config --get user.email`.cwd(dir).quiet().text()).trim()).toBe("task@example.com") + }) +}) From cf7d4ee776bc69725e22dfd7f9dfe5a95c29d5f7 Mon Sep 17 00:00:00 2001 From: sdevare-nv Date: Mon, 3 Aug 2026 15:24:17 -0700 Subject: [PATCH 40/49] Refactor deny rules for command patterns --- packages/opencode/src/bench/cli.ts | 140 ++++++++++++++--------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index c1374c03a383..916f6b9b4335 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -207,36 +207,36 @@ async function buildConfigDir(args: { "*": "allow", // process termination - "killall*": "deny", - "pkill*": "deny", - "kill -1*": "deny", - "kill 0*": "deny", + "*killall*": "deny", + "*pkill*": "deny", + "*kill -1*": "deny", + "*kill 0*": "deny", // filesystem destruction - "rm -rf /": "deny", - "rm -rf /*": "deny", - "rm -rf /bin*": "deny", - "rm -rf /usr*": "deny", - "rm -rf /etc*": "deny", - "rm -rf /var*": "deny", - "rm -rf /home*": "deny", - "rm -rf /root*": "deny", - "rm -rf /opt*": "deny", - "rm -rf /lib*": "deny", - "rm -rf /lib64*": "deny", - "rm -rf /sbin*": "deny", - "rm -rf /boot*": "deny", - "rm -rf /dev*": "deny", - "rm -rf /proc*": "deny", - "rm -rf /sys*": "deny", + "*rm -rf /": "deny", + "*rm -rf /*": "deny", + "*rm -rf /bin*": "deny", + "*rm -rf /usr*": "deny", + "*rm -rf /etc*": "deny", + "*rm -rf /var*": "deny", + "*rm -rf /home*": "deny", + "*rm -rf /root*": "deny", + "*rm -rf /opt*": "deny", + "*rm -rf /lib*": "deny", + "*rm -rf /lib64*": "deny", + "*rm -rf /sbin*": "deny", + "*rm -rf /boot*": "deny", + "*rm -rf /dev*": "deny", + "*rm -rf /proc*": "deny", + "*rm -rf /sys*": "deny", // system control - "shutdown*": "deny", - "reboot*": "deny", - "poweroff*": "deny", - "halt*": "deny", - "init 0*": "deny", - "init 6*": "deny", + // "*shutdown*": "deny", + // "*reboot*": "deny", + // "*poweroff*": "deny", + // "*halt*": "deny", + // "init 0*": "deny", + // "init 6*": "deny", // disk devices "dd *of=/dev/sd*": "deny", @@ -245,46 +245,46 @@ async function buildConfigDir(args: { "dd *of=/dev/null*": "deny", // git network - "git fetch*": "deny", - "git pull*": "deny", - "git clone*": "deny", - "git ls-remote*": "deny", - "git remote add*": "deny", - "git remote set-url*": "deny", - "git remote set-head*": "deny", - "git remote update*": "deny", - "git remote rename*": "deny", - "git remote set-branches*": "deny", - "git submodule add*": "deny", - "git submodule update*": "deny", - "git submodule sync*": "deny", - "git submodule init*": "deny", - "git archive*--remote*": "deny", - "git *://*": "deny", - "git *@*:*": "deny", + "*git fetch*": "deny", + "*git pull*": "deny", + "*git clone*": "deny", + "*git ls-remote*": "deny", + "*git remote add*": "deny", + "*git remote set-url*": "deny", + "*git remote set-head*": "deny", + "*git remote update*": "deny", + "*git remote rename*": "deny", + "*git remote set-branches*": "deny", + "*git submodule add*": "deny", + "*git submodule update*": "deny", + "*git submodule sync*": "deny", + "*git submodule init*": "deny", + "*git archive*--remote*": "deny", + "*git *://*": "deny", + "*git *@*:*": "deny", // git history mining - "git log*--all*": "deny", - "git log*--branches*": "deny", - "git log*--remotes*": "deny", - "git log*--walk-reflogs*": "deny", - "git log*--grep*": "deny", - "git rev-list*--all*": "deny", - "git rev-list*--branches*": "deny", - "git rev-list*--remotes*": "deny", - "git rev-list*--grep*": "deny", - "git shortlog*--all*": "deny", - "git reflog*": "deny", - "git cat-file*": "deny", - "git fsck*": "deny", - "git verify-pack*": "deny", - "git unpack-objects*": "deny", - "git cherry*": "deny", - "git show*": "deny", - "git merge-base*--is-ancestor*": "deny", - "git branch*--contains*": "deny", - "git tag*--contains*": "deny", - "git for-each-ref*--contains*": "deny", + "*git log*--all*": "deny", + "*git log*--branches*": "deny", + "*git log*--remotes*": "deny", + "*git log*--walk-reflogs*": "deny", + "*git log*--grep*": "deny", + "*git rev-list*--all*": "deny", + "*git rev-list*--branches*": "deny", + "*git rev-list*--remotes*": "deny", + "*git rev-list*--grep*": "deny", + "*git shortlog*--all*": "deny", + "*git reflog*": "deny", + "*git cat-file*": "deny", + "*git fsck*": "deny", + "*git verify-pack*": "deny", + "*git unpack-objects*": "deny", + "*git cherry*": "deny", + "*git show*": "deny", + "*git merge-base*--is-ancestor*": "deny", + "*git branch*--contains*": "deny", + "*git tag*--contains*": "deny", + "*git for-each-ref*--contains*": "deny", // git internals (substring match on path) "*.git/logs*": "deny", @@ -294,12 +294,12 @@ async function buildConfigDir(args: { "*.git/refs*": "deny", // online lookups - "curl *github.com*": "deny", - "wget *github.com*": "deny", - "curl *githubusercontent.com*": "deny", - "wget *githubusercontent.com*": "deny", - "curl *github.io*": "deny", - "wget *github.io*": "deny" + "*curl *github.com*": "deny", + "*wget *github.com*": "deny", + "*curl *githubusercontent.com*": "deny", + "*wget *githubusercontent.com*": "deny", + "*curl *github.io*": "deny", + "*wget *github.io*": "deny" } }, tools: { From f261004b4e4269a6e193fd242e5ec8ced9a7aec0 Mon Sep 17 00:00:00 2001 From: sdevare-nv Date: Tue, 4 Aug 2026 18:12:40 -0700 Subject: [PATCH 41/49] Update CLI configuration for permissions and sandboxing Removed the '--dangerously-skip-permissions' option and added OPENCODE_PERMISSION configuration for subagents. --- packages/opencode/src/bench/cli.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 916f6b9b4335..37cc72d3b3cf 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -368,7 +368,6 @@ function runOpencode(args: { `nemo-gym/${args.modelName}`, "--format", "json", - "--dangerously-skip-permissions", "--dir", args.workspaceRoot, ], @@ -529,6 +528,10 @@ async function main() { OPENCODE_DB: ":memory:", OPENCODE_DATA: path.join(tmpRoot, "data"), OPENCODE_CONFIG: configFile, + // The benchmark already runs inside a SIF sandbox, so make that the + // security boundary. This final config override applies to subagents too. + OPENCODE_PERMISSION: JSON.stringify({ "*": "allow" }), + // Disable opencode's built-in plugin loaders; the bench harness doesn't need them. OPENCODE_PURE: "1", // Skip the dynamic env block (working dir + Today's date) in the system From 354e1b9f113380963da0fa9095863c641bb23c5c Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Tue, 11 Aug 2026 18:18:25 -0700 Subject: [PATCH 42/49] feat(replay): replay opencode subagent sessions Signed-off-by: Sugam Devare --- .../benchmarks/swe_bench/scripts/run_infer.sh | 6 + packages/opencode/src/bench/cli.ts | 19 +- packages/opencode/src/bench/replay.ts | 103 ++++++- .../src/provider/sdk/nemo-gym/index.ts | 16 +- .../provider/sdk/nemo-gym/language-model.ts | 288 +++++++++++++----- packages/opencode/src/session/llm.ts | 3 + packages/opencode/src/session/message-v2.ts | 2 + packages/opencode/src/session/prompt.ts | 3 + packages/opencode/src/tool/task.ts | 8 + packages/opencode/test/bench/replay.test.ts | 78 ++++- .../provider/nemo-gym/language-model.test.ts | 197 ++++++++++++ packages/opencode/test/tool/task.test.ts | 11 + 12 files changed, 655 insertions(+), 79 deletions(-) diff --git a/evaluation/benchmarks/swe_bench/scripts/run_infer.sh b/evaluation/benchmarks/swe_bench/scripts/run_infer.sh index 154dc6fdcf7b..c9bc18eb1ee2 100755 --- a/evaluation/benchmarks/swe_bench/scripts/run_infer.sh +++ b/evaluation/benchmarks/swe_bench/scripts/run_infer.sh @@ -16,6 +16,7 @@ # $12 SYSTEM_PROMPT_PATH optional system-prompt override # $13 REPLAY_MESSAGES_PATH optional JSON file of prior chat-completion # messages to replay before continuing live +# $14 REPLAY_SUBAGENTS_PATH optional causal subagent replay manifest # # Environment (set by gym): # NEMO_GYM_MODEL_SERVER_NAME proxy name on the gym head server @@ -41,6 +42,7 @@ WORKSPACE_ROOT="${10:-}" USER_MESSAGE_PATH="${11:-}" SYSTEM_PROMPT_PATH="${12:-}" REPLAY_MESSAGES_PATH="${13:-}" +REPLAY_SUBAGENTS_PATH="${14:-}" if [ -z "$SELECTED_ID" ]; then echo "ERROR: SELECTED_ID (\$7) is required." @@ -96,6 +98,7 @@ echo "WORKSPACE_ROOT: $WORKSPACE_ROOT" echo "USER_MESSAGE_PATH: $USER_MESSAGE_PATH" echo "SYSTEM_PROMPT_PATH: $SYSTEM_PROMPT_PATH" echo "REPLAY_MESSAGES_PATH: $REPLAY_MESSAGES_PATH" +echo "REPLAY_SUBAGENTS_PATH: $REPLAY_SUBAGENTS_PATH" echo "MODEL_SERVER: $NEMO_GYM_MODEL_SERVER_NAME @ $NEMO_GYM_MODEL_SERVER_BASE_URL" cmd=( @@ -117,6 +120,9 @@ fi if [ -n "$REPLAY_MESSAGES_PATH" ]; then cmd+=(--replay-messages-file "$REPLAY_MESSAGES_PATH") fi +if [ -n "$REPLAY_SUBAGENTS_PATH" ]; then + cmd+=(--replay-subagents-file "$REPLAY_SUBAGENTS_PATH") +fi if [ "${ENABLE_SUBAGENTS:-0}" = "1" ] || [ "${ENABLE_SUBAGENTS:-}" = "true" ]; then cmd+=(--enable-subagents) fi diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 06bd91d5b01e..33b9d36b3314 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -28,8 +28,8 @@ import { bootstrapRepoIfMissing } from "./bootstrap_repo" // opencode's built-in anthropic system prompt — Bun bundles .txt as a string. // Used as the default when no --system-prompt override is passed. import PROMPT_ANTHROPIC from "../session/prompt/anthropic.txt" -import type { NemoGymReplayTurn } from "../provider/sdk/nemo-gym/language-model" -import { parseReplayMessages } from "./replay" +import type { NemoGymReplayManifest, NemoGymReplayTurn } from "../provider/sdk/nemo-gym/language-model" +import { parseReplayManifest, parseReplayMessages } from "./replay" interface CliArgs { instanceDictPath: string @@ -52,6 +52,8 @@ interface CliArgs { * before continuing live (trajectory resume). See language-model.ts. */ replayMessagesFile?: string + /** Causal parent-task-call -> recorded child-session replay graph. */ + replaySubagentsFile?: string } function parseArgs(argv: string[]): CliArgs { @@ -105,6 +107,9 @@ function parseArgs(argv: string[]): CliArgs { case "--replay-messages-file": out.replayMessagesFile = next() break + case "--replay-subagents-file": + out.replaySubagentsFile = next() + break default: if (a.startsWith("--")) throw new Error(`Unknown flag: ${a}`) } @@ -167,6 +172,8 @@ async function buildConfigDir(args: { replayTurns?: NemoGymReplayTurn[] /** Subsequent user messages trailing the last replayed turn. */ replayTrailingUserTexts?: string[] + /** Per-recorded-subagent replay queues and their parent task-call links. */ + replayManifest?: NemoGymReplayManifest }): Promise<{ tmpRoot: string; configFile: string }> { const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), `bench-${args.instanceId}-`)) await fs.mkdir(tmpRoot, { recursive: true }) @@ -200,6 +207,7 @@ async function buildConfigDir(args: { ...(args.replayTrailingUserTexts?.length ? { replayTrailingUserTexts: args.replayTrailingUserTexts } : {}), + ...(args.replayManifest ? { replayManifest: args.replayManifest } : {}), }, models: { [args.modelName]: { @@ -530,6 +538,7 @@ async function main() { let userPrompt: string let replayTurns: NemoGymReplayTurn[] | undefined let replayTrailingUserTexts: string[] | undefined + let replayManifest: NemoGymReplayManifest | undefined if (args.replayMessagesFile) { const raw = await fs.readFile(args.replayMessagesFile, "utf8") const parsed = parseReplayMessages(raw) @@ -541,6 +550,9 @@ async function main() { // on dataset_name); we just read it as-is and pass it to opencode. userPrompt = await fs.readFile(args.userMessageFile, "utf8") } + if (args.replaySubagentsFile) { + replayManifest = parseReplayManifest(await fs.readFile(args.replaySubagentsFile, "utf8")) + } const { tmpRoot, configFile } = await buildConfigDir({ instanceId: instance.instance_id, @@ -549,12 +561,13 @@ async function main() { completionsDir, maxTurns: args.maxTurns, systemPromptPath: args.systemPromptPath, - enableSubagents: args.enableSubagents, + enableSubagents: args.enableSubagents || Boolean(replayManifest?.sessions.length), temperature: forcedTemperature, topP: forcedTopP, maxTokens: forcedMaxTokens, replayTurns, replayTrailingUserTexts, + replayManifest, }) const startedAt = Date.now() diff --git a/packages/opencode/src/bench/replay.ts b/packages/opencode/src/bench/replay.ts index fa44312ae4c7..45aab7fa8e82 100644 --- a/packages/opencode/src/bench/replay.ts +++ b/packages/opencode/src/bench/replay.ts @@ -26,7 +26,7 @@ * instead of relying on session history to carry them forward. */ -import type { NemoGymReplayTurn } from "../provider/sdk/nemo-gym/language-model" +import type { NemoGymReplayManifest, NemoGymReplayTurn } from "../provider/sdk/nemo-gym/language-model" export interface ReplayChatMessage { role: "system" | "user" | "assistant" | "tool" @@ -90,3 +90,104 @@ export function parseReplayMessages(raw: string): ParsedReplay { ...(pendingUserTexts.length ? { trailingUserTexts: pendingUserTexts } : {}), } } + +function replayManifestError(message: string): never { + throw new Error(`replay-subagents-file: ${message}`) +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +/** Parse Gym's snake_case causal subagent manifest into provider replay queues. */ +export function parseReplayManifest(raw: string): NemoGymReplayManifest { + const input: unknown = JSON.parse(raw) + if (!isRecord(input)) return replayManifestError("expected an object") + if (input.version !== 1) return replayManifestError("version must be 1") + if (typeof input.root_session_id !== "string" || !input.root_session_id) { + return replayManifestError("root_session_id must be a non-empty string") + } + if (!Array.isArray(input.sessions)) return replayManifestError("sessions must be an array") + + const seen = new Set() + const seenSpawns = new Set() + const sessions = input.sessions.map((value, index) => { + if (!isRecord(value)) return replayManifestError(`sessions[${index}] must be an object`) + const sessionId = value.session_id + const parentSessionId = value.parent_session_id + const spawnCallId = value.spawn_call_id + const spawnIndex = value.spawn_index + if (typeof sessionId !== "string" || !sessionId) { + return replayManifestError(`sessions[${index}].session_id must be a non-empty string`) + } + if (sessionId === input.root_session_id) { + return replayManifestError(`sessions[${index}].session_id duplicates root_session_id`) + } + if (seen.has(sessionId)) return replayManifestError(`duplicate session_id ${sessionId}`) + seen.add(sessionId) + if (typeof parentSessionId !== "string" || !parentSessionId) { + return replayManifestError(`sessions[${index}].parent_session_id must be a non-empty string`) + } + if (typeof spawnCallId !== "string" || !spawnCallId) { + return replayManifestError(`sessions[${index}].spawn_call_id must be a non-empty string`) + } + if (!Number.isInteger(spawnIndex) || (spawnIndex as number) < 0) { + return replayManifestError(`sessions[${index}].spawn_index must be a non-negative integer`) + } + const spawnKey = `${parentSessionId}\u0000${spawnCallId}` + if (seenSpawns.has(spawnKey)) { + return replayManifestError(`duplicate spawn_call_id ${spawnCallId} in parent ${parentSessionId}`) + } + seenSpawns.add(spawnKey) + if (!Array.isArray(value.messages)) { + return replayManifestError(`sessions[${index}].messages must be an array`) + } + + const parsed = parseReplayMessages(JSON.stringify(value.messages)) + return { + sessionId, + parentSessionId, + spawnCallId, + spawnIndex: spawnIndex as number, + ...(typeof value.subagent_type === "string" ? { subagentType: value.subagent_type } : {}), + messageCount: value.messages.length, + // Unlike the root session, every later child user message is recreated + // by replaying its parent's task(task_id=...) call. Injecting the text + // here as well would duplicate resumed-task prompts. + replayTurns: parsed.replayTurns.map(({ content, toolCalls }) => ({ + content, + ...(toolCalls ? { toolCalls } : {}), + })), + } + }) + + const knownParents = new Set([input.root_session_id, ...sessions.map((session) => session.sessionId)]) + for (const session of sessions) { + if (!knownParents.has(session.parentSessionId)) { + return replayManifestError( + `session ${session.sessionId} references unknown parent_session_id ${session.parentSessionId}`, + ) + } + } + + const reachable = new Set([input.root_session_id]) + let changed = true + while (changed) { + changed = false + for (const session of sessions) { + if (reachable.has(session.sessionId) || !reachable.has(session.parentSessionId)) continue + reachable.add(session.sessionId) + changed = true + } + } + const unreachable = sessions.find((session) => !reachable.has(session.sessionId)) + if (unreachable) { + return replayManifestError(`session ${unreachable.sessionId} is not reachable from root_session_id`) + } + + return { + version: 1, + rootSessionId: input.root_session_id, + sessions, + } +} diff --git a/packages/opencode/src/provider/sdk/nemo-gym/index.ts b/packages/opencode/src/provider/sdk/nemo-gym/index.ts index 0294debd3a90..bd439b1182d5 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/index.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/index.ts @@ -9,7 +9,14 @@ * provider plumbing (Provider.Service.getModel) works without special-casing. */ -import { NemoGymLanguageModel, type NemoGymLanguageModelConfig, type NemoGymReplayTurn } from "./language-model" +import { + NemoGymLanguageModel, + type NemoGymLanguageModelConfig, + type NemoGymReplayManifest, + type NemoGymReplayTurn, +} from "./language-model" + +export type { NemoGymReplayManifest, NemoGymReplaySession, NemoGymReplayTurn } from "./language-model" export interface CreateNemoGymOptions { /** Base URL of the gym model server (`http://host:port`). */ @@ -38,18 +45,20 @@ export interface CreateNemoGymOptions { topP?: number /** Optional forced max_tokens; unset = no cap (vLLM generates to remaining context). */ maxTokens?: number - /** Optional turn counter shared across all model calls in a session. */ + /** Optional request-order counter shared across all live model calls. */ turnCounter?: { next(): number } /** Optional callback invoked after each successful chat-completion. */ onCompletion?: NemoGymLanguageModelConfig["onCompletion"] /** - * Scripted assistant turns to replay (main session only) before falling + * Scripted assistant turns to replay in the root session before falling * through to live HTTP calls. Set by the bench harness when the request * carries a prior trajectory to resume. See language-model.ts's docblock. */ replayTurns?: NemoGymReplayTurn[] /** Subsequent user messages trailing the last replayed turn. See language-model.ts's docblock. */ replayTrailingUserTexts?: string[] + /** Causal replay graph for child and nested-child sessions. */ + replayManifest?: NemoGymReplayManifest } export interface NemoGymProvider { @@ -78,6 +87,7 @@ export function createNemoGym(opts: CreateNemoGymOptions): NemoGymProvider { onCompletion: opts.onCompletion, replayTurns: opts.replayTurns, replayTrailingUserTexts: opts.replayTrailingUserTexts, + replayManifest: opts.replayManifest, }) }, } diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index eb72b9ec3a07..66d5f16b2baa 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -18,10 +18,11 @@ * openhands' `llm_completions//*.json` exactly so gym's * `get_openhands_trajectory_from_completions` reads it without changes. * - * Replay: when `cfg.replayTurns` is set, the first N tool-bearing calls on - * the top-level agentic-loop session (see `_popReplayTurn`'s docblock for - * exactly how that's identified — it's not as simple as "sessionID is the - * main session") are answered from that scripted queue instead of a real + * Replay: `cfg.replayTurns` drives the root session and `cfg.replayManifest` + * supplies one queue per recorded child. A live child is bound by its + * recorded parent plus the exact task call ID that created it, so parallel + * siblings and nested agents cannot consume one another's turns. Tool-bearing + * calls are answered from that session's scripted queue instead of a real * HTTP call — same synthesized-stream shape as a real response, so * opencode's *real* tool-execution path (streamText -> the AI-SDK `tool.execute` closures * built in session/prompt.ts's resolveTools()) runs each replayed tool call @@ -133,6 +134,23 @@ export interface NemoGymReplayTurn { precedingUserTexts?: string[] } +export interface NemoGymReplaySession { + sessionId: string + parentSessionId: string + spawnCallId: string + spawnIndex: number + subagentType?: string + messageCount: number + replayTurns: NemoGymReplayTurn[] + replayTrailingUserTexts?: string[] +} + +export interface NemoGymReplayManifest { + version: 1 + rootSessionId: string + sessions: NemoGymReplaySession[] +} + export interface NemoGymLanguageModelConfig { /** Provider id used to namespace providerMetadata. Defaults to "nemo-gym". */ provider: string @@ -180,7 +198,7 @@ export interface NemoGymLanguageModelConfig { requestParams: Record }) => void | Promise /** - * Scripted assistant turns to replay (main session only) before falling + * Scripted assistant turns to replay in the root session before falling * through to live HTTP calls. See the module docblock for why replayed * tool calls must run for real rather than just replaying recorded text. */ @@ -192,12 +210,35 @@ export interface NemoGymLanguageModelConfig { * NemoGymReplayTurn.precedingUserTexts. See the module docblock. */ replayTrailingUserTexts?: string[] + /** Causal parent-task-call -> recorded child-session replay graph. */ + replayManifest?: NemoGymReplayManifest } // --------------------------------------------------------------------------- // Implementation // --------------------------------------------------------------------------- +interface ReplayState { + recordedSessionID: string + recordedParentSessionID?: string + spawnCallID?: string + spawnIndex?: number + subagentType?: string + messageCount: number + turns: NemoGymReplayTurn[] + trailingUserTexts?: string[] + index: number + pendingUserInjections: Array<{ beforeAssistantOrdinal: number; text: string }> + livePrefixMessageCount?: number +} + +interface SessionHeaders { + sessionID: string + parentSessionID: string | undefined + parentToolCallID: string | undefined + agentName: string | undefined +} + export class NemoGymLanguageModel implements LanguageModelV3 { readonly specificationVersion = "v3" readonly modelId: string @@ -209,19 +250,13 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // subagents spawned via the task tool get their own sessionID, so keeping // a Map keeps their dump filenames from clobbering the main session's. private readonly turnCounters: Map = new Map() - // Replay only ever applies to the main session (subagent sessions never - // existed in the recorded trajectory) — a single counter is sufficient. - private replayIndex = 0 - // Precomputed once from cfg.replayTurns[i].precedingUserTexts / - // cfg.replayTrailingUserTexts. `beforeAssistantOrdinal` is the 0-based - // count of assistant-role messages that must already be in the wire - // message list before this text is inserted — i.e. insert right before - // the (beforeAssistantOrdinal + 1)-th assistant message overall, or at - // the end if that many don't exist yet. All replayed assistant turns are - // always fully present in session history by the time any live call - // happens, so this ordinal reliably resolves to the same chronological - // spot on every call — see _injectPendingUserMessages. - private readonly pendingUserInjections: Array<{ beforeAssistantOrdinal: number; text: string }> = [] + private readonly replayByRecordedSession = new Map() + private readonly replayChildBySpawn = new Map() + private readonly liveToRecordedSession = new Map() + private readonly recordedToLiveSession = new Map() + private readonly rootRecordedSessionID: string | undefined + private globalTurn = 0 + private readonly sessionStartGlobalTurn = new Map() constructor(modelId: string, cfg: NemoGymLanguageModelConfig) { this.modelId = modelId @@ -231,13 +266,33 @@ export class NemoGymLanguageModel implements LanguageModelV3 { requestTimeoutMs: cfg.requestTimeoutMs ?? 600_000, retries: cfg.retries ?? 3, } - cfg.replayTurns?.forEach((turn, i) => { - for (const text of turn.precedingUserTexts ?? []) { - this.pendingUserInjections.push({ beforeAssistantOrdinal: i, text }) - } - }) - for (const text of cfg.replayTrailingUserTexts ?? []) { - this.pendingUserInjections.push({ beforeAssistantOrdinal: cfg.replayTurns?.length ?? 0, text }) + this.rootRecordedSessionID = cfg.replayManifest?.rootSessionId ?? (cfg.replayTurns ? "__main__" : undefined) + if (this.rootRecordedSessionID) { + this.replayByRecordedSession.set( + this.rootRecordedSessionID, + this._makeReplayState({ + recordedSessionID: this.rootRecordedSessionID, + messageCount: 0, + turns: cfg.replayTurns ?? [], + trailingUserTexts: cfg.replayTrailingUserTexts, + }), + ) + } + for (const session of cfg.replayManifest?.sessions ?? []) { + this.replayByRecordedSession.set( + session.sessionId, + this._makeReplayState({ + recordedSessionID: session.sessionId, + recordedParentSessionID: session.parentSessionId, + spawnCallID: session.spawnCallId, + spawnIndex: session.spawnIndex, + subagentType: session.subagentType, + messageCount: session.messageCount, + turns: session.replayTurns, + trailingUserTexts: session.replayTrailingUserTexts, + }), + ) + this.replayChildBySpawn.set(this._spawnKey(session.parentSessionId, session.spawnCallId), session.sessionId) } } @@ -247,54 +302,110 @@ export class NemoGymLanguageModel implements LanguageModelV3 { return n } - private _sessionFromHeaders(headers: unknown): { sessionID: string; parentSessionID: string | undefined } { + private _nextGlobalTurn(sessionID: string): { globalTurn: number; sessionStartGlobalTurn: number } { + const globalTurn = this.cfg.turnCounter?.next() ?? this.globalTurn++ + const sessionStartGlobalTurn = this.sessionStartGlobalTurn.get(sessionID) ?? globalTurn + this.sessionStartGlobalTurn.set(sessionID, sessionStartGlobalTurn) + return { globalTurn, sessionStartGlobalTurn } + } + + private _makeReplayState( + input: Omit, + ): ReplayState { + return { + ...input, + index: 0, + pendingUserInjections: [ + ...input.turns.flatMap((turn, i) => + (turn.precedingUserTexts ?? []).map((text) => ({ beforeAssistantOrdinal: i, text })), + ), + ...(input.trailingUserTexts ?? []).map((text) => ({ + beforeAssistantOrdinal: input.turns.length, + text, + })), + ], + } + } + + private _spawnKey(parentSessionID: string, callID: string): string { + return `${parentSessionID}\u0000${callID}` + } + + private _sessionFromHeaders(headers: unknown): SessionHeaders { let sid = "" let pid: string | undefined + let callID: string | undefined + let agentName: string | undefined if (headers && typeof headers === "object") { const h = headers as Record const v = h["x-session-affinity"] ?? h["X-Session-Affinity"] if (typeof v === "string") sid = v const p = h["x-parent-session-id"] ?? h["X-Parent-Session-Id"] if (typeof p === "string") pid = p + const c = h["x-parent-tool-call-id"] ?? h["X-Parent-Tool-Call-Id"] + if (typeof c === "string") callID = c + const a = h["x-opencode-agent"] ?? h["X-Opencode-Agent"] + if (typeof a === "string") agentName = a + } + return { sessionID: sid || "main", parentSessionID: pid, parentToolCallID: callID, agentName } + } + + private _bindReplaySession(liveSessionID: string, recordedSessionID: string): ReplayState | undefined { + const existing = this.liveToRecordedSession.get(liveSessionID) + if (existing && existing !== recordedSessionID) { + throw new Error( + `nemo-gym replay: live session ${liveSessionID} is already bound to ${existing}, cannot bind ${recordedSessionID}`, + ) + } + const otherLive = this.recordedToLiveSession.get(recordedSessionID) + if (otherLive && otherLive !== liveSessionID) { + throw new Error( + `nemo-gym replay: recorded session ${recordedSessionID} is already bound to ${otherLive}, cannot bind ${liveSessionID}`, + ) + } + this.liveToRecordedSession.set(liveSessionID, recordedSessionID) + this.recordedToLiveSession.set(recordedSessionID, liveSessionID) + return this.replayByRecordedSession.get(recordedSessionID) + } + + private _replayState(session: SessionHeaders): ReplayState | undefined { + const recorded = this.liveToRecordedSession.get(session.sessionID) + if (recorded) return this.replayByRecordedSession.get(recorded) + if (!session.parentSessionID) { + if (!this.rootRecordedSessionID) return undefined + return this._bindReplaySession(session.sessionID, this.rootRecordedSessionID) } - return { sessionID: sid || "main", parentSessionID: pid } + if (!session.parentToolCallID) return undefined + const recordedParent = this.liveToRecordedSession.get(session.parentSessionID) + if (!recordedParent) return undefined + const recordedChild = this.replayChildBySpawn.get(this._spawnKey(recordedParent, session.parentToolCallID)) + if (!recordedChild) return undefined + return this._bindReplaySession(session.sessionID, recordedChild) } - // Pops the next scripted turn for the main agentic-loop session, or - // undefined once the replay queue is exhausted / not applicable. Advances - // state, so call at most once per doStream / doGenerate invocation. - // - // Two exclusions matter here, both confirmed against a real opencode run - // (not just unit tests against a bare LanguageModelV3CallOptions): - // - // 1. Subagent sessions: excluded via `parentSessionID` being set, NOT via - // comparing `sessionID` to a sentinel string. `session/llm.ts` sets - // `x-session-affinity: input.sessionID` UNCONDITIONALLY for every call - // on the nemo-gym provider — including the top-level/main session's own - // calls — so sessionID is *always* the real session id (e.g. "ses_..."), - // never a fallback "main" placeholder. A `sessionID !== "main"` check - // (the original implementation here) is therefore always true and - // replay never fires at all in a real run — a subagent session is the - // one with `parentSessionID` set, not the one whose id happens to equal - // a literal string. - // 2. Auxiliary no-tool calls on the SAME session: opencode's own `runLoop` - // forks off title-generation and summary-generation model calls on step - // 1 (`session/prompt.ts`), using the same session id as the real - // agentic loop. Those race against the loop's own first call and, if - // unfiltered, silently consume scripted turns meant for the real agent - // (confirmed: they reach this provider before the real loop's first - // call in practice). They're reliably distinguishable because they - // never pass `tools` — only the real agentic loop resolves and sends - // the tool registry — so `hasTools` gates them out. - private _popReplayTurn( - session: { sessionID: string; parentSessionID: string | undefined }, - hasTools: boolean, - ): NemoGymReplayTurn | undefined { - if (!this.cfg.replayTurns) return undefined - if (session.parentSessionID) return undefined + // Auxiliary title/summary calls never pass tools, so they must not consume + // a session's scripted agentic-loop turns. + private _popReplayTurn(session: SessionHeaders, hasTools: boolean): NemoGymReplayTurn | undefined { if (!hasTools) return undefined - if (this.replayIndex >= this.cfg.replayTurns.length) return undefined - return this.cfg.replayTurns[this.replayIndex++] + const replay = this._replayState(session) + if (!replay || replay.index >= replay.turns.length) return undefined + return replay.turns[replay.index++] + } + + private _rewriteTaskResumeArguments(tool: NonNullable[number]): string { + if (tool.name !== "task" || !tool.arguments.includes('"task_id"')) return tool.arguments + let parsed: unknown + try { + parsed = JSON.parse(tool.arguments) + } catch { + return tool.arguments + } + if (!parsed || typeof parsed !== "object") return tool.arguments + const input = parsed as Record + if (typeof input.task_id !== "string") return tool.arguments + const liveSessionID = this.recordedToLiveSession.get(input.task_id) + if (!liveSessionID) return tool.arguments + return JSON.stringify({ ...input, task_id: liveSessionID }) } private _messageFromReplayTurn(turn: NemoGymReplayTurn): ChatResponseChoice["message"] { @@ -304,7 +415,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { tool_calls: turn.toolCalls?.map((tc) => ({ id: tc.id, type: "function" as const, - function: { name: tc.name, arguments: tc.arguments }, + function: { name: tc.name, arguments: this._rewriteTaskResumeArguments(tc) }, })), } } @@ -370,10 +481,10 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // the outgoing message list, in place. Processed in descending ordinal // order so inserting a later text doesn't shift the index about to be // looked up for an earlier one. - private _injectPendingUserMessages(messages: ChatRequestMessage[]): void { - if (!this.pendingUserInjections.length) return + private _injectPendingUserMessages(messages: ChatRequestMessage[], replay: ReplayState | undefined): void { + if (!replay?.pendingUserInjections.length) return const byOrdinal = new Map() - for (const { beforeAssistantOrdinal, text } of this.pendingUserInjections) { + for (const { beforeAssistantOrdinal, text } of replay.pendingUserInjections) { const list = byOrdinal.get(beforeAssistantOrdinal) ?? [] list.push(text) byOrdinal.set(beforeAssistantOrdinal, list) @@ -435,7 +546,8 @@ export class NemoGymLanguageModel implements LanguageModelV3 { } } - const { warnings, loggedMessages, requestParams } = await this._buildRequestParams(options) + const { warnings, loggedMessages, requestParams, globalTurn, sessionStartGlobalTurn } = + await this._buildRequestParams(options, session) const { responseJson } = await this._postChat(requestParams) const choice = responseJson.choices[0] @@ -466,6 +578,8 @@ export class NemoGymLanguageModel implements LanguageModelV3 { providerSpecificFields, requestParams, session, + globalTurn, + sessionStartGlobalTurn, }) return { @@ -507,7 +621,8 @@ export class NemoGymLanguageModel implements LanguageModelV3 { return { stream, request: { body: "{}" }, response: {} } } - const { warnings, loggedMessages, requestParams } = await this._buildRequestParams(options) + const { warnings, loggedMessages, requestParams, globalTurn, sessionStartGlobalTurn } = + await this._buildRequestParams(options, session) // Fire the HTTP call eagerly so any error surfaces synchronously when the // stream is consumed. We then synthesize parts in `start`. @@ -546,6 +661,8 @@ export class NemoGymLanguageModel implements LanguageModelV3 { providerSpecificFields, requestParams, session, + globalTurn, + sessionStartGlobalTurn, }) controller.enqueue({ @@ -580,13 +697,15 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // Helpers // ----------------------------------------------------------------------- - private async _buildRequestParams(options: LanguageModelV3CallOptions): Promise<{ + private async _buildRequestParams(options: LanguageModelV3CallOptions, session: SessionHeaders): Promise<{ warnings: SharedV3Warning[] messages: ChatRequestMessage[] loggedMessages: ChatRequestMessage[] tools: unknown toolChoice: unknown requestParams: Record + globalTurn: number + sessionStartGlobalTurn: number }> { const warnings: SharedV3Warning[] = [] // Reuse opencode's existing OpenAI-compatible message converter so all @@ -618,7 +737,8 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // identically. Re-applied on every call — nothing else carries them // forward — at a position fixed relative to the replayed assistant // turns, so they land in the same chronological spot every time. - this._injectPendingUserMessages(messages as ChatRequestMessage[]) + const replay = this._replayState(session) + this._injectPendingUserMessages(messages as ChatRequestMessage[], replay) // Token-ID handling, mirroring OpenHands' nemo_gym_client.py exactly: // - WIRE request: token IDs on the MOST RECENT assistant message only @@ -639,6 +759,9 @@ export class NemoGymLanguageModel implements LanguageModelV3 { const loggedMessages = (messages as Array>).map((m) => ({ ...m, })) as unknown as ChatRequestMessage[] + if (replay && options.tools?.length && replay.livePrefixMessageCount === undefined) { + replay.livePrefixMessageCount = loggedMessages.length + } { const promptAssistants = options.prompt.filter((m) => m.role === "assistant") const wireAssistants = (messages as Array>).filter((m) => m["role"] === "assistant") @@ -720,7 +843,15 @@ export class NemoGymLanguageModel implements LanguageModelV3 { if (requestParams[k] === undefined) delete requestParams[k] } - return { warnings, messages, loggedMessages, tools, toolChoice, requestParams } + return { + warnings, + messages, + loggedMessages, + tools, + toolChoice, + requestParams, + ...this._nextGlobalTurn(session.sessionID), + } } private async _postChat(params: Record): Promise<{ responseJson: ChatResponse }> { @@ -850,9 +981,16 @@ export class NemoGymLanguageModel implements LanguageModelV3 { response: ChatResponse providerSpecificFields: Record requestParams: Record - session: { sessionID: string; parentSessionID: string | undefined } + session: SessionHeaders + globalTurn: number + sessionStartGlobalTurn: number }) { const turn = this._nextTurn(args.session.sessionID) + const replay = this._replayState(args.session) + const recordedParentSessionID = replay?.recordedParentSessionID ?? + (args.session.parentSessionID + ? this.liveToRecordedSession.get(args.session.parentSessionID) + : undefined) if (this.cfg.onCompletion) { try { await this.cfg.onCompletion({ turn, ...args }) @@ -883,7 +1021,15 @@ export class NemoGymLanguageModel implements LanguageModelV3 { kwargs, session_id: args.session.sessionID, parent_session_id: args.session.parentSessionID ?? null, + recorded_session_id: replay?.recordedSessionID ?? null, + recorded_parent_session_id: recordedParentSessionID ?? null, + spawn_call_id: replay?.spawnCallID ?? args.session.parentToolCallID ?? null, + spawn_index: replay?.spawnIndex ?? null, + subagent_type: replay?.subagentType ?? args.session.agentName ?? null, + replay_prefix_message_count: replay?.livePrefixMessageCount ?? null, turn, + global_turn: args.globalTurn, + session_start_global_turn: args.sessionStartGlobalTurn, timestamp: Date.now() / 1000, } const tmp = `${fpath}.tmp` diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index e76583f2d347..dce7d2e0aa26 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -37,6 +37,7 @@ export type StreamInput = { user: MessageV2.User sessionID: string parentSessionID?: string + parentToolCallID?: string model: Provider.Model agent: Agent.Info permission?: Permission.Ruleset @@ -381,6 +382,8 @@ const live: Layer.Layer< : { "x-session-affinity": input.sessionID, ...(input.parentSessionID ? { "x-parent-session-id": input.parentSessionID } : {}), + ...(input.parentToolCallID ? { "x-parent-tool-call-id": input.parentToolCallID } : {}), + "x-opencode-agent": input.agent.name, "User-Agent": `opencode/${InstallationVersion}`, }), ...input.model.headers, diff --git a/packages/opencode/src/session/message-v2.ts b/packages/opencode/src/session/message-v2.ts index 237fb527c078..347e9b202b86 100644 --- a/packages/opencode/src/session/message-v2.ts +++ b/packages/opencode/src/session/message-v2.ts @@ -378,6 +378,8 @@ const messageBase = { export const User = Schema.Struct({ ...messageBase, role: Schema.Literal("user"), + /** Task tool call in the parent session that created this child session. */ + parentToolCallID: Schema.optional(Schema.String), time: Schema.Struct({ created: NonNegativeInt, }), diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index f7c59fe4cba0..8595ceda5390 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -943,6 +943,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the id: input.messageID ?? MessageID.ascending(), role: "user", sessionID: input.sessionID, + parentToolCallID: input.parentToolCallID, time: { created: Date.now() }, tools: input.tools, agent: ag.name, @@ -1584,6 +1585,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the permission: session.permission, sessionID, parentSessionID: session.parentID, + parentToolCallID: lastUser.parentToolCallID, system, messages: [...modelMsgs, ...(isLastStep ? [{ role: "user" as const, content: MAX_STEPS }] : [])], tools, @@ -1811,6 +1813,7 @@ const ModelRef = Schema.Struct({ export const PromptInput = Schema.Struct({ sessionID: SessionID, messageID: Schema.optional(MessageID), + parentToolCallID: Schema.optional(Schema.String), model: Schema.optional(ModelRef), agent: Schema.optional(Schema.String), noReply: Schema.optional(Schema.Boolean), diff --git a/packages/opencode/src/tool/task.ts b/packages/opencode/src/tool/task.ts index 22e4e5671c89..8e2ade8177a9 100644 --- a/packages/opencode/src/tool/task.ts +++ b/packages/opencode/src/tool/task.ts @@ -100,6 +100,13 @@ export const TaskTool = Tool.define( })) ?? []), ], })) + const spawnMessage = session + ? (yield* sessions.messages({ sessionID: nextSession.id })).find( + (message) => message.info.role === "user" && message.info.parentToolCallID, + ) + : undefined + const spawnToolCallID = + spawnMessage?.info.role === "user" ? (spawnMessage.info.parentToolCallID ?? ctx.callID) : ctx.callID const msg = yield* Effect.sync(() => MessageV2.get({ sessionID: ctx.sessionID, messageID: ctx.messageID })) if (msg.info.role !== "assistant") return yield* Effect.fail(new Error("Not an assistant message")) @@ -138,6 +145,7 @@ export const TaskTool = Tool.define( const result = yield* ops.prompt({ messageID, sessionID: nextSession.id, + parentToolCallID: spawnToolCallID, model: { modelID: model.modelID, providerID: model.providerID, diff --git a/packages/opencode/test/bench/replay.test.ts b/packages/opencode/test/bench/replay.test.ts index d60299be2611..25ab4c23b67b 100644 --- a/packages/opencode/test/bench/replay.test.ts +++ b/packages/opencode/test/bench/replay.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect } from "bun:test" -import { parseReplayMessages, replayMessageText } from "@/bench/replay" +import { parseReplayManifest, parseReplayMessages, replayMessageText } from "@/bench/replay" describe("replayMessageText", () => { test("returns string content verbatim", () => { @@ -142,3 +142,79 @@ describe("parseReplayMessages", () => { expect(() => parseReplayMessages(raw)).toThrow(/no user message/) }) }) + +describe("parseReplayManifest", () => { + test("parses child and nested-child replay queues without relying on array order", () => { + const manifest = parseReplayManifest( + JSON.stringify({ + version: 1, + root_session_id: "recorded-root", + sessions: [ + { + session_id: "recorded-grandchild", + parent_session_id: "recorded-child", + spawn_call_id: "call_nested", + spawn_index: 0, + subagent_type: "explore", + messages: [ + { role: "user", content: "nested work" }, + { role: "assistant", content: "nested result" }, + ], + }, + { + session_id: "recorded-child", + parent_session_id: "recorded-root", + spawn_call_id: "call_child", + spawn_index: 1, + subagent_type: "general", + messages: [ + { role: "user", content: "child work" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_nested", + function: { name: "task", arguments: '{"prompt":"nested work"}' }, + }, + ], + }, + ], + }, + ], + }), + ) + + expect(manifest.rootSessionId).toBe("recorded-root") + expect(manifest.sessions.map((session) => session.sessionId)).toEqual([ + "recorded-grandchild", + "recorded-child", + ]) + expect(manifest.sessions[0]).toMatchObject({ + parentSessionId: "recorded-child", + spawnCallId: "call_nested", + messageCount: 2, + replayTurns: [{ content: "nested result" }], + }) + }) + + test("rejects an unlinked parent", () => { + expect(() => + parseReplayManifest( + JSON.stringify({ + version: 1, + root_session_id: "root", + sessions: [ + { + session_id: "child", + parent_session_id: "missing", + spawn_call_id: "call_1", + spawn_index: 0, + messages: [{ role: "user", content: "work" }], + }, + ], + }), + ), + ).toThrow(/unknown parent_session_id/) + }) +}) diff --git a/packages/opencode/test/provider/nemo-gym/language-model.test.ts b/packages/opencode/test/provider/nemo-gym/language-model.test.ts index 8d088e654d9d..0badd04a8741 100644 --- a/packages/opencode/test/provider/nemo-gym/language-model.test.ts +++ b/packages/opencode/test/provider/nemo-gym/language-model.test.ts @@ -285,4 +285,201 @@ describe("NemoGymLanguageModel replay", () => { const messages = bodies[0].messages as unknown as Array<{ role: string; content?: unknown }> expect(messages[messages.length - 1]).toMatchObject({ role: "user", content: "now also check the tests" }) }) + + test("binds parallel child queues by parent task call id, not manifest or execution order", async () => { + const fetchSpy = mock(async () => { + throw new Error("network should not be called during replay") + }) + // @ts-expect-error test override + globalThis.fetch = fetchSpy + + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + replayTurns: [{ + content: null, + toolCalls: [ + { id: "call_a", name: "task", arguments: '{"prompt":"A","subagent_type":"general"}' }, + { id: "call_b", name: "task", arguments: '{"prompt":"B","subagent_type":"explore"}' }, + ], + }], + replayManifest: { + version: 1, + rootSessionId: "recorded-root", + // Deliberately opposite the parent task-call order. + sessions: [ + { + sessionId: "recorded-b", + parentSessionId: "recorded-root", + spawnCallId: "call_b", + spawnIndex: 1, + messageCount: 2, + replayTurns: [{ content: "result B" }], + }, + { + sessionId: "recorded-a", + parentSessionId: "recorded-root", + spawnCallId: "call_a", + spawnIndex: 0, + messageCount: 2, + replayTurns: [{ content: "result A" }], + }, + ], + }, + }) + + await drain( + (await model.doStream({ ...CALL_OPTIONS, headers: { "x-session-affinity": "live-root" } })).stream, + ) + + const childB = await drain( + ( + await model.doStream({ + ...CALL_OPTIONS, + headers: { + "x-session-affinity": "live-b", + "x-parent-session-id": "live-root", + "x-parent-tool-call-id": "call_b", + }, + }) + ).stream, + ) + const childA = await drain( + ( + await model.doStream({ + ...CALL_OPTIONS, + headers: { + "x-session-affinity": "live-a", + "x-parent-session-id": "live-root", + "x-parent-tool-call-id": "call_a", + }, + }) + ).stream, + ) + + expect(childB.find((part) => part.type === "text-delta")).toMatchObject({ delta: "result B" }) + expect(childA.find((part) => part.type === "text-delta")).toMatchObject({ delta: "result A" }) + expect(fetchSpy).not.toHaveBeenCalled() + }) + + test("recursively binds a nested subagent to the task call in its recorded parent", async () => { + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + replayTurns: [{ + content: null, + toolCalls: [{ id: "call_child", name: "task", arguments: "{}" }], + }], + replayManifest: { + version: 1, + rootSessionId: "recorded-root", + sessions: [ + { + sessionId: "recorded-child", + parentSessionId: "recorded-root", + spawnCallId: "call_child", + spawnIndex: 0, + messageCount: 3, + replayTurns: [{ + content: null, + toolCalls: [{ id: "call_grandchild", name: "task", arguments: "{}" }], + }], + }, + { + sessionId: "recorded-grandchild", + parentSessionId: "recorded-child", + spawnCallId: "call_grandchild", + spawnIndex: 0, + messageCount: 2, + replayTurns: [{ content: "nested result" }], + }, + ], + }, + }) + + await drain( + (await model.doStream({ ...CALL_OPTIONS, headers: { "x-session-affinity": "live-root" } })).stream, + ) + await drain( + ( + await model.doStream({ + ...CALL_OPTIONS, + headers: { + "x-session-affinity": "live-child", + "x-parent-session-id": "live-root", + "x-parent-tool-call-id": "call_child", + }, + }) + ).stream, + ) + const nested = await drain( + ( + await model.doStream({ + ...CALL_OPTIONS, + headers: { + "x-session-affinity": "live-grandchild", + "x-parent-session-id": "live-child", + "x-parent-tool-call-id": "call_grandchild", + }, + }) + ).stream, + ) + + expect(nested.find((part) => part.type === "text-delta")).toMatchObject({ delta: "nested result" }) + }) + + test("rewrites a recorded task_id to the bound live child session id when resuming it", async () => { + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + replayTurns: [ + { content: null, toolCalls: [{ id: "call_child", name: "task", arguments: "{}" }] }, + { + content: null, + toolCalls: [ + { + id: "call_resume", + name: "task", + arguments: '{"task_id":"recorded-child","prompt":"continue"}', + }, + ], + }, + ], + replayManifest: { + version: 1, + rootSessionId: "recorded-root", + sessions: [ + { + sessionId: "recorded-child", + parentSessionId: "recorded-root", + spawnCallId: "call_child", + spawnIndex: 0, + messageCount: 2, + replayTurns: [{ content: "first child result" }], + }, + ], + }, + }) + + const rootHeaders = { "x-session-affinity": "live-root" } + await drain((await model.doStream({ ...CALL_OPTIONS, headers: rootHeaders })).stream) + await drain( + ( + await model.doStream({ + ...CALL_OPTIONS, + headers: { + "x-session-affinity": "live-child", + "x-parent-session-id": "live-root", + "x-parent-tool-call-id": "call_child", + }, + }) + ).stream, + ) + const resumed = await drain((await model.doStream({ ...CALL_OPTIONS, headers: rootHeaders })).stream) + const call = resumed.find((part) => part.type === "tool-call") + expect(call).toMatchObject({ + toolCallId: "call_resume", + input: '{"task_id":"live-child","prompt":"continue"}', + }) + }) }) diff --git a/packages/opencode/test/tool/task.test.ts b/packages/opencode/test/tool/task.test.ts index f75fcf84b8a9..6cf0a302966c 100644 --- a/packages/opencode/test/tool/task.test.ts +++ b/packages/opencode/test/tool/task.test.ts @@ -196,6 +196,15 @@ describe("tool.task", () => { const sessions = yield* Session.Service const { chat, assistant } = yield* seed() const child = yield* sessions.create({ parentID: chat.id, title: "Existing child" }) + yield* sessions.updateMessage({ + id: MessageID.ascending(), + role: "user", + sessionID: child.id, + parentToolCallID: "call_spawn", + agent: "general", + model: ref, + time: { created: Date.now() }, + }) const tool = yield* TaskTool const def = yield* tool.init() let seen: SessionPrompt.PromptInput | undefined @@ -211,6 +220,7 @@ describe("tool.task", () => { { sessionID: chat.id, messageID: assistant.id, + callID: "call_resume", agent: "build", abort: new AbortController().signal, extra: { promptOps }, @@ -226,6 +236,7 @@ describe("tool.task", () => { expect(result.metadata.sessionId).toBe(child.id) expect(result.output).toContain(`task_id: ${child.id}`) expect(seen?.sessionID).toBe(child.id) + expect(seen?.parentToolCallID).toBe("call_spawn") }), ) From 38a11e833de1511a774126a28c7ef2c49ded23fd Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Tue, 18 Aug 2026 22:33:48 -0700 Subject: [PATCH 43/49] fix(bench): surface terminal replay errors --- packages/opencode/src/bench/cli.ts | 54 +++++++++++-------- packages/opencode/src/bench/terminal_error.ts | 38 +++++++++++++ packages/opencode/src/session/processor.ts | 2 + packages/opencode/src/session/prompt.ts | 2 + .../test/bench/terminal_error.test.ts | 30 +++++++++++ 5 files changed, 104 insertions(+), 22 deletions(-) create mode 100644 packages/opencode/src/bench/terminal_error.ts create mode 100644 packages/opencode/test/bench/terminal_error.test.ts diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 33b9d36b3314..d02f76b10b30 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -30,6 +30,7 @@ import { bootstrapRepoIfMissing } from "./bootstrap_repo" import PROMPT_ANTHROPIC from "../session/prompt/anthropic.txt" import type { NemoGymReplayManifest, NemoGymReplayTurn } from "../provider/sdk/nemo-gym/language-model" import { parseReplayManifest, parseReplayMessages } from "./replay" +import * as BenchTerminalError from "./terminal_error" interface CliArgs { instanceDictPath: string @@ -178,9 +179,7 @@ async function buildConfigDir(args: { const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), `bench-${args.instanceId}-`)) await fs.mkdir(tmpRoot, { recursive: true }) - const systemPrompt = args.systemPromptPath - ? await fs.readFile(args.systemPromptPath, "utf8") - : DEFAULT_SYSTEM_PROMPT + const systemPrompt = args.systemPromptPath ? await fs.readFile(args.systemPromptPath, "utf8") : DEFAULT_SYSTEM_PROMPT const cfg: Record = { $schema: "https://opencode.ai/config.json", @@ -204,9 +203,7 @@ async function buildConfigDir(args: { ...(args.topP !== undefined ? { topP: args.topP } : {}), ...(args.maxTokens !== undefined ? { maxTokens: args.maxTokens } : {}), ...(args.replayTurns?.length ? { replayTurns: args.replayTurns } : {}), - ...(args.replayTrailingUserTexts?.length - ? { replayTrailingUserTexts: args.replayTrailingUserTexts } - : {}), + ...(args.replayTrailingUserTexts?.length ? { replayTrailingUserTexts: args.replayTrailingUserTexts } : {}), ...(args.replayManifest ? { replayManifest: args.replayManifest } : {}), }, models: { @@ -231,13 +228,13 @@ async function buildConfigDir(args: { edit: { "**": "allow" }, bash: { "*": "allow", - + // process termination "*killall*": "deny", "*pkill*": "deny", "*kill -1*": "deny", "*kill 0*": "deny", - + // filesystem destruction "*rm -rf /": "deny", "*rm -rf /*": "deny", @@ -255,7 +252,7 @@ async function buildConfigDir(args: { "*rm -rf /dev*": "deny", "*rm -rf /proc*": "deny", "*rm -rf /sys*": "deny", - + // system control // "*shutdown*": "deny", // "*reboot*": "deny", @@ -263,13 +260,13 @@ async function buildConfigDir(args: { // "*halt*": "deny", // "init 0*": "deny", // "init 6*": "deny", - + // disk devices "dd *of=/dev/sd*": "deny", "dd *of=/dev/nvme*": "deny", "dd *of=/dev/hd*": "deny", "dd *of=/dev/null*": "deny", - + // git network "*git fetch*": "deny", "*git pull*": "deny", @@ -288,7 +285,7 @@ async function buildConfigDir(args: { "*git archive*--remote*": "deny", "*git *://*": "deny", "*git *@*:*": "deny", - + // git history mining "*git log*--all*": "deny", "*git log*--branches*": "deny", @@ -311,22 +308,22 @@ async function buildConfigDir(args: { "*git branch*--contains*": "deny", "*git tag*--contains*": "deny", "*git for-each-ref*--contains*": "deny", - + // git internals (substring match on path) "*.git/logs*": "deny", "*.git/packed-refs*": "deny", "*.git/ORIG_HEAD*": "deny", "*.git/FETCH_HEAD*": "deny", "*.git/refs*": "deny", - + // online lookups "*curl *github.com*": "deny", "*wget *github.com*": "deny", "*curl *githubusercontent.com*": "deny", "*wget *githubusercontent.com*": "deny", "*curl *github.io*": "deny", - "*wget *github.io*": "deny" - } + "*wget *github.io*": "deny", + }, }, tools: { bash: true, @@ -373,7 +370,7 @@ function runOpencode(args: { env: NodeJS.ProcessEnv opencodeBin: string agent: string -}): Promise<{ exitCode: number; stdout: string; stderr: string }> { +}): Promise<{ exitCode: number; stdout: string; stderr: string; terminalError?: BenchTerminalError.Kind }> { // Use the same bun binary that's currently running — guaranteed to exist // and avoids PATH lookup quirks under Bun's posix_spawn. const bunPath = process.execPath @@ -404,6 +401,13 @@ function runOpencode(args: { ) let stdout = "" let stderr = "" + let terminalError: BenchTerminalError.Kind | undefined + let terminalSignalBuffer = "" + const observeTerminalSignal = (chunk: string) => { + // Retain enough overlap to recognize a marker split across pipe chunks. + terminalSignalBuffer = (terminalSignalBuffer + chunk).slice(-256) + terminalError = BenchTerminalError.prefer(terminalError, BenchTerminalError.detect(terminalSignalBuffer)) + } // Strip bulky token-ID metadata from echoed event lines. The IDs already // live in the llm_completions dumps; leaving them in the event stream // makes each turn re-echo that turn's full-context prompt_token_ids -> @@ -427,7 +431,9 @@ function runOpencode(args: { const MAX_KEEP = 256 * 1024 // keep only a bounded tail for error reporting let lineBuf = "" child.stdout?.on("data", (b) => { - lineBuf += b.toString("utf8") + const chunk = b.toString("utf8") + observeTerminalSignal(chunk) + lineBuf += chunk let idx: number while ((idx = lineBuf.indexOf("\n")) >= 0) { const line = scrub(lineBuf.slice(0, idx)) @@ -439,13 +445,14 @@ function runOpencode(args: { }) child.stderr?.on("data", (b) => { const chunk = b.toString("utf8") + observeTerminalSignal(chunk) stderr = (stderr + chunk).slice(-MAX_KEEP) process.stderr.write(chunk) }) - child.on("close", (code) => resolve({ exitCode: code ?? 0, stdout, stderr })) + child.on("close", (code) => resolve({ exitCode: code ?? 0, stdout, stderr, terminalError })) child.on("error", (err) => { stderr += String(err) - resolve({ exitCode: 999, stdout, stderr }) + resolve({ exitCode: 999, stdout, stderr, terminalError }) }) }) } @@ -587,6 +594,9 @@ async function main() { // prompt — keeps the RL prompt-token prefix invariant stable across turns // (a midnight rollover would otherwise shift `Today's date: ...`). OPENCODE_DISABLE_ENV_PROMPT: "1", + // Have all agent sessions report terminal states to this bench wrapper. + // This is bench-only and does not alter normal opencode runs. + [BenchTerminalError.ENV]: "1", } // Bootstrap a git repo if the SIF shipped a flat source tree (swe-bench-ext @@ -616,7 +626,7 @@ async function main() { const patch = await captureGitDiff(workspaceRoot) const benchRunTime = (Date.now() - startedAt) / 1000 - const error: string | null = result.exitCode === 0 ? null : `opencode_exit_${result.exitCode}` + const error = BenchTerminalError.toGymError(result.exitCode, result.terminalError) const outPath = await writeOutputJsonl(args.outputDir, instance.instance_id, { instance_id: instance.instance_id, test_result: { git_patch: patch }, @@ -636,7 +646,7 @@ async function main() { // child-stdio pipes from the opencode subprocess). Gym's runner treats any // non-zero apptainer exit as `Agent command failed` and discards the // already-written patch, so we MUST exit 0 deterministically on success. - process.exit(result.exitCode === 0 ? 0 : 1) + process.exit(BenchTerminalError.shouldExitSuccessfully(result.exitCode, result.terminalError) ? 0 : 1) } main().catch((err) => { diff --git a/packages/opencode/src/bench/terminal_error.ts b/packages/opencode/src/bench/terminal_error.ts new file mode 100644 index 000000000000..d8a0b64064ef --- /dev/null +++ b/packages/opencode/src/bench/terminal_error.ts @@ -0,0 +1,38 @@ +export type Kind = "max_iteration" | "context_window" + +export const ENV = "OPENCODE_BENCH_TERMINAL_SIGNALS" +export const PREFIX = "[opencode-bench-terminal] " + +export function encode(kind: Kind): string { + return PREFIX + kind +} + +/** Report terminal agent states from any session, including subagents. */ +export function report(kind: Kind): void { + if (process.env[ENV] !== "1") return + process.stderr.write(encode(kind) + "\n") +} + +export function detect(text: string): Kind | undefined { + if (text.includes(encode("context_window"))) return "context_window" + if (text.includes(encode("max_iteration"))) return "max_iteration" + return undefined +} + +/** Context overflow wins when the forced final max-step call also overflows. */ +export function prefer(current: Kind | undefined, incoming: Kind | undefined): Kind | undefined { + if (!current) return incoming + if (!incoming) return current + if (current === "context_window" || incoming === "context_window") return "context_window" + return "max_iteration" +} + +export function toGymError(exitCode: number, kind?: Kind): string | null { + if (kind === "max_iteration") return "maximum iteration reached" + if (kind === "context_window") return "context window exceeded" + return exitCode === 0 ? null : `opencode_exit_${exitCode}` +} + +export function shouldExitSuccessfully(exitCode: number, kind?: Kind): boolean { + return exitCode === 0 || kind !== undefined +} diff --git a/packages/opencode/src/session/processor.ts b/packages/opencode/src/session/processor.ts index f22da92927d2..727eaed99526 100644 --- a/packages/opencode/src/session/processor.ts +++ b/packages/opencode/src/session/processor.ts @@ -24,6 +24,7 @@ import { EventV2 } from "@/v2/event" import { SessionEvent } from "@/v2/session-event" import { Modelv2 } from "@/v2/model" import * as DateTime from "effect/DateTime" +import * as BenchTerminalError from "@/bench/terminal_error" const DOOM_LOOP_THRESHOLD = 3 const log = Log.create({ service: "session.processor" }) @@ -647,6 +648,7 @@ export const layer: Layer.Layer< slog.error("process", { error: errorMessage(e), stack: e instanceof Error ? e.stack : undefined }) const error = parse(e) if (MessageV2.ContextOverflowError.isInstance(error)) { + BenchTerminalError.report("context_window") ctx.needsCompaction = true yield* bus.publish(Session.Event.Error, { sessionID: ctx.sessionID, error }) return diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 8595ceda5390..4e250c788790 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -61,6 +61,7 @@ import * as DateTime from "effect/DateTime" import { eq } from "@/storage/db" import * as Database from "@/storage/db" import { SessionTable } from "./session.sql" +import * as BenchTerminalError from "@/bench/terminal_error" // @ts-ignore globalThis.AI_SDK_LOG_WARNINGS = false @@ -1496,6 +1497,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the } const maxSteps = agent.steps ?? Infinity const isLastStep = step >= maxSteps + if (isLastStep) BenchTerminalError.report("max_iteration") msgs = yield* insertReminders({ messages: msgs, agent, session }) const msg: MessageV2.Assistant = { diff --git a/packages/opencode/test/bench/terminal_error.test.ts b/packages/opencode/test/bench/terminal_error.test.ts new file mode 100644 index 000000000000..5985446408f7 --- /dev/null +++ b/packages/opencode/test/bench/terminal_error.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, test } from "bun:test" +import * as BenchTerminalError from "@/bench/terminal_error" + +describe("bench terminal error signals", () => { + test("detects max-iteration and context-window markers", () => { + expect(BenchTerminalError.detect(`before ${BenchTerminalError.encode("max_iteration")} after`)).toBe( + "max_iteration", + ) + expect(BenchTerminalError.detect(BenchTerminalError.encode("context_window"))).toBe("context_window") + expect(BenchTerminalError.detect("ordinary opencode stderr")).toBeUndefined() + }) + + test("prefers context overflow when both terminal states occur", () => { + expect(BenchTerminalError.prefer("max_iteration", "context_window")).toBe("context_window") + expect(BenchTerminalError.prefer("context_window", "max_iteration")).toBe("context_window") + }) + + test("writes errors that Gym classifies and preserves ordinary exit errors", () => { + expect(BenchTerminalError.toGymError(0, "max_iteration")).toBe("maximum iteration reached") + expect(BenchTerminalError.toGymError(0, "context_window")).toBe("context window exceeded") + expect(BenchTerminalError.toGymError(17)).toBe("opencode_exit_17") + expect(BenchTerminalError.toGymError(0)).toBeNull() + }) + + test("keeps terminal trajectories even when opencode exits nonzero", () => { + expect(BenchTerminalError.shouldExitSuccessfully(1, "context_window")).toBeTrue() + expect(BenchTerminalError.shouldExitSuccessfully(1, "max_iteration")).toBeTrue() + expect(BenchTerminalError.shouldExitSuccessfully(1)).toBeFalse() + }) +}) From 0e84c9db11e0af23f5c1deb3e9eaad50fd998523 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Tue, 18 Aug 2026 23:24:48 -0700 Subject: [PATCH 44/49] fix(nemo-gym): propagate context overflow completion --- .../provider/sdk/nemo-gym/language-model.ts | 99 +++++++++- .../provider/nemo-gym/language-model.test.ts | 182 +++++++++++++----- 2 files changed, 225 insertions(+), 56 deletions(-) diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 66d5f16b2baa..6225f05324ea 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -114,6 +114,72 @@ interface ChatResponse { usage?: ChatResponseUsage } +/** + * `_postChat` normally retries transport/server failures because training + * backpressure can last for minutes. A context-limit response is terminal, + * though: retrying the same request can never succeed. Preserve it as the + * structured stream-error shape consumed by MessageV2.parseStreamError so + * SessionProcessor can surface ContextOverflowError immediately. + */ +class ContextOverflowResponseError extends Error { + constructor(streamError: string) { + super(streamError) + this.name = "ContextOverflowResponseError" + } +} + +const CONTEXT_OVERFLOW_RESPONSE_PATTERNS = [ + /context[_ -]?length[_ -]?exceeded/i, + /maximum context length/i, + /exceeds? (?:the )?context window/i, + /context window exceeded/i, + /input length.*context length/i, + /prompt (?:is )?too long/i, + /request entity too large/i, +] + +function contextOverflowStreamError(message: string): string { + return JSON.stringify({ + type: "error", + error: { + code: "context_length_exceeded", + message: message.slice(0, 2_000) || "Input exceeds context window of this model", + }, + }) +} + +function normalizeContextOverflowResponse(status: number, text: string): string | undefined { + let code = "" + let message = text + try { + const body = JSON.parse(text) as Record + const error = body.error + if (error && typeof error === "object") { + const obj = error as Record + if (obj.code !== undefined) code = String(obj.code) + if (typeof obj.message === "string") message = obj.message + } else if (typeof error === "string") { + message = error + } + if (!code && body.code !== undefined) code = String(body.code) + if (message === text && typeof body.message === "string") message = body.message + if (message === text && typeof body.detail === "string") message = body.detail + } catch {} + + const evidence = `${code}\n${message}\n${text}` + if (status !== 413 && !CONTEXT_OVERFLOW_RESPONSE_PATTERNS.some((pattern) => pattern.test(evidence))) return + + return contextOverflowStreamError(message) +} + +function isGymContextOverflowCompletion(choice: ChatResponseChoice): boolean { + // Gym's vLLM wrapper translates an upstream context-overflow HTTP 400 into + // a successful empty completion. The stable signal it returns is exactly + // this pair. `content == null` alone is not enough because valid tool-call + // completions also normally carry null assistant content. + return choice.finish_reason === "length" && choice.message?.content == null +} + // --------------------------------------------------------------------------- // Config // --------------------------------------------------------------------------- @@ -552,7 +618,13 @@ export class NemoGymLanguageModel implements LanguageModelV3 { const choice = responseJson.choices[0] if (!choice) throw new Error("nemo-gym: empty choices in response") - const msg: ChatResponseChoice["message"] = choice.message ?? ({ role: "assistant" } as ChatResponseChoice["message"]) + if (isGymContextOverflowCompletion(choice)) { + throw new ContextOverflowResponseError( + contextOverflowStreamError("NeMo Gym returned an empty length completion for an overlong context"), + ) + } + const msg: ChatResponseChoice["message"] = + choice.message ?? ({ role: "assistant" } as ChatResponseChoice["message"]) const providerSpecificFields = this._extractProviderFields(msg) const providerMetadata = this._buildProviderMetadata(providerSpecificFields) @@ -637,6 +709,11 @@ export class NemoGymLanguageModel implements LanguageModelV3 { const choice = responseJson.choices[0] if (!choice) throw new Error("nemo-gym: empty choices in response") + if (isGymContextOverflowCompletion(choice)) { + throw new ContextOverflowResponseError( + contextOverflowStreamError("NeMo Gym returned an empty length completion for an overlong context"), + ) + } const msg: ChatResponseChoice["message"] = choice.message ?? ({ role: "assistant" } as ChatResponseChoice["message"]) @@ -697,7 +774,10 @@ export class NemoGymLanguageModel implements LanguageModelV3 { // Helpers // ----------------------------------------------------------------------- - private async _buildRequestParams(options: LanguageModelV3CallOptions, session: SessionHeaders): Promise<{ + private async _buildRequestParams( + options: LanguageModelV3CallOptions, + session: SessionHeaders, + ): Promise<{ warnings: SharedV3Warning[] messages: ChatRequestMessage[] loggedMessages: ChatRequestMessage[] @@ -897,6 +977,8 @@ export class NemoGymLanguageModel implements LanguageModelV3 { if (timer) clearTimeout(timer) if (!res.ok) { const text = await res.text().catch(() => "") + const contextOverflow = normalizeContextOverflowResponse(res.status, text) + if (contextOverflow) throw new ContextOverflowResponseError(contextOverflow) throw new Error(`NeMoGym ${url} ${res.status}: ${text.slice(0, 500)}`) } const setCookie = res.headers.get("set-cookie") @@ -911,6 +993,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { return { responseJson } } catch (err) { if (timer) clearTimeout(timer) + if (err instanceof ContextOverflowResponseError) throw err lastErr = err if (attempt === retries - 1) break // Cap exponential backoff at 60s so unlimited-retries configs don't blow up the delay. @@ -945,7 +1028,10 @@ export class NemoGymLanguageModel implements LanguageModelV3 { return md } - private _mapFinishReason(raw: string | null): { unified: "stop" | "length" | "tool-calls" | "error" | "other"; raw: string | undefined } { + private _mapFinishReason(raw: string | null): { + unified: "stop" | "length" | "tool-calls" | "error" | "other" + raw: string | undefined + } { if (!raw) return { unified: "other", raw: undefined } switch (raw) { case "stop": @@ -987,10 +1073,9 @@ export class NemoGymLanguageModel implements LanguageModelV3 { }) { const turn = this._nextTurn(args.session.sessionID) const replay = this._replayState(args.session) - const recordedParentSessionID = replay?.recordedParentSessionID ?? - (args.session.parentSessionID - ? this.liveToRecordedSession.get(args.session.parentSessionID) - : undefined) + const recordedParentSessionID = + replay?.recordedParentSessionID ?? + (args.session.parentSessionID ? this.liveToRecordedSession.get(args.session.parentSessionID) : undefined) if (this.cfg.onCompletion) { try { await this.cfg.onCompletion({ turn, ...args }) diff --git a/packages/opencode/test/provider/nemo-gym/language-model.test.ts b/packages/opencode/test/provider/nemo-gym/language-model.test.ts index 0badd04a8741..a48f346eb12a 100644 --- a/packages/opencode/test/provider/nemo-gym/language-model.test.ts +++ b/packages/opencode/test/provider/nemo-gym/language-model.test.ts @@ -70,15 +70,16 @@ describe("NemoGymLanguageModel replay", () => { }) test("doStream falls through to the real HTTP path once the replay queue is exhausted", async () => { - const fetchSpy = mock(async () => - new Response( - JSON.stringify({ - id: "resp_1", - model: "test-model", - choices: [{ finish_reason: "stop", message: { role: "assistant", content: "live turn" } }], - }), - { status: 200, headers: { "Content-Type": "application/json" } }, - ), + const fetchSpy = mock( + async () => + new Response( + JSON.stringify({ + id: "resp_1", + model: "test-model", + choices: [{ finish_reason: "stop", message: { role: "assistant", content: "live turn" } }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), ) // @ts-expect-error test override globalThis.fetch = fetchSpy @@ -101,16 +102,90 @@ describe("NemoGymLanguageModel replay", () => { expect(textDelta).toMatchObject({ delta: "live turn" }) }) + test("doStream surfaces a context-limit HTTP response without retrying it", async () => { + const fetchSpy = mock( + async () => + new Response( + JSON.stringify({ + error: { + code: "context_length_exceeded", + message: "This model's maximum context length is 32 tokens; the request has 64 tokens.", + }, + }), + { status: 400, headers: { "Content-Type": "application/json" } }, + ), + ) + // @ts-expect-error test override + globalThis.fetch = fetchSpy + + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + retries: Number.MAX_SAFE_INTEGER, + }) + + const parts = await drain((await model.doStream(CALL_OPTIONS)).stream) + expect(fetchSpy).toHaveBeenCalledTimes(1) + + const error = parts.find((part) => part.type === "error") + expect(error?.type).toBe("error") + if (error?.type !== "error" || typeof error.error !== "string") throw new Error("missing stream error") + expect(JSON.parse(error.error)).toMatchObject({ + type: "error", + error: { code: "context_length_exceeded" }, + }) + }) + + test("doStream recognizes Gym's null-content length completion as context overflow", async () => { + const fetchSpy = mock( + async () => + new Response( + JSON.stringify({ + id: "chtcmpl-123", + model: "test-model", + choices: [ + { + index: 0, + finish_reason: "length", + message: { role: "assistant", content: null, tool_calls: null }, + }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ) + // @ts-expect-error test override + globalThis.fetch = fetchSpy + + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + retries: Number.MAX_SAFE_INTEGER, + }) + + const parts = await drain((await model.doStream(CALL_OPTIONS)).stream) + expect(fetchSpy).toHaveBeenCalledTimes(1) + + const error = parts.find((part) => part.type === "error") + expect(error?.type).toBe("error") + if (error?.type !== "error" || typeof error.error !== "string") throw new Error("missing stream error") + expect(JSON.parse(error.error)).toMatchObject({ + type: "error", + error: { code: "context_length_exceeded" }, + }) + }) + test("replay is scoped to the top-level session — a subagent session (x-parent-session-id set) calls through to HTTP", async () => { - const fetchSpy = mock(async () => - new Response( - JSON.stringify({ - id: "resp_1", - model: "test-model", - choices: [{ finish_reason: "stop", message: { role: "assistant", content: "subagent turn" } }], - }), - { status: 200, headers: { "Content-Type": "application/json" } }, - ), + const fetchSpy = mock( + async () => + new Response( + JSON.stringify({ + id: "resp_1", + model: "test-model", + choices: [{ finish_reason: "stop", message: { role: "assistant", content: "subagent turn" } }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), ) // @ts-expect-error test override globalThis.fetch = fetchSpy @@ -136,15 +211,16 @@ describe("NemoGymLanguageModel replay", () => { }) test("an auxiliary no-tool call on the main session (e.g. opencode's own title/summary generation) does not consume the replay queue", async () => { - const fetchSpy = mock(async () => - new Response( - JSON.stringify({ - id: "resp_1", - model: "test-model", - choices: [{ finish_reason: "stop", message: { role: "assistant", content: "Fix the parser bug" } }], - }), - { status: 200, headers: { "Content-Type": "application/json" } }, - ), + const fetchSpy = mock( + async () => + new Response( + JSON.stringify({ + id: "resp_1", + model: "test-model", + choices: [{ finish_reason: "stop", message: { role: "assistant", content: "Fix the parser bug" } }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), ) // @ts-expect-error test override globalThis.fetch = fetchSpy @@ -212,7 +288,13 @@ describe("NemoGymLanguageModel replay", () => { // Attached to turn 0 (ordinal 0): must land BEFORE the first assistant // message in the eventual wire request, i.e. right after the initial // user message. - replayTurns: [{ content: null, toolCalls: [{ id: "call_1", name: "bash", arguments: "{}" }], precedingUserTexts: ["please also fix the other bug"] }], + replayTurns: [ + { + content: null, + toolCalls: [{ id: "call_1", name: "bash", arguments: "{}" }], + precedingUserTexts: ["please also fix the other bug"], + }, + ], }) // Turn 0 replays without hitting the network. @@ -296,13 +378,15 @@ describe("NemoGymLanguageModel replay", () => { const model = new NemoGymLanguageModel("test-model", { provider: "nemo-gym", baseURL: "http://unused.invalid", - replayTurns: [{ - content: null, - toolCalls: [ - { id: "call_a", name: "task", arguments: '{"prompt":"A","subagent_type":"general"}' }, - { id: "call_b", name: "task", arguments: '{"prompt":"B","subagent_type":"explore"}' }, - ], - }], + replayTurns: [ + { + content: null, + toolCalls: [ + { id: "call_a", name: "task", arguments: '{"prompt":"A","subagent_type":"general"}' }, + { id: "call_b", name: "task", arguments: '{"prompt":"B","subagent_type":"explore"}' }, + ], + }, + ], replayManifest: { version: 1, rootSessionId: "recorded-root", @@ -328,9 +412,7 @@ describe("NemoGymLanguageModel replay", () => { }, }) - await drain( - (await model.doStream({ ...CALL_OPTIONS, headers: { "x-session-affinity": "live-root" } })).stream, - ) + await drain((await model.doStream({ ...CALL_OPTIONS, headers: { "x-session-affinity": "live-root" } })).stream) const childB = await drain( ( @@ -366,10 +448,12 @@ describe("NemoGymLanguageModel replay", () => { const model = new NemoGymLanguageModel("test-model", { provider: "nemo-gym", baseURL: "http://unused.invalid", - replayTurns: [{ - content: null, - toolCalls: [{ id: "call_child", name: "task", arguments: "{}" }], - }], + replayTurns: [ + { + content: null, + toolCalls: [{ id: "call_child", name: "task", arguments: "{}" }], + }, + ], replayManifest: { version: 1, rootSessionId: "recorded-root", @@ -380,10 +464,12 @@ describe("NemoGymLanguageModel replay", () => { spawnCallId: "call_child", spawnIndex: 0, messageCount: 3, - replayTurns: [{ - content: null, - toolCalls: [{ id: "call_grandchild", name: "task", arguments: "{}" }], - }], + replayTurns: [ + { + content: null, + toolCalls: [{ id: "call_grandchild", name: "task", arguments: "{}" }], + }, + ], }, { sessionId: "recorded-grandchild", @@ -397,9 +483,7 @@ describe("NemoGymLanguageModel replay", () => { }, }) - await drain( - (await model.doStream({ ...CALL_OPTIONS, headers: { "x-session-affinity": "live-root" } })).stream, - ) + await drain((await model.doStream({ ...CALL_OPTIONS, headers: { "x-session-affinity": "live-root" } })).stream) await drain( ( await model.doStream({ From a849cc3b7162180c11b8b16e1d6bd14389b877cb Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Tue, 18 Aug 2026 23:28:14 -0700 Subject: [PATCH 45/49] refactor(nemo-gym): use exact Gym overflow sentinel --- .../provider/sdk/nemo-gym/language-model.ts | 55 +------------------ .../provider/nemo-gym/language-model.test.ts | 34 ------------ 2 files changed, 2 insertions(+), 87 deletions(-) diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 6225f05324ea..07203dcb21fc 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -114,30 +114,6 @@ interface ChatResponse { usage?: ChatResponseUsage } -/** - * `_postChat` normally retries transport/server failures because training - * backpressure can last for minutes. A context-limit response is terminal, - * though: retrying the same request can never succeed. Preserve it as the - * structured stream-error shape consumed by MessageV2.parseStreamError so - * SessionProcessor can surface ContextOverflowError immediately. - */ -class ContextOverflowResponseError extends Error { - constructor(streamError: string) { - super(streamError) - this.name = "ContextOverflowResponseError" - } -} - -const CONTEXT_OVERFLOW_RESPONSE_PATTERNS = [ - /context[_ -]?length[_ -]?exceeded/i, - /maximum context length/i, - /exceeds? (?:the )?context window/i, - /context window exceeded/i, - /input length.*context length/i, - /prompt (?:is )?too long/i, - /request entity too large/i, -] - function contextOverflowStreamError(message: string): string { return JSON.stringify({ type: "error", @@ -148,30 +124,6 @@ function contextOverflowStreamError(message: string): string { }) } -function normalizeContextOverflowResponse(status: number, text: string): string | undefined { - let code = "" - let message = text - try { - const body = JSON.parse(text) as Record - const error = body.error - if (error && typeof error === "object") { - const obj = error as Record - if (obj.code !== undefined) code = String(obj.code) - if (typeof obj.message === "string") message = obj.message - } else if (typeof error === "string") { - message = error - } - if (!code && body.code !== undefined) code = String(body.code) - if (message === text && typeof body.message === "string") message = body.message - if (message === text && typeof body.detail === "string") message = body.detail - } catch {} - - const evidence = `${code}\n${message}\n${text}` - if (status !== 413 && !CONTEXT_OVERFLOW_RESPONSE_PATTERNS.some((pattern) => pattern.test(evidence))) return - - return contextOverflowStreamError(message) -} - function isGymContextOverflowCompletion(choice: ChatResponseChoice): boolean { // Gym's vLLM wrapper translates an upstream context-overflow HTTP 400 into // a successful empty completion. The stable signal it returns is exactly @@ -619,7 +571,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { const choice = responseJson.choices[0] if (!choice) throw new Error("nemo-gym: empty choices in response") if (isGymContextOverflowCompletion(choice)) { - throw new ContextOverflowResponseError( + throw new Error( contextOverflowStreamError("NeMo Gym returned an empty length completion for an overlong context"), ) } @@ -710,7 +662,7 @@ export class NemoGymLanguageModel implements LanguageModelV3 { const choice = responseJson.choices[0] if (!choice) throw new Error("nemo-gym: empty choices in response") if (isGymContextOverflowCompletion(choice)) { - throw new ContextOverflowResponseError( + throw new Error( contextOverflowStreamError("NeMo Gym returned an empty length completion for an overlong context"), ) } @@ -977,8 +929,6 @@ export class NemoGymLanguageModel implements LanguageModelV3 { if (timer) clearTimeout(timer) if (!res.ok) { const text = await res.text().catch(() => "") - const contextOverflow = normalizeContextOverflowResponse(res.status, text) - if (contextOverflow) throw new ContextOverflowResponseError(contextOverflow) throw new Error(`NeMoGym ${url} ${res.status}: ${text.slice(0, 500)}`) } const setCookie = res.headers.get("set-cookie") @@ -993,7 +943,6 @@ export class NemoGymLanguageModel implements LanguageModelV3 { return { responseJson } } catch (err) { if (timer) clearTimeout(timer) - if (err instanceof ContextOverflowResponseError) throw err lastErr = err if (attempt === retries - 1) break // Cap exponential backoff at 60s so unlimited-retries configs don't blow up the delay. diff --git a/packages/opencode/test/provider/nemo-gym/language-model.test.ts b/packages/opencode/test/provider/nemo-gym/language-model.test.ts index a48f346eb12a..bfbda261b67b 100644 --- a/packages/opencode/test/provider/nemo-gym/language-model.test.ts +++ b/packages/opencode/test/provider/nemo-gym/language-model.test.ts @@ -102,40 +102,6 @@ describe("NemoGymLanguageModel replay", () => { expect(textDelta).toMatchObject({ delta: "live turn" }) }) - test("doStream surfaces a context-limit HTTP response without retrying it", async () => { - const fetchSpy = mock( - async () => - new Response( - JSON.stringify({ - error: { - code: "context_length_exceeded", - message: "This model's maximum context length is 32 tokens; the request has 64 tokens.", - }, - }), - { status: 400, headers: { "Content-Type": "application/json" } }, - ), - ) - // @ts-expect-error test override - globalThis.fetch = fetchSpy - - const model = new NemoGymLanguageModel("test-model", { - provider: "nemo-gym", - baseURL: "http://unused.invalid", - retries: Number.MAX_SAFE_INTEGER, - }) - - const parts = await drain((await model.doStream(CALL_OPTIONS)).stream) - expect(fetchSpy).toHaveBeenCalledTimes(1) - - const error = parts.find((part) => part.type === "error") - expect(error?.type).toBe("error") - if (error?.type !== "error" || typeof error.error !== "string") throw new Error("missing stream error") - expect(JSON.parse(error.error)).toMatchObject({ - type: "error", - error: { code: "context_length_exceeded" }, - }) - }) - test("doStream recognizes Gym's null-content length completion as context overflow", async () => { const fetchSpy = mock( async () => From fd05c42eb7e13f9dd4e3960cdefc382eec331dfd Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Wed, 19 Aug 2026 00:05:25 -0700 Subject: [PATCH 46/49] fix(nemo-gym): surface maskable terminal states --- packages/opencode/src/bench/cli.ts | 50 ++++++++------ packages/opencode/src/bench/terminal_error.ts | 38 +++++++++++ .../provider/sdk/nemo-gym/language-model.ts | 36 +++++++++- packages/opencode/src/session/processor.ts | 2 + packages/opencode/src/session/prompt.ts | 2 + .../test/bench/terminal_error.test.ts | 30 +++++++++ .../nemo-gym/context-overflow.test.ts | 65 +++++++++++++++++++ 7 files changed, 202 insertions(+), 21 deletions(-) create mode 100644 packages/opencode/src/bench/terminal_error.ts create mode 100644 packages/opencode/test/bench/terminal_error.test.ts create mode 100644 packages/opencode/test/provider/nemo-gym/context-overflow.test.ts diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 37cc72d3b3cf..5d8f9772e1f5 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -25,6 +25,7 @@ import os from "node:os" import { spawn } from "node:child_process" import { runDeepReset } from "./deep_reset" import { bootstrapRepoIfMissing } from "./bootstrap_repo" +import * as BenchTerminalError from "./terminal_error" // opencode's built-in anthropic system prompt — Bun bundles .txt as a string. // Used as the default when no --system-prompt override is passed. import PROMPT_ANTHROPIC from "../session/prompt/anthropic.txt" @@ -157,9 +158,7 @@ async function buildConfigDir(args: { const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), `bench-${args.instanceId}-`)) await fs.mkdir(tmpRoot, { recursive: true }) - const systemPrompt = args.systemPromptPath - ? await fs.readFile(args.systemPromptPath, "utf8") - : DEFAULT_SYSTEM_PROMPT + const systemPrompt = args.systemPromptPath ? await fs.readFile(args.systemPromptPath, "utf8") : DEFAULT_SYSTEM_PROMPT const cfg: Record = { $schema: "https://opencode.ai/config.json", @@ -205,13 +204,13 @@ async function buildConfigDir(args: { edit: { "**": "allow" }, bash: { "*": "allow", - + // process termination "*killall*": "deny", "*pkill*": "deny", "*kill -1*": "deny", "*kill 0*": "deny", - + // filesystem destruction "*rm -rf /": "deny", "*rm -rf /*": "deny", @@ -229,7 +228,7 @@ async function buildConfigDir(args: { "*rm -rf /dev*": "deny", "*rm -rf /proc*": "deny", "*rm -rf /sys*": "deny", - + // system control // "*shutdown*": "deny", // "*reboot*": "deny", @@ -237,13 +236,13 @@ async function buildConfigDir(args: { // "*halt*": "deny", // "init 0*": "deny", // "init 6*": "deny", - + // disk devices "dd *of=/dev/sd*": "deny", "dd *of=/dev/nvme*": "deny", "dd *of=/dev/hd*": "deny", "dd *of=/dev/null*": "deny", - + // git network "*git fetch*": "deny", "*git pull*": "deny", @@ -262,7 +261,7 @@ async function buildConfigDir(args: { "*git archive*--remote*": "deny", "*git *://*": "deny", "*git *@*:*": "deny", - + // git history mining "*git log*--all*": "deny", "*git log*--branches*": "deny", @@ -285,22 +284,22 @@ async function buildConfigDir(args: { "*git branch*--contains*": "deny", "*git tag*--contains*": "deny", "*git for-each-ref*--contains*": "deny", - + // git internals (substring match on path) "*.git/logs*": "deny", "*.git/packed-refs*": "deny", "*.git/ORIG_HEAD*": "deny", "*.git/FETCH_HEAD*": "deny", "*.git/refs*": "deny", - + // online lookups "*curl *github.com*": "deny", "*wget *github.com*": "deny", "*curl *githubusercontent.com*": "deny", "*wget *githubusercontent.com*": "deny", "*curl *github.io*": "deny", - "*wget *github.io*": "deny" - } + "*wget *github.io*": "deny", + }, }, tools: { bash: true, @@ -347,7 +346,7 @@ function runOpencode(args: { env: NodeJS.ProcessEnv opencodeBin: string agent: string -}): Promise<{ exitCode: number; stdout: string; stderr: string }> { +}): Promise<{ exitCode: number; stdout: string; stderr: string; terminalError?: BenchTerminalError.Kind }> { // Use the same bun binary that's currently running — guaranteed to exist // and avoids PATH lookup quirks under Bun's posix_spawn. const bunPath = process.execPath @@ -378,6 +377,13 @@ function runOpencode(args: { ) let stdout = "" let stderr = "" + let terminalError: BenchTerminalError.Kind | undefined + let terminalSignalBuffer = "" + const observeTerminalSignal = (chunk: string) => { + // Retain enough overlap to recognize a marker split across pipe chunks. + terminalSignalBuffer = (terminalSignalBuffer + chunk).slice(-256) + terminalError = BenchTerminalError.prefer(terminalError, BenchTerminalError.detect(terminalSignalBuffer)) + } // Strip bulky token-ID metadata from echoed event lines. The IDs already // live in the llm_completions dumps; leaving them in the event stream // makes each turn re-echo that turn's full-context prompt_token_ids -> @@ -401,7 +407,9 @@ function runOpencode(args: { const MAX_KEEP = 256 * 1024 // keep only a bounded tail for error reporting let lineBuf = "" child.stdout?.on("data", (b) => { - lineBuf += b.toString("utf8") + const chunk = b.toString("utf8") + observeTerminalSignal(chunk) + lineBuf += chunk let idx: number while ((idx = lineBuf.indexOf("\n")) >= 0) { const line = scrub(lineBuf.slice(0, idx)) @@ -413,13 +421,14 @@ function runOpencode(args: { }) child.stderr?.on("data", (b) => { const chunk = b.toString("utf8") + observeTerminalSignal(chunk) stderr = (stderr + chunk).slice(-MAX_KEEP) process.stderr.write(chunk) }) - child.on("close", (code) => resolve({ exitCode: code ?? 0, stdout, stderr })) + child.on("close", (code) => resolve({ exitCode: code ?? 0, stdout, stderr, terminalError })) child.on("error", (err) => { stderr += String(err) - resolve({ exitCode: 999, stdout, stderr }) + resolve({ exitCode: 999, stdout, stderr, terminalError }) }) }) } @@ -538,6 +547,9 @@ async function main() { // prompt — keeps the RL prompt-token prefix invariant stable across turns // (a midnight rollover would otherwise shift `Today's date: ...`). OPENCODE_DISABLE_ENV_PROMPT: "1", + // Have all agent sessions report terminal states to this bench wrapper. + // This is bench-only and does not alter normal opencode runs. + [BenchTerminalError.ENV]: "1", } // Bootstrap a git repo if the SIF shipped a flat source tree (swe-bench-ext @@ -567,7 +579,7 @@ async function main() { const patch = await captureGitDiff(workspaceRoot) const benchRunTime = (Date.now() - startedAt) / 1000 - const error: string | null = result.exitCode === 0 ? null : `opencode_exit_${result.exitCode}` + const error = BenchTerminalError.toGymError(result.exitCode, result.terminalError) const outPath = await writeOutputJsonl(args.outputDir, instance.instance_id, { instance_id: instance.instance_id, test_result: { git_patch: patch }, @@ -587,7 +599,7 @@ async function main() { // child-stdio pipes from the opencode subprocess). Gym's runner treats any // non-zero apptainer exit as `Agent command failed` and discards the // already-written patch, so we MUST exit 0 deterministically on success. - process.exit(result.exitCode === 0 ? 0 : 1) + process.exit(BenchTerminalError.shouldExitSuccessfully(result.exitCode, result.terminalError) ? 0 : 1) } main().catch((err) => { diff --git a/packages/opencode/src/bench/terminal_error.ts b/packages/opencode/src/bench/terminal_error.ts new file mode 100644 index 000000000000..d8a0b64064ef --- /dev/null +++ b/packages/opencode/src/bench/terminal_error.ts @@ -0,0 +1,38 @@ +export type Kind = "max_iteration" | "context_window" + +export const ENV = "OPENCODE_BENCH_TERMINAL_SIGNALS" +export const PREFIX = "[opencode-bench-terminal] " + +export function encode(kind: Kind): string { + return PREFIX + kind +} + +/** Report terminal agent states from any session, including subagents. */ +export function report(kind: Kind): void { + if (process.env[ENV] !== "1") return + process.stderr.write(encode(kind) + "\n") +} + +export function detect(text: string): Kind | undefined { + if (text.includes(encode("context_window"))) return "context_window" + if (text.includes(encode("max_iteration"))) return "max_iteration" + return undefined +} + +/** Context overflow wins when the forced final max-step call also overflows. */ +export function prefer(current: Kind | undefined, incoming: Kind | undefined): Kind | undefined { + if (!current) return incoming + if (!incoming) return current + if (current === "context_window" || incoming === "context_window") return "context_window" + return "max_iteration" +} + +export function toGymError(exitCode: number, kind?: Kind): string | null { + if (kind === "max_iteration") return "maximum iteration reached" + if (kind === "context_window") return "context window exceeded" + return exitCode === 0 ? null : `opencode_exit_${exitCode}` +} + +export function shouldExitSuccessfully(exitCode: number, kind?: Kind): boolean { + return exitCode === 0 || kind !== undefined +} diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 0a0f9e56c386..fca48b9a9311 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -85,6 +85,24 @@ interface ChatResponse { usage?: ChatResponseUsage } +function contextOverflowStreamError(message: string): string { + return JSON.stringify({ + type: "error", + error: { + code: "context_length_exceeded", + message: message.slice(0, 2_000) || "Input exceeds context window of this model", + }, + }) +} + +function isGymContextOverflowCompletion(choice: ChatResponseChoice): boolean { + // Gym's vLLM wrapper translates an upstream context-overflow HTTP 400 into + // a successful empty completion. The stable signal it returns is exactly + // this pair. `content == null` alone is not enough because valid tool-call + // completions also normally carry null assistant content. + return choice.finish_reason === "length" && choice.message?.content == null +} + // --------------------------------------------------------------------------- // Config // --------------------------------------------------------------------------- @@ -197,7 +215,13 @@ export class NemoGymLanguageModel implements LanguageModelV3 { const choice = responseJson.choices[0] if (!choice) throw new Error("nemo-gym: empty choices in response") - const msg: ChatResponseChoice["message"] = choice.message ?? ({ role: "assistant" } as ChatResponseChoice["message"]) + if (isGymContextOverflowCompletion(choice)) { + throw new Error( + contextOverflowStreamError("NeMo Gym returned an empty length completion for an overlong context"), + ) + } + const msg: ChatResponseChoice["message"] = + choice.message ?? ({ role: "assistant" } as ChatResponseChoice["message"]) const providerSpecificFields = this._extractProviderFields(msg) const providerMetadata = this._buildProviderMetadata(providerSpecificFields) @@ -253,6 +277,11 @@ export class NemoGymLanguageModel implements LanguageModelV3 { const choice = responseJson.choices[0] if (!choice) throw new Error("nemo-gym: empty choices in response") + if (isGymContextOverflowCompletion(choice)) { + throw new Error( + contextOverflowStreamError("NeMo Gym returned an empty length completion for an overlong context"), + ) + } const msg: ChatResponseChoice["message"] = choice.message ?? ({ role: "assistant" } as ChatResponseChoice["message"]) @@ -575,7 +604,10 @@ export class NemoGymLanguageModel implements LanguageModelV3 { return md } - private _mapFinishReason(raw: string | null): { unified: "stop" | "length" | "tool-calls" | "error" | "other"; raw: string | undefined } { + private _mapFinishReason(raw: string | null): { + unified: "stop" | "length" | "tool-calls" | "error" | "other" + raw: string | undefined + } { if (!raw) return { unified: "other", raw: undefined } switch (raw) { case "stop": diff --git a/packages/opencode/src/session/processor.ts b/packages/opencode/src/session/processor.ts index f22da92927d2..727eaed99526 100644 --- a/packages/opencode/src/session/processor.ts +++ b/packages/opencode/src/session/processor.ts @@ -24,6 +24,7 @@ import { EventV2 } from "@/v2/event" import { SessionEvent } from "@/v2/session-event" import { Modelv2 } from "@/v2/model" import * as DateTime from "effect/DateTime" +import * as BenchTerminalError from "@/bench/terminal_error" const DOOM_LOOP_THRESHOLD = 3 const log = Log.create({ service: "session.processor" }) @@ -647,6 +648,7 @@ export const layer: Layer.Layer< slog.error("process", { error: errorMessage(e), stack: e instanceof Error ? e.stack : undefined }) const error = parse(e) if (MessageV2.ContextOverflowError.isInstance(error)) { + BenchTerminalError.report("context_window") ctx.needsCompaction = true yield* bus.publish(Session.Event.Error, { sessionID: ctx.sessionID, error }) return diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index f7c59fe4cba0..a16b3a08c867 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -61,6 +61,7 @@ import * as DateTime from "effect/DateTime" import { eq } from "@/storage/db" import * as Database from "@/storage/db" import { SessionTable } from "./session.sql" +import * as BenchTerminalError from "@/bench/terminal_error" // @ts-ignore globalThis.AI_SDK_LOG_WARNINGS = false @@ -1495,6 +1496,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the } const maxSteps = agent.steps ?? Infinity const isLastStep = step >= maxSteps + if (isLastStep) BenchTerminalError.report("max_iteration") msgs = yield* insertReminders({ messages: msgs, agent, session }) const msg: MessageV2.Assistant = { diff --git a/packages/opencode/test/bench/terminal_error.test.ts b/packages/opencode/test/bench/terminal_error.test.ts new file mode 100644 index 000000000000..5985446408f7 --- /dev/null +++ b/packages/opencode/test/bench/terminal_error.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, test } from "bun:test" +import * as BenchTerminalError from "@/bench/terminal_error" + +describe("bench terminal error signals", () => { + test("detects max-iteration and context-window markers", () => { + expect(BenchTerminalError.detect(`before ${BenchTerminalError.encode("max_iteration")} after`)).toBe( + "max_iteration", + ) + expect(BenchTerminalError.detect(BenchTerminalError.encode("context_window"))).toBe("context_window") + expect(BenchTerminalError.detect("ordinary opencode stderr")).toBeUndefined() + }) + + test("prefers context overflow when both terminal states occur", () => { + expect(BenchTerminalError.prefer("max_iteration", "context_window")).toBe("context_window") + expect(BenchTerminalError.prefer("context_window", "max_iteration")).toBe("context_window") + }) + + test("writes errors that Gym classifies and preserves ordinary exit errors", () => { + expect(BenchTerminalError.toGymError(0, "max_iteration")).toBe("maximum iteration reached") + expect(BenchTerminalError.toGymError(0, "context_window")).toBe("context window exceeded") + expect(BenchTerminalError.toGymError(17)).toBe("opencode_exit_17") + expect(BenchTerminalError.toGymError(0)).toBeNull() + }) + + test("keeps terminal trajectories even when opencode exits nonzero", () => { + expect(BenchTerminalError.shouldExitSuccessfully(1, "context_window")).toBeTrue() + expect(BenchTerminalError.shouldExitSuccessfully(1, "max_iteration")).toBeTrue() + expect(BenchTerminalError.shouldExitSuccessfully(1)).toBeFalse() + }) +}) diff --git a/packages/opencode/test/provider/nemo-gym/context-overflow.test.ts b/packages/opencode/test/provider/nemo-gym/context-overflow.test.ts new file mode 100644 index 000000000000..6e6117163f5b --- /dev/null +++ b/packages/opencode/test/provider/nemo-gym/context-overflow.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, mock, test } from "bun:test" +import type { LanguageModelV3CallOptions, LanguageModelV3StreamPart } from "@ai-sdk/provider" +import { NemoGymLanguageModel } from "@/provider/sdk/nemo-gym/language-model" + +async function drain(stream: ReadableStream): Promise { + const reader = stream.getReader() + const parts: LanguageModelV3StreamPart[] = [] + while (true) { + const { done, value } = await reader.read() + if (done) break + parts.push(value) + } + return parts +} + +const CALL_OPTIONS: LanguageModelV3CallOptions = { + prompt: [{ role: "user", content: [{ type: "text", text: "hi" }] }], + tools: [{ type: "function", name: "bash", inputSchema: { type: "object", properties: {} } }], +} + +describe("NemoGymLanguageModel context overflow", () => { + test("recognizes Gym's null-content length completion as context overflow", async () => { + const originalFetch = globalThis.fetch + const fetchSpy = mock( + async () => + new Response( + JSON.stringify({ + id: "chatcmpl-123", + model: "test-model", + choices: [ + { + index: 0, + finish_reason: "length", + message: { role: "assistant", content: null, tool_calls: null }, + }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ) + // @ts-expect-error test override + globalThis.fetch = fetchSpy + + try { + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + retries: Number.MAX_SAFE_INTEGER, + }) + + const parts = await drain((await model.doStream(CALL_OPTIONS)).stream) + expect(fetchSpy).toHaveBeenCalledTimes(1) + + const error = parts.find((part) => part.type === "error") + expect(error?.type).toBe("error") + if (error?.type !== "error" || typeof error.error !== "string") throw new Error("missing stream error") + expect(JSON.parse(error.error)).toMatchObject({ + type: "error", + error: { code: "context_length_exceeded" }, + }) + } finally { + globalThis.fetch = originalFetch + } + }) +}) From c012cdfe9494d13d9dfdc30ac36e4cc2e0c83499 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Fri, 21 Aug 2026 13:47:48 -0700 Subject: [PATCH 47/49] feat: disable logging --- packages/opencode/src/bench/cli.ts | 28 +++++----------------------- packages/opencode/src/cli/cmd/run.ts | 6 ++++++ 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 5d8f9772e1f5..5c4efb969743 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -12,7 +12,7 @@ * A subprocess gives us: * - clean process isolation per instance (matters for many parallel SIFs) * - identical bootstrapping path to `opencode run`, so we don't drift - * - the JSON event stream on stdout for free (--format json) + * - a compact event-type stream on stdout * * Trajectory capture: the nemo-gym provider (registered via this config) * writes `/.json` per LLM call BEFORE returning. On @@ -384,26 +384,6 @@ function runOpencode(args: { terminalSignalBuffer = (terminalSignalBuffer + chunk).slice(-256) terminalError = BenchTerminalError.prefer(terminalError, BenchTerminalError.detect(terminalSignalBuffer)) } - // Strip bulky token-ID metadata from echoed event lines. The IDs already - // live in the llm_completions dumps; leaving them in the event stream - // makes each turn re-echo that turn's full-context prompt_token_ids -> - // O(n^2) log growth (observed: 22GB driver logs, multi-MB agent logs - // within minutes at 1024-way training concurrency). - const TOKEN_FIELDS = ["prompt_token_ids", "generation_token_ids", "generation_log_probs"] - const scrub = (line: string): string => { - if (!(line.includes('"nemo-gym"') && line.includes('"prompt_token_ids"'))) return line - try { - const evt = JSON.parse(line) - const md = evt?.part?.metadata?.["nemo-gym"] - if (md) { - for (const k of TOKEN_FIELDS) { - if (Array.isArray(md[k])) md[k] = `<${md[k].length} stripped>` - } - return JSON.stringify(evt) - } - } catch {} - return line - } const MAX_KEEP = 256 * 1024 // keep only a bounded tail for error reporting let lineBuf = "" child.stdout?.on("data", (b) => { @@ -412,9 +392,9 @@ function runOpencode(args: { lineBuf += chunk let idx: number while ((idx = lineBuf.indexOf("\n")) >= 0) { - const line = scrub(lineBuf.slice(0, idx)) + const line = lineBuf.slice(0, idx) lineBuf = lineBuf.slice(idx + 1) - // Forward to our stdout so the gym log captures the event stream. + // Forward the event type so the gym log captures progress cheaply. process.stdout.write(line + "\n") stdout = (stdout + line + "\n").slice(-MAX_KEEP) } @@ -550,6 +530,8 @@ async function main() { // Have all agent sessions report terminal states to this bench wrapper. // This is bench-only and does not alter normal opencode runs. [BenchTerminalError.ENV]: "1", + // Avoid serializing and piping full event payloads into the gym log. + OPENCODE_BENCH_EVENT_TYPES_ONLY: "1", } // Bootstrap a git repo if the SIF shipped a flat source tree (swe-bench-ext diff --git a/packages/opencode/src/cli/cmd/run.ts b/packages/opencode/src/cli/cmd/run.ts index a05b273e4489..4ab9f3be6868 100644 --- a/packages/opencode/src/cli/cmd/run.ts +++ b/packages/opencode/src/cli/cmd/run.ts @@ -411,6 +411,8 @@ export const RunCommand = effectCmd({ } async function execute(sdk: OpencodeClient) { + const benchEventTypesOnly = process.env.OPENCODE_BENCH_EVENT_TYPES_ONLY === "1" + function tool(part: ToolPart) { try { if (part.tool === ShellID.ToolID) return shell(props(part)) @@ -432,6 +434,10 @@ export const RunCommand = effectCmd({ function emit(type: string, data: Record) { if (args.format === "json") { + if (benchEventTypesOnly) { + process.stdout.write(type + EOL) + return true + } process.stdout.write(JSON.stringify({ type, timestamp: Date.now(), sessionID, ...data }) + EOL) return true } From 145000c76bb493320ccf4868cf1aac58b4a58dee Mon Sep 17 00:00:00 2001 From: sdevare-nv Date: Wed, 26 Aug 2026 12:47:20 -0700 Subject: [PATCH 48/49] Add title option to agent configuration --- packages/opencode/src/bench/cli.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 5c4efb969743..a38ef8a3877b 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -194,6 +194,9 @@ async function buildConfigDir(args: { }, }, agent: { + title: { + disable: true, + }, "swe-bench": { mode: "primary", model: `nemo-gym/${args.modelName}`, From cae838d149970d3f98a309759d78d29e041f79e0 Mon Sep 17 00:00:00 2001 From: Sugam Devare Date: Wed, 26 Aug 2026 17:32:13 -0700 Subject: [PATCH 49/49] remove duplicate import Signed-off-by: Sugam Devare --- packages/opencode/src/bench/cli.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 6b58ab8c25e0..6a566612a6be 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -33,7 +33,6 @@ import * as BenchTerminalError from "./terminal_error" import PROMPT_ANTHROPIC from "../session/prompt/anthropic.txt" import type { NemoGymReplayManifest, NemoGymReplayTurn } from "../provider/sdk/nemo-gym/language-model" import { parseReplayManifest, parseReplayMessages } from "./replay" -import * as BenchTerminalError from "./terminal_error" interface CliArgs { instanceDictPath: string