diff --git a/CLAUDE.md b/CLAUDE.md index 03dceeec..40a74ddc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,6 +28,14 @@ npm run test:discover-eval # Score discovery output quality (gates at npx tsx src/eval/rca-eval.ts # Score RCA report quality npx tsx src/eval/rca-eval.ts --save # Score + save baseline npx tsx src/eval/rca-eval.ts --compare src/eval/baselines/2026-03-22.json # Compare to baseline +# Deep-investigation (autonomous orchestrator) quality: run a batch live, then score it +node src/eval/deep-investigation-run.mjs /tmp/runs.json # live batch (server on :3000) +npx tsx src/eval/deep-eval.ts --results /tmp/runs.json # score: correct / confident-wrong / category-error rates +npx tsx src/eval/deep-eval.ts --results /tmp/runs.json --save # writes baselines/deep-latest.json +npx tsx src/eval/deep-eval.ts --results /tmp/runs.json --max-confident-wrong 0 --max-category-error 0 # CI gate (keyword) +# Ground-truth-anchored (the metric that actually tracks quality — confirm-on-healthy = false-confirm): +npx tsx src/eval/deep-investigation-groundtruth.mts > /tmp/gt.json # fetch real health via grafana-mcp +npx tsx src/eval/deep-eval.ts --results /tmp/runs.json --ground-truth /tmp/gt.json --max-false-confirm 0 --max-missed 0 npx vitest run # Run all tests npx vitest run src/path # Run a single test file npx tsc --noEmit # Type check @@ -71,6 +79,7 @@ npx tsc --noEmit # Type check | Shared PromQL parser | `src/lib/prom-metric.ts` — `extractMetricExpression()`, imported by both server metric extraction and the web MetricsPanel empty-card titles | | Investigation export helpers | `src/web/lib/exportInvestigation.ts` — `downloadPng` (html-to-image + font preload), `downloadMarkdown`, `copyMarkdown` | | RCA eval harness | `src/eval/rca-eval.ts` — scores RCA reports on 5 quality dimensions, baselines in `src/eval/baselines/` | +| Deep-investigation eval | `src/eval/deep-eval.ts` (scorer: correct / confident-wrong / category-error rates) + `src/eval/deep-investigation-run.mjs` (live batch runner). Labels: `src/eval/fixtures/deep-investigation-labels.json`. The orchestrator is non-deterministic — this turns a manual batch into objective rates | | LLM quirk workarounds | `src/agents/shared/prepare-step.ts` (`prepareStep` hook) | | LLM retry & graceful failure | `src/agents/shared/llm-retry.ts` (`withLlmRetry`, `safeAgentRetryConfig`), `src/agents/shared/llm-errors.ts` (`LlmUnavailableError`, `isLlmUnavailable`). Tool-using agent paths only retry when `readOnlyTools: true` to avoid replaying write tool calls | | Shared types | `src/types/` — RCA report, agent interfaces, LLM types, WebSocket protocol | diff --git a/skills/consul-bare-metal-discovery.md b/skills/consul-bare-metal-discovery.md index d59ba0d7..dadc9106 100644 --- a/skills/consul-bare-metal-discovery.md +++ b/skills/consul-bare-metal-discovery.md @@ -10,24 +10,9 @@ tags: - bigdata scope: - discovery - - investigation --- ## When to use -This stack runs services on bare-metal hosts (not K8s). They are registered in Consul and export health status via the `consul_health_service_status` metric. - -## When investigating a root cause (read this first) -If a service appears down but has **no Kubernetes Deployment / Pod**, that is EXPECTED here — it is a bare-metal Consul service, not a k8s workload. Do NOT report "deployment missing" or "not deployed in the cluster" as the root cause. Its health signal is the Consul metric: -``` -max by (service_name) (consul_health_service_status{service_name="",status="passing"}) -``` -A value of `0` (or no row) means the bare-metal service is failing its Consul health check — that is the real signal to investigate (check the host process, its logs via the bare-metal logLabels, and any upstream it depends on). Only conclude a k8s cause for services that actually have k8s objects. - -### To CONFIRM it (so the test actually verifies, not "absent") -When you hypothesize that a bare-metal service is unhealthy, attach this EXACT checkable prediction — the keystone matches the metric name literally, so use it verbatim: -```json -{"kind":"metric-threshold","metric":"consul_health_service_status","op":"<","value":1} -``` -Then `query` it: the evidence gather runs that metric for the service and reports its value. If the passing-status value is `< 1` (i.e. 0), the `test` move returns **satisfied** → you can `conclude`. A vague hypothesis with no `consul_health_service_status` metric-threshold prediction will always come back "couldn't verify" (absent), so the run stalls — always make the prediction this exact metric. +This stack runs services on bare-metal hosts (not K8s). They are registered in Consul and export health status via the `consul_health_service_status` metric. This runbook is for the **discovery agent** — finding those services and writing their registry entries. (Investigating a Consul service incident is a separate skill: "Consul Bare-Metal Service Investigation".) ## Discovery strategy diff --git a/skills/consul-bare-metal-investigation.md b/skills/consul-bare-metal-investigation.md new file mode 100644 index 00000000..dcb962c5 --- /dev/null +++ b/skills/consul-bare-metal-investigation.md @@ -0,0 +1,67 @@ +--- +title: Consul Bare-Metal Service Investigation +services: [] +alerts: [] +appliesToServiceMetric: consul_health_service_status +healthySignal: 'max by (service_name) (consul_health_service_status{service_name="$service",status="passing"})' +failureSignal: 'max by (service_name) (consul_health_service_status{service_name="$service",status="critical"})' +failureCause: '$service is failing its Consul health check (status=critical) — the bare-metal host process is down or unhealthy.' +identityHint: '$service is registered in Consul (identity metric consul_health_service_status) — it has NO Kubernetes Deployment by design. Investigate its Consul health (status="critical") and host process FIRST; do not form k8s hypotheses for it.' +incompatibleClaims: 'kubernetes|k8s deployment|scaled to zero|scaled to 0|zero replicas|0 replicas|namespace' +tags: + - investigation + - rca + - consul + - bare-metal + - host-process +scope: + - investigation +--- +## When to use +This runbook applies ONLY to services registered in Consul on bare-metal hosts — +identified by the `consul_health_service_status` metric. It is injected into an +investigation only when the incident service is Consul-tracked and this skill is +enabled. If the incident service is a Kubernetes workload (it has a +`kube_deployment_*` metric), this runbook does not apply — investigate its +deployment/pod state instead. + +## The health signal +A bare-metal Consul service has **no Kubernetes Deployment/Pod by design** — do NOT +report "deployment missing / not deployed in the cluster". Its health is the Consul +check, which emits one row per (node × status). Read it aggregated: +``` +max by (service_name) (consul_health_service_status{service_name="",status="passing"}) +``` +- value `1` → the service is **passing** its Consul health check (healthy on this axis). +- value `0` (or no passing row) → it is **failing** — confirm via the critical row: +``` +max by (service_name) (consul_health_service_status{service_name="",status="critical"}) +``` +A critical value of `1` is direct evidence the bare-metal service is down/unhealthy. + +## Healthy ≠ a root cause +If the passing value is `1` and the critical value is `0`, the Consul service is +**healthy** — do not manufacture a Consul cause. Either the incident is on a +different axis (the host process is up but a dependency or the data plane is +degraded — investigate that) or there is no live incident, in which case +**conclude inconclusive**. Never confirm "Consul health failing" for a service +whose passing row reads `1`. + +## To CONFIRM a failing Consul service (so the test verifies, not "absent") +When you hypothesize the service is failing its Consul check, attach a checkable +prediction whose metric the gather can reduce to a single value. Use the +aggregated passing form so the evidence comes back as one number, not mixed rows: +```json +{"kind":"metric-threshold","metric":"consul_health_service_status","op":"<","value":1} +``` +Then `query` it — the gather must run the AGGREGATED query +`max by (service_name) (consul_health_service_status{service_name="",status="passing"})` +so it returns a single value. If that value is `< 1` (i.e. `0`), the `test` returns +**satisfied** → you can `conclude` "bare-metal Consul service failing its health +check". A bare `consul_health_service_status{...}` selector (no `status` filter, no +aggregation) returns multiple 0/1 rows the keystone cannot reduce — it will always +come back "couldn't verify". Always predict/gather the aggregated passing form. + +Once confirmed, deepen the cause: investigate the host process, its logs via the +bare-metal logLabels, and any upstream dependency it relies on — but the failing +Consul health check is itself a valid, grounded root cause to report. diff --git a/skills/k8s-deployment-investigation.md b/skills/k8s-deployment-investigation.md new file mode 100644 index 00000000..58a6955d --- /dev/null +++ b/skills/k8s-deployment-investigation.md @@ -0,0 +1,61 @@ +--- +title: Kubernetes Workload Investigation +services: [] +alerts: [] +appliesToServiceMetric: kube_ +healthySignal: '(kube_deployment_status_replicas_available{deployment="$service"} == kube_deployment_spec_replicas{deployment="$service"}) and (kube_deployment_spec_replicas{deployment="$service"} > 0)' +failureSignal: 'kube_deployment_status_replicas_unavailable{deployment="$service"} > 0' +failureCause: '$service has unavailable replicas (pods not ready) — the Kubernetes deployment is degraded.' +identityHint: '$service is a Kubernetes workload (kube_deployment). Check its replica STATE first (available vs spec). A fully-available deployment is healthy — do not confirm CPU/OOM/readiness/pod faults without gathering the metric that shows the fault. spec==0 means scaled to zero (a valid root cause).' +incompatibleClaims: '\bconsul\b|consul_health|bare-metal consul' +tags: + - investigation + - rca + - kubernetes + - k8s +scope: + - investigation +--- +## When to use +This runbook applies to Kubernetes workloads — identified by a `kube_deployment_*` +(or `kube_statefulset_*` / `kube_pod_*`) metric. It is injected into an +investigation only when the incident service is k8s-tracked and this skill is +enabled. If the service is a bare-metal Consul service (it has a +`consul_health_service_status` metric, no `kube_*`), this does not apply. + +## The health signal — check it FIRST +Read the deployment's replica state before forming any pod-level hypothesis: +``` +kube_deployment_spec_replicas{deployment=""} +kube_deployment_status_replicas_available{deployment=""} +kube_deployment_status_replicas_unavailable{deployment=""} +``` +- **available == spec, unavailable == 0** → the deployment is **healthy** on its + primary signal. +- **spec == 0** → scaled to zero — a complete, valid root cause for unavailability + (predict `kube_deployment_status_replicas < 1`; it reads 0 → confirmed). With + zero replicas there are **no pods**, so pod-runtime causes (GPU/OOM/readiness) + do not apply. +- **available < spec / unavailable > 0** → some pods are down — investigate pod + status, restarts, OOMKills, image pulls, scheduling. + +## Healthy ≠ a root cause (read this before confirming) +If `available == spec` and `unavailable == 0`, the workload is **healthy** — do NOT +manufacture a cause. In particular do NOT confirm "CPU throttling", "OOMKilled", +"readiness failing", or "pods crashing" **unless you actually gathered the metric +that shows it**: +``` +rate(container_cpu_cfs_throttled_seconds_total{pod=~".*"}[5m]) # > 0 to claim throttling +rate(kube_pod_container_status_restarts_total{pod=~".*"}[15m]) # > 0 to claim crashloop +kube_pod_container_status_last_terminated_reason{reason="OOMKilled"} # to claim OOM +``` +If those return no data / zero, the symptom isn't happening — **conclude +inconclusive** ("deployment appears healthy; no incident found"), never a confirm. +A confirm must name a fault for which a gathered observation crossed a threshold. + +## To CONFIRM a real k8s incident +Attach a checkable prediction over the metric that shows the fault, e.g. +`{"kind":"metric-threshold","metric":"kube_deployment_status_replicas_unavailable","op":">","value":0}` +for unavailable replicas, or a restart-rate / throttle-rate threshold for the +specific pod-level fault. The `test` move must come back **satisfied** on real, +gathered evidence before you `conclude`. diff --git a/src/agents/orchestrator-llm.ts b/src/agents/orchestrator-llm.ts index 565ad3aa..3143005f 100644 --- a/src/agents/orchestrator-llm.ts +++ b/src/agents/orchestrator-llm.ts @@ -148,7 +148,7 @@ Moves — emit EXACTLY ONE as a single JSON object (no prose, no code fence): add a candidate cause with a CHECKABLE prediction. PREDICTION is one of: {"kind":"metric-threshold","metric":"","op":">"|"<"|">="|"<=","value":} {"kind":"log-pattern","pattern":"","present":true|false} - {"kind":"infra-status","resource":"","status":""} + {"kind":"infra-status","resource":"","status":"","present":true|false} {"kind":"change-in-window","withinMinutesBefore":} - {"move":"query","target":} gather read-only evidence for that hypothesis's prediction. - {"move":"test","target":} score that hypothesis against gathered evidence. @@ -169,7 +169,9 @@ Rules: - After a follow-cause or subagent returns findings, those findings are your BEST lead. Immediately hypothesize the specific cause they point to (with a checkable prediction) and test it — never stop right after following without turning the finding into a tested hypothesis. - CROSS-SERVICE CAUSES NEED A FOLLOW-CAUSE: observing that a dependency is unhealthy is only CORRELATIONAL. To conclude that a dependency caused this incident you MUST follow-cause into it first and establish the failure there — you cannot confirm "caused by " from the incident service's metrics alone. - GROUND EVERY CLAIM IN OBSERVED EVIDENCE: a hypothesis/rationale may only state metric values, replica counts, pod statuses, or service names you ACTUALLY saw in gathered evidence. Never invent specifics — do NOT write "1/2 replicas ready", "OOMKilled", or name a service you did not observe in the evidence. If you didn't query it, you can't claim it. A plausible-sounding story with numbers you didn't measure is a FALSE confirmation, not a root cause. -- DON'T ASSUME KUBERNETES: a service may run outside k8s (Consul-registered, a VM, an external endpoint). The absence of a k8s Deployment/Pod for a service is NOT automatically the root cause — only conclude "not deployed / deployment missing" if the evidence shows the service IS a k8s workload whose Deployment genuinely vanished (e.g. kube_deployment_* metrics existed before and are now gone). If a service has no k8s objects but is monitored elsewhere (Consul/up{}/health checks), investigate THAT health signal instead of reporting "not deployed". +- DON'T ASSUME A DEPLOYMENT PLATFORM: a service may run on any of several platforms (a container orchestrator, a registered bare-metal/VM process, an external endpoint). Determine the service's actual identity and primary health signal from the gathered evidence and the injected team-knowledge (Skills) BEFORE forming hypotheses. Do NOT conclude "not deployed / missing" just because one platform's objects are absent — the service may be monitored via a different signal; investigate THAT signal. The injected Skills tell you which signal applies to this service. +- VERIFYING AN ABSENCE (scaled to zero / no replicas / not running / deleted): the confirming signal is the ABSENCE of something, which a threshold over a RUNTIME metric cannot catch — that metric vanishes when the resource is at zero, so the gather returns no data and the cause can never be confirmed. Predict it a verifiable way instead: (a) over a STATE metric that still reports a value at zero (predict it < 1); or (b) assert the absence explicitly with present:false — e.g. {"kind":"infra-status","resource":"","status":"running","present":false}. Never predict an absence over a runtime metric that disappears at zero. The injected Skills give the exact state metric for this service's platform. +- IDLE IS NOT FAILING: a rate / ratio / percentage / throughput cause ("0% success rate", "high error rate", "queries returning zero results", "request rate dropped") is only valid if you gathered evidence of NON-ZERO traffic. Zero requests / queries / throughput means the service is IDLE, not broken — an absent denominator is not a failure. If volume is ~0 and no other signal is abnormal, conclude inconclusive rather than confirming a rate-based cause. - Be decisive — your budget is limited. Prefer the most likely cause first. Output ONLY the JSON object for your chosen move.`; @@ -276,6 +278,11 @@ export interface CreateLlmDecideMoveOptions { retryBackoffMs?: number; /** Team-knowledge skills (already formatted) appended to the system prompt. */ skillContext?: string; + /** One-line incident-service identity steer prepended to the decide-move + * prompt. Supplied by the adapter from the matched investigation skill's + * declared `identityHint` ($service already substituted) — the engine holds + * no infra literals. Undefined when no matched skill declared one. */ + identityHint?: string; } /** @@ -310,11 +317,23 @@ export function createLlmDecideMove( }); const backoffMs = opts.retryBackoffMs ?? 250; - // Append team-knowledge skills to the system rules so stack-level context (e.g. - // "these services are bare-metal Consul, not k8s") informs every move choice. - const systemPrompt = opts.skillContext - ? `${SYSTEM_PROMPT}\n\n${wrapUntrusted("team_skills", opts.skillContext)}` - : SYSTEM_PROMPT; + // Append team-knowledge skills to the system rules so stack-level, infra-type + // context (declared in the skills) informs every move choice. + let systemPrompt = opts.identityHint ? `${SYSTEM_PROMPT}\n\n${opts.identityHint}` : SYSTEM_PROMPT; + if (opts.skillContext) { + systemPrompt = `${systemPrompt}\n\n${wrapUntrusted("team_skills", opts.skillContext)}`; + } + logger.debug( + { + hasIdentityHint: !!opts.identityHint, + hasSkillContext: !!opts.skillContext, + skillContextChars: opts.skillContext?.length ?? 0, + // skill section headers present in the injected context (e.g. "### Skill: Consul Bare-Metal Service Investigation") + skillTitles: (opts.skillContext?.match(/### Skill: [^\n]+/g) ?? []), + systemPromptChars: systemPrompt.length, + }, + "decide-move: system prompt assembled", + ); return async (state) => { const basePrompt = buildStatePrompt(opts.focus, state, opts.guards); // Corrective retries on a bad reply. The model runs at temperature 0, so @@ -384,6 +403,19 @@ export interface RunAutonomousOrchestratorOptions { /** All known service names — the cross-service guard checks these too, so a * false-confirm blaming another service is caught even with an empty dep graph. */ knownServices?: string[]; + /** The incident service's discovered identity metric queries (context only). */ + incidentServiceMetrics?: string[]; + /** Identity steer for the decide-move prompt (from the matched skill's + * identityHint, $service substituted). */ + identityHint?: string; + /** incompatibleClaims regexes from the matched investigation skills — the + * service-type guard rejects a confirm matching any. */ + incompatibleClaims?: string[]; + /** Confirm-gate: returns true when the service reads healthy on its primary + * signal (from the matched skill's healthySignal). Adapter-wired query. */ + checkHealthy?: () => Promise; + /** Failure-floor: grounded failure-cause text when the service is definitely failing. */ + checkFailing?: () => Promise; /** Interactive strike-limit hook (increment 5). Absent → the strike limit * stops directly. Wired by the orchestrate adapter to the WS pause card. */ onOperatorPause?: ( @@ -396,9 +428,8 @@ export interface RunAutonomousOrchestratorOptions { /** Follow a lead: an optional operator hunch that seeds the run from move 1. */ initialLead?: string; /** Team-knowledge skills (already formatted) injected into the decide-move - * system prompt so the agent has stack-level runbook context — e.g. that - * certain services are bare-metal Consul services with no k8s Deployment, so - * "deployment missing" is the wrong conclusion for them. */ + * system prompt so the agent has stack-level, infra-type runbook context — + * the skills declare the right framing for each service's platform. */ skillContext?: string; } @@ -433,6 +464,7 @@ export async function runAutonomousOrchestrator( llmCallMs: opts.llmCallMs, onUsage: addTokens, skillContext: opts.skillContext, + identityHint: opts.identityHint, }); return runOrchestrator({ @@ -447,6 +479,10 @@ export async function runAutonomousOrchestrator( dependencies: opts.dependencies, incidentService: opts.incidentService, knownServices: opts.knownServices, + incidentServiceMetrics: opts.incidentServiceMetrics, + incompatibleClaims: opts.incompatibleClaims, + checkHealthy: opts.checkHealthy, + checkFailing: opts.checkFailing, onOperatorPause: opts.onOperatorPause, signal: opts.signal, onMoveBoundary: opts.onMoveBoundary, diff --git a/src/agents/orchestrator.test.ts b/src/agents/orchestrator.test.ts index 6f935e72..7d5ad0ca 100644 --- a/src/agents/orchestrator.test.ts +++ b/src/agents/orchestrator.test.ts @@ -774,3 +774,217 @@ describe("runOrchestrator — onMoveBoundary park hook (PR-2c)", () => { expect(result.outcome).toBe("aborted"); }); }); + +describe("runOrchestrator — observability-artifact guard", () => { + it("rejects a confirm that blames the observability tooling (Grafana/datasource) for a service incident", async () => { + const result = await runOrchestrator( + makeDeps({ + incidentService: "minimax-m25-metrics-proxy", + evaluate: () => "satisfied", + decideMove: scripted([ + { type: "hypothesize", hypothesis: h("Grafana datasource for Loki is misconfigured — datasource not found") }, + { type: "query", target: 0 }, + { type: "test", target: 0 }, + { type: "conclude", leading: 0, confidence: 0.9, rationale: "datasource not found in logs" }, + null, + ]), + }), + ); + // Query-layer artifact, not a service root cause → blocked → exhausts. + expect(result.outcome).toBe("exhausted"); + expect(result.confirmed).toBeUndefined(); + expect(result.trace.some((t) => t.move === "conclude" && /observability tooling/.test(t.detail))).toBe(true); + }); + + it("ALLOWS a datasource/Grafana cause when the incident service IS an observability component", async () => { + const result = await runOrchestrator( + makeDeps({ + incidentService: "grafana", + evaluate: () => "satisfied", + decideMove: scripted([ + { type: "hypothesize", hypothesis: h("grafana datasource provisioning failed on restart") }, + { type: "query", target: 0 }, + { type: "test", target: 0 }, + { type: "conclude", leading: 0, confidence: 0.9, rationale: "provisioning error" }, + ]), + }), + ); + expect(result.outcome).toBe("confirmed"); // grafana itself — legitimately about the observability stack + }); +}); + +describe("runOrchestrator — service-type consistency guard (incompatibleClaims)", () => { + const scriptConclude = (hyp_: string) => ({ + evaluate: () => "satisfied" as const, + decideMove: scripted([ + { type: "hypothesize", hypothesis: h(hyp_) }, + { type: "query", target: 0 }, + { type: "test", target: 0 }, + { type: "conclude", leading: 0, confidence: 0.9, rationale: "evidence backs it" }, + null, + ]), + }); + + it("rejects a confirm matching a matched skill's incompatibleClaims pattern", async () => { + const result = await runOrchestrator(makeDeps({ + incidentService: "impala", + incompatibleClaims: ["kubernetes|k8s deployment|scaled to zero"], + ...scriptConclude("impala is a Kubernetes Deployment scaled to zero replicas"), + })); + expect(result.outcome).toBe("exhausted"); + expect(result.confirmed).toBeUndefined(); + expect(result.trace.some((t) => t.move === "conclude" && /contradicts/.test(t.detail))).toBe(true); + }); + + it("allows a confirm that does NOT match the incompatibleClaims pattern", async () => { + const result = await runOrchestrator(makeDeps({ + incidentService: "vllm", + incompatibleClaims: ["\\bconsul\\b"], // k8s service: a consul cause is incompatible + ...scriptConclude("deployment is scaled to zero replicas"), + })); + expect(result.outcome).toBe("confirmed"); + }); + + it("rejects the reverse — a Consul cause for a k8s workload", async () => { + const result = await runOrchestrator(makeDeps({ + incidentService: "vllm", + incompatibleClaims: ["\\bconsul\\b|consul_health"], + ...scriptConclude("service removed from k8s but still has active consul registration"), + })); + expect(result.outcome).toBe("exhausted"); + }); + + it("stays silent when no incompatibleClaims are declared", async () => { + const result = await runOrchestrator(makeDeps({ + incidentService: "edge", + incompatibleClaims: [], + ...scriptConclude("deployment scaled to zero"), + })); + expect(result.outcome).toBe("confirmed"); + }); +}); + +describe("runOrchestrator — health-gate (skill-declared healthySignal)", () => { + const scriptConclude = () => ({ + evaluate: () => "satisfied" as const, + decideMove: scripted([ + { type: "hypothesize", hypothesis: h("something is broken") }, + { type: "query", target: 0 }, + { type: "test", target: 0 }, + { type: "conclude", leading: 0, confidence: 0.9, rationale: "evidence backs it" }, + null, + ]), + }); + + it("blocks a confirm when checkHealthy reports the service is HEALTHY", async () => { + const result = await runOrchestrator(makeDeps({ + incidentService: "impala", + checkHealthy: async () => true, + ...scriptConclude(), + })); + expect(result.outcome).toBe("exhausted"); + expect(result.confirmed).toBeUndefined(); + expect(result.trace.some((t) => t.move === "conclude" && /reads HEALTHY/.test(t.detail))).toBe(true); + }); + + it("allows a confirm when checkHealthy reports NOT healthy (false)", async () => { + const result = await runOrchestrator(makeDeps({ incidentService: "bd", checkHealthy: async () => false, ...scriptConclude() })); + expect(result.outcome).toBe("confirmed"); + }); + + it("allows a confirm when health is undeterminable (null)", async () => { + const result = await runOrchestrator(makeDeps({ incidentService: "x", checkHealthy: async () => null, ...scriptConclude() })); + expect(result.outcome).toBe("confirmed"); + }); + + it("confirms normally when no checkHealthy is wired", async () => { + const result = await runOrchestrator(makeDeps({ incidentService: "x", ...scriptConclude() })); + expect(result.outcome).toBe("confirmed"); + }); +}); + +describe("runOrchestrator — failure floor (checkFailing)", () => { + it("confirms the primary-signal failure when the run gives up while failureSignal fires", async () => { + const result = await runOrchestrator(makeDeps({ + incidentService: "bd", + checkFailing: async () => "bd is failing its Consul health check (status=critical)", + decideMove: scripted([null]), // immediate give-up → would be exhausted + })); + expect(result.outcome).toBe("confirmed"); + expect(result.confirmed?.hypothesis).toMatch(/failing its Consul health check/); + expect(result.trace.some((t) => /primary-signal floor/.test(t.detail))).toBe(true); + }); + + it("leaves the give-up outcome when checkFailing returns null (service not definitely failing)", async () => { + const result = await runOrchestrator(makeDeps({ + incidentService: "x", + checkFailing: async () => null, + decideMove: scripted([null]), + })); + expect(result.outcome).toBe("exhausted"); + }); + + it("does not override an explicit confirm", async () => { + const result = await runOrchestrator(makeDeps({ + incidentService: "bd", + checkFailing: async () => "floor cause", + evaluate: () => "satisfied", + decideMove: scripted([ + { type: "hypothesize", hypothesis: h("real cause") }, + { type: "query", target: 0 }, + { type: "test", target: 0 }, + { type: "conclude", leading: 0, confidence: 0.9, rationale: "x" }, + ]), + })); + expect(result.outcome).toBe("confirmed"); + expect(result.confirmed?.hypothesis).toBe("real cause"); // the agent's confirm, not the floor + }); +}); + +describe("runOrchestrator — grounding gate (pod-runtime cause vs zero pods)", () => { + const concludeOn = (hyp_: string) => ({ + evaluate: () => "satisfied" as const, + decideMove: scripted([ + { type: "hypothesize", hypothesis: h(hyp_) }, + { type: "query", target: 0 }, + { type: "test", target: 0 }, + { type: "conclude", leading: 0, confidence: 0.95, rationale: "evidence backs it" }, + null, + ]), + }); + + it("rejects a GPU/pod-runtime confirm when evidence shows zero replicas (the fabrication)", async () => { + const result = await runOrchestrator( + makeDeps({ + incidentService: "vllm-bench", + gatherEvidence: async () => [{ phase: "metrics", subject: "kube_deployment_status_replicas{deployment=\"vllm-bench\"}", value: 0 }], + ...concludeOn("GPU resource exhaustion or CUDA error causing service degradation"), + }), + ); + expect(result.outcome).toBe("exhausted"); + expect(result.confirmed).toBeUndefined(); + expect(result.trace.some((t) => t.move === "conclude" && /zero running pods/.test(t.detail))).toBe(true); + }); + + it("ALLOWS the scaled-to-zero cause itself with the same zero-replica evidence", async () => { + const result = await runOrchestrator( + makeDeps({ + incidentService: "vllm-bench", + gatherEvidence: async () => [{ phase: "metrics", subject: "kube_deployment_status_replicas{deployment=\"vllm-bench\"}", value: 0 }], + ...concludeOn("deployment is scaled to zero replicas"), + }), + ); + expect(result.outcome).toBe("confirmed"); + }); + + it("does NOT fire when no zero-replica evidence was gathered (a real GPU incident can confirm)", async () => { + const result = await runOrchestrator( + makeDeps({ + incidentService: "vllm-bench", + gatherEvidence: async () => [{ phase: "metrics", subject: "DCGM_FI_DEV_GPU_UTIL{pod=\"vllm-bench-0\"}", value: 100 }], + ...concludeOn("GPU utilization saturated at 100% causing latency"), + }), + ); + expect(result.outcome).toBe("confirmed"); + }); +}); diff --git a/src/agents/orchestrator.ts b/src/agents/orchestrator.ts index 1203c80f..18cc76df 100644 --- a/src/agents/orchestrator.ts +++ b/src/agents/orchestrator.ts @@ -158,6 +158,25 @@ export interface OrchestratorDeps { * another service is caught even when the dependency graph is empty/missing * (inc-7 #3 — the keystone can be independently true but not causally linked). */ knownServices?: string[]; + /** The incident service's discovered identity metric queries — passed through + * for context; the engine no longer hardcodes any infra-type detection on it. */ + incidentServiceMetrics?: string[]; + /** Regex patterns (from the matched investigation skills' `incompatibleClaims`) + * describing conclusion text that contradicts this service's infra type. The + * service-type guard rejects a confirm matching any of them. Infra knowledge + * lives in the skill, not here — the engine just applies the declared pattern. */ + incompatibleClaims?: string[]; + /** Confirm-gate (from the matched skill's `healthySignal`): returns true when + * the incident service reads HEALTHY on its primary signal (→ block the confirm, + * force inconclusive), false when not-healthy, null when undeterminable. Wired + * by the adapter to a PromQL query of the skill-declared signal — no infra + * literals in the engine. */ + checkHealthy?: () => Promise; + /** Failure-floor (from the matched skill's failureSignal + failureCause): + * returns the grounded failure-cause text when the service is DEFINITELY + * failing on its primary signal, else null. Used so a broken service is never + * missed when the run would otherwise give up. No infra literals in the engine. */ + checkFailing?: () => Promise; /** * Strikes-limit hook: instead of silently stopping at the strike limit, ask a * human. "continue" resets the strike counter and resumes the loop (the other @@ -327,6 +346,24 @@ export async function runOrchestrator(deps: OrchestratorDeps): Promise => { + if (deps.checkFailing) { + try { + const cause = await deps.checkFailing(); + if (cause) { + record({ move: "conclude", detail: `confirmed (primary-signal floor): ${cause}` }); + return finish("confirmed", { hypothesis: cause, prediction: { kind: "metric-threshold", metric: "primary-signal", op: ">", value: 0 } }); + } + } catch { /* undeterminable → fall through to the original outcome */ } + } + return finish(outcome); + }; + while (moves < MAX_MOVES) { // Guards are checked BEFORE spending the next move so a tripped limit never // does "one more" expensive thing. @@ -338,9 +375,9 @@ export async function runOrchestrator(deps: OrchestratorDeps): Promise= deps.guards.maxTokens) return finish("budget-exhausted"); - if (toolCalls >= deps.guards.maxToolCalls) return finish("tool-cap"); - if (elapsed() >= deps.guards.wallClockMs) return finish("wall-clock"); + if (tokensSpent >= deps.guards.maxTokens) return await giveUp("budget-exhausted"); + if (toolCalls >= deps.guards.maxToolCalls) return await giveUp("tool-cap"); + if (elapsed() >= deps.guards.wallClockMs) return await giveUp("wall-clock"); // strikes → operator pause: the design's headline safety feature. The signal // is ambiguous (N hypotheses failed, nothing discriminating emerged); rather // than guess, hand the call to a human (if wired) — who can resume the run @@ -364,12 +401,12 @@ export async function runOrchestrator(deps: OrchestratorDeps): Promise= MAX_STALL) return finish("inconclusive"); + if (stall >= MAX_STALL) return await giveUp("inconclusive"); // Fast no-evidence bail: several queries in and nothing surfaced anywhere → // the service is quiet. Stop now rather than burning the full run (inc-7 #5). - if (toolCalls >= NO_EVIDENCE_BAIL_QUERIES && evidence.length === 0) return finish("inconclusive"); + if (toolCalls >= NO_EVIDENCE_BAIL_QUERIES && evidence.length === 0) return await giveUp("inconclusive"); const state: OrchestratorState = { hypotheses, @@ -399,7 +436,7 @@ export async function runOrchestrator(deps: OrchestratorDeps): Promise { + try { return new RegExp(p, "i").test(lead.hypothesis.hypothesis); } catch { return false; } + }); + if (incompatible) { + record({ + move: "conclude", + detail: `not confirmed — "${lead.hypothesis.hypothesis}" contradicts ${deps.incidentService ?? "this service"}'s known infrastructure type (it matched a team-knowledge skill's incompatible-cause pattern). Investigate a cause consistent with the service's actual type.`, + }); + stall++; + break; + } + // GROUNDING GATE: a confirm that blames a pod-RUNTIME failure (GPU/CUDA + // exhaustion, OOM, readiness/liveness, crash-loop, CPU throttling) is + // impossible when the gathered evidence shows the workload has zero + // running pods (scaled to zero). With no pods, there is nothing to + // exhaust a GPU or fail a readiness probe — such a cause is a + // fabrication/misattribution (e.g. cluster-wide GPU usage pinned on a + // service that has no pods). Only fires when an observation actually + // establishes zero replicas/pods, so it's grounded in real evidence. + const sawZeroReplicas = evidence.some((o) => { + const subj = (o.subject ?? "").toLowerCase(); + const txt = (o.text ?? "").toLowerCase(); + const isReplicaSignal = /replica|kube_deployment|kube_pod|\bpods?\b/.test(subj); + if (isReplicaSignal && o.value === 0) return true; + return /\b(0 replicas|zero replicas|scaled to zero|scaled to 0|no (running |ready )?pods|no pods (running|scheduled|exist))\b/.test(`${subj} ${txt}`); + }); + const claimsPodRuntime = /\b(gpu|cuda|oom|out of memory|readiness|liveness|crash[- ]?loop|cpu throttl)\b/i.test(lead.hypothesis.hypothesis); + if (sawZeroReplicas && claimsPodRuntime) { + record({ + move: "conclude", + detail: `not confirmed — "${lead.hypothesis.hypothesis}" blames a pod-runtime failure, but the gathered evidence shows ${deps.incidentService ?? "this workload"} has zero running pods (scaled to zero). With no pods there is no GPU/OOM/readiness failure to have — the cause is the scaled-to-zero state itself, not a runtime fault.`, + }); + stall++; + break; + } + // HEALTH-GATE: if a matched skill declared a healthySignal and it reads + // HEALTHY, the service has no incident to explain — never confirm a + // manufactured cause; conclude inconclusive. Skill-declared PromQL; the + // engine only evaluates it (no infra literals). The deterministic backstop + // for the over-confirm that prompt guidance alone could not stop. + if (deps.checkHealthy) { + const healthy = await deps.checkHealthy(); + if (healthy === true) { + record({ + move: "conclude", + detail: `not confirmed — ${deps.incidentService ?? "this service"} reads HEALTHY on its primary signal (the matched skill's healthySignal). A healthy service has no incident to confirm; concluding inconclusive rather than manufacturing a cause.`, + }); + stall++; + break; + } + } record({ move: "conclude", detail: `confirmed: ${lead.hypothesis.hypothesis}` }); return finish("confirmed", lead.hypothesis); } @@ -579,5 +689,5 @@ export async function runOrchestrator(deps: OrchestratorDeps): Promise { + it("marks a confirmed correct Consul cause as correct", () => { + const s = scoreRun({ service: "impala", outcome: "confirmed", rootCause: "impala is a bare-metal Consul service with failing health checks" }, consulLabel); + expect(s.verdict).toBe("correct"); + expect(s.categoryError).toBe(false); + }); + + it("flags a k8s cause for a Consul service as confident-wrong + category-error", () => { + const s = scoreRun({ service: "impala", outcome: "confirmed", rootCause: "impala is a Kubernetes Deployment scaled to zero replicas" }, consulLabel); + expect(s.verdict).toBe("confident-wrong"); + expect(s.categoryError).toBe(true); + expect(s.matchedWrongPattern).toBeTruthy(); + }); + + it("flags a fabricated GPU cause for a scaled-to-zero k8s service as confident-wrong (not category-error)", () => { + const s = scoreRun({ service: "vllm-bench", outcome: "confirmed", rootCause: "GPU resource exhaustion or CUDA error causing degradation" }, k8sLabel); + expect(s.verdict).toBe("confident-wrong"); + expect(s.matchedWrongPattern).toBe("gpu"); + expect(s.categoryError).toBe(false); // gpu is a fabrication, not a cross-type term + }); + + it("accepts the 'not running / cleaned up' family as correct for a scaled-to-zero k8s service", () => { + const s = scoreRun({ service: "vllm-bench", outcome: "confirmed", rootCause: "Benchmark job was never created or was cleaned up after completion" }, k8sLabel); + expect(s.verdict).toBe("correct"); + }); + + it("treats a wall-clock as honest-inconclusive when listed acceptable", () => { + const s = scoreRun({ service: "impala", outcome: "wall-clock", rootCause: null }, consulLabel); + expect(s.verdict).toBe("honest-inconclusive"); + }); + + it("treats an unlisted non-confirmed outcome as unexpected-inconclusive", () => { + const s = scoreRun({ service: "impala", outcome: "error", rootCause: null }, consulLabel); + expect(s.verdict).toBe("unexpected-inconclusive"); + }); + + it("marks a run with no label as unlabeled", () => { + const s = scoreRun({ service: "unknown-svc", outcome: "confirmed", rootCause: "anything" }, undefined); + expect(s.verdict).toBe("unlabeled"); + }); + + it("a confirmed cause with no expected keyword is confident-wrong even without a wrong pattern", () => { + const s = scoreRun({ service: "vllm-bench", outcome: "confirmed", rootCause: "network partition between regions" }, k8sLabel); + expect(s.verdict).toBe("confident-wrong"); + expect(s.matchedWrongPattern).toBeNull(); + }); +}); + +describe("scoreRuns aggregate", () => { + it("computes rates and per-service breakdown over a mixed batch", () => { + const runs = [ + { service: "impala", outcome: "confirmed", rootCause: "Consul health check failing" }, // correct + { service: "impala", outcome: "wall-clock", rootCause: null }, // honest + { service: "impala", outcome: "confirmed", rootCause: "Kubernetes Deployment scaled to zero" }, // confident-wrong + cat-error + { service: "vllm-bench", outcome: "confirmed", rootCause: "deployment scaled to zero, 0 replicas" }, // correct + { service: "vllm-bench", outcome: "confirmed", rootCause: "GPU resource exhaustion" }, // confident-wrong + ]; + const card = scoreRuns(runs, [consulLabel, k8sLabel]); + expect(card.total).toBe(5); + expect(card.labeled).toBe(5); + expect(card.correct).toBe(2); + expect(card.confidentWrong).toBe(2); + expect(card.categoryError).toBe(1); + expect(card.honestInconclusive).toBe(1); + expect(card.correctRate).toBe(40); + expect(card.confidentWrongRate).toBe(40); + expect(card.perService["impala"]).toEqual({ runs: 3, correct: 1, confidentWrong: 1 }); + expect(card.perService["vllm-bench"]).toEqual({ runs: 2, correct: 1, confidentWrong: 1 }); + }); + + it("zero labeled runs → 0 rates, not NaN", () => { + const card = scoreRuns([{ service: "ghost", outcome: "confirmed", rootCause: "x" }], [consulLabel]); + expect(card.labeled).toBe(0); + expect(card.correctRate).toBe(0); + expect(card.confidentWrongRate).toBe(0); + }); +}); + +describe("ground-truth-anchored scoring (F2)", () => { + it("confirm on a HEALTHY service = false-confirm (the false-PASS the keyword scorer missed)", () => { + expect(scoreRunGroundTruth({ service: "impala", outcome: "confirmed", rootCause: "data plane failure" }, "healthy").verdict).toBe("false-confirm"); + }); + it("confirm on an UNHEALTHY service = correct-confirm", () => { + expect(scoreRunGroundTruth({ service: "bd", outcome: "confirmed", rootCause: "consul critical" }, "unhealthy").verdict).toBe("correct-confirm"); + }); + it("decline on a HEALTHY service = correct-decline", () => { + expect(scoreRunGroundTruth({ service: "kudu", outcome: "wall-clock", rootCause: null }, "healthy").verdict).toBe("correct-decline"); + }); + it("decline on an UNHEALTHY service = missed-incident", () => { + expect(scoreRunGroundTruth({ service: "bd", outcome: "wall-clock", rootCause: null }, "unhealthy").verdict).toBe("missed-incident"); + }); + it("unknown ground truth = unknown (not scored either way)", () => { + expect(scoreRunGroundTruth({ service: "x", outcome: "confirmed", rootCause: "y" }, "unknown").verdict).toBe("unknown"); + }); + + it("aggregate flags false-confirms and missed-incidents (would have caught the false PASS)", () => { + // Mirrors the real 8-run finding: confirms on healthy services + a missed real incident. + const runs = [ + { service: "impala", outcome: "confirmed", rootCause: "daemon failing" }, // healthy → false-confirm + { service: "ingestion", outcome: "confirmed", rootCause: "cpu throttling" }, // healthy → false-confirm + { service: "bd", outcome: "wall-clock", rootCause: null }, // unhealthy → missed + { service: "kudu", outcome: "wall-clock", rootCause: null }, // healthy → correct-decline + ]; + const gt = { impala: "healthy", ingestion: "healthy", bd: "unhealthy", kudu: "healthy" } as const; + const g = scoreRunsGroundTruth(runs, gt); + expect(g.falseConfirm).toBe(2); + expect(g.missedIncident).toBe(1); + expect(g.correctDecline).toBe(1); + expect(g.correctConfirm).toBe(0); + }); +}); diff --git a/src/eval/deep-eval.ts b/src/eval/deep-eval.ts new file mode 100644 index 00000000..06ef0e37 --- /dev/null +++ b/src/eval/deep-eval.ts @@ -0,0 +1,357 @@ +/** + * deep-eval.ts — deep-investigation (autonomous orchestrator) quality scoring CLI + * + * The orchestrator is LLM-driven and non-deterministic: the same incident can + * confirm the right cause, wall-clock, or confirm a wrong one on different runs. + * Eyeballing a manual batch can't tell whether a change helped. This harness + * turns a batch of orchestrator runs into objective rates so accuracy regressions + * (category errors, fabrications, confident-wrong confirms) are measured, not guessed. + * + * It scores RUN RESULTS (not live execution) so it's fast and deterministic. Produce + * the results with the live runner, then score them here. + * + * Usage: + * npx tsx src/eval/deep-eval.ts --results /tmp/orch-batch-results.json + * npx tsx src/eval/deep-eval.ts --results runs.json --save + * npx tsx src/eval/deep-eval.ts --results runs.json --compare src/eval/baselines/deep-2026-06-11.json + * npx tsx src/eval/deep-eval.ts --results runs.json --max-confident-wrong 0 --min-correct 50 + * + * Results file: a JSON array of run objects. Only three fields are read: + * { "service": "", "outcome": "confirmed|wall-clock|...", "rootCause": "" } + * (the live runner's output is a superset of this — it is read directly). + * + * Gates (CI): + * --max-confident-wrong N : exit 1 if confident-wrong runs exceed N (default: off) + * --max-category-error N : exit 1 if category-error runs exceed N (default: off) + * --min-correct PCT : exit 1 if correct-rate (of labeled runs) is below PCT + */ + +import { readFileSync, writeFileSync, mkdirSync } from "fs"; +import { resolve, dirname } from "path"; +import { fileURLToPath } from "url"; + +// ── Types ─────────────────────────────────────────────────────────────────── + +export interface IncidentLabel { + service: string; + infraType: string; + summary?: string; + /** A confirmed cause is CORRECT if its text contains any of these (case-insensitive). */ + expectedCauseKeywords: string[]; + /** A confirmed cause is clearly WRONG if its text contains any of these — a + * cross-infra-type category error or a fabrication for this service. */ + wrongCausePatterns: string[]; + /** Non-confirmed outcomes that are HONEST (not a quality failure) for this incident. */ + acceptableInconclusive: string[]; +} + +export interface DeepRun { + service: string; + outcome?: string | null; + rootCause?: string | null; +} + +export type RunVerdict = + | "correct" // confirmed AND matches an expected cause AND no wrong pattern + | "confident-wrong" // confirmed but not correct (category error / fabrication / off-target) + | "honest-inconclusive" // not confirmed, and the outcome is an acceptable decline + | "unexpected-inconclusive" // not confirmed, but the outcome wasn't listed acceptable + | "unlabeled"; // no label for this service — can't score + +export interface RunScore { + service: string; + outcome: string; + rootCause: string | null; + verdict: RunVerdict; + /** confident-wrong specifically because the cause names the other infra type. */ + categoryError: boolean; + /** confident-wrong specifically because the cause matched a flagged wrong pattern. */ + matchedWrongPattern: string | null; +} + +export interface Scorecard { + total: number; + labeled: number; + correct: number; + confidentWrong: number; + categoryError: number; + honestInconclusive: number; + unexpectedInconclusive: number; + unlabeled: number; + /** correct / labeled, as a 0–100 integer (0 when no labeled runs). */ + correctRate: number; + /** confidentWrong / labeled, 0–100. The "do no harm" bar — target 0. */ + confidentWrongRate: number; + perService: Record; + runs: RunScore[]; +} + +// ── Scoring (pure, exported for tests) ─────────────────────────────────────── + +const norm = (s: string | null | undefined): string => (s ?? "").toLowerCase(); + +function firstMatch(haystack: string, needles: string[]): string | null { + for (const n of needles) { + if (n && haystack.includes(n.toLowerCase())) return n; + } + return null; +} + +/** Score one run against its label. */ +export function scoreRun(run: DeepRun, label: IncidentLabel | undefined): RunScore { + const outcome = (run.outcome ?? "").toLowerCase() || "unknown"; + const cause = run.rootCause ?? null; + const base = { service: run.service, outcome, rootCause: cause }; + + if (!label) { + return { ...base, verdict: "unlabeled", categoryError: false, matchedWrongPattern: null }; + } + + if (outcome !== "confirmed") { + const honest = label.acceptableInconclusive.map((s) => s.toLowerCase()).includes(outcome); + return { + ...base, + verdict: honest ? "honest-inconclusive" : "unexpected-inconclusive", + categoryError: false, + matchedWrongPattern: null, + }; + } + + // Confirmed: judge the cause text. + const text = norm(cause); + const wrong = firstMatch(text, label.wrongCausePatterns); + const hasExpected = firstMatch(text, label.expectedCauseKeywords) !== null; + const correct = hasExpected && !wrong; + // A category error is a wrong confirm that names the OTHER infra type. We treat + // any matched wrong pattern as the trigger; the distinction in reporting is + // whether the matched pattern is an infra-type term vs. a fabrication term, but + // for the headline rate both are "confident-wrong". + const INFRA_TYPE_TERMS = ["kubernetes", "k8s", "consul", "namespace", "deployment"]; + const categoryError = wrong !== null && INFRA_TYPE_TERMS.some((t) => wrong.toLowerCase().includes(t)); + + return { + ...base, + verdict: correct ? "correct" : "confident-wrong", + categoryError, + matchedWrongPattern: wrong, + }; +} + +/** Aggregate a batch of runs against the label set. */ +export function scoreRuns(runs: DeepRun[], labels: IncidentLabel[]): Scorecard { + const byService = new Map(labels.map((l) => [l.service, l])); + const scored = runs.map((r) => scoreRun(r, byService.get(r.service))); + + const card: Scorecard = { + total: scored.length, + labeled: scored.filter((s) => s.verdict !== "unlabeled").length, + correct: scored.filter((s) => s.verdict === "correct").length, + confidentWrong: scored.filter((s) => s.verdict === "confident-wrong").length, + categoryError: scored.filter((s) => s.categoryError).length, + honestInconclusive: scored.filter((s) => s.verdict === "honest-inconclusive").length, + unexpectedInconclusive: scored.filter((s) => s.verdict === "unexpected-inconclusive").length, + unlabeled: scored.filter((s) => s.verdict === "unlabeled").length, + correctRate: 0, + confidentWrongRate: 0, + perService: {}, + runs: scored, + }; + if (card.labeled > 0) { + card.correctRate = Math.round((card.correct / card.labeled) * 100); + card.confidentWrongRate = Math.round((card.confidentWrong / card.labeled) * 100); + } + for (const s of scored) { + if (s.verdict === "unlabeled") continue; + const ps = (card.perService[s.service] ??= { runs: 0, correct: 0, confidentWrong: 0 }); + ps.runs++; + if (s.verdict === "correct") ps.correct++; + if (s.verdict === "confident-wrong") ps.confidentWrong++; + } + return card; +} + +// ── Ground-truth-anchored scoring (F2) ─────────────────────────────────────── +// +// Keyword scoring can't tell a grounded-SOUNDING false-confirm from a real cause +// — it gave a false PASS. This scores each run against the service's ACTUAL +// health (queried live, e.g. by deep-investigation-groundtruth.mjs): a confirm +// on a healthy service is a FALSE-CONFIRM; a decline on a broken service is a +// MISSED-INCIDENT. These are the rates that actually track quality. + +export type GroundTruthState = "healthy" | "unhealthy" | "unknown"; + +export type GtVerdict = + | "false-confirm" // confirmed a cause on a service that is actually healthy + | "correct-confirm" // confirmed on a genuinely-unhealthy service + | "correct-decline" // declined on a healthy service (right call) + | "missed-incident" // declined on a genuinely-unhealthy service + | "unknown"; // ground truth undeterminable for this service + +export interface GtRunScore { + service: string; + outcome: string; + state: GroundTruthState; + verdict: GtVerdict; +} + +export interface GtScorecard { + total: number; + judged: number; // ground truth known + falseConfirm: number; // ← target 0 + missedIncident: number; // ← target 0 + correctConfirm: number; + correctDecline: number; + unknown: number; + runs: GtRunScore[]; +} + +/** Score one run against its service's ground-truth health state. */ +export function scoreRunGroundTruth(run: DeepRun, state: GroundTruthState): GtRunScore { + const outcome = (run.outcome ?? "").toLowerCase() || "unknown"; + const confirmed = outcome === "confirmed"; + let verdict: GtVerdict; + if (state === "unknown") verdict = "unknown"; + else if (confirmed) verdict = state === "healthy" ? "false-confirm" : "correct-confirm"; + else verdict = state === "healthy" ? "correct-decline" : "missed-incident"; + return { service: run.service, outcome, state, verdict }; +} + +/** Aggregate runs against a ground-truth map (service → state). */ +export function scoreRunsGroundTruth(runs: DeepRun[], groundTruth: Record): GtScorecard { + const scored = runs.map((r) => scoreRunGroundTruth(r, groundTruth[r.service] ?? "unknown")); + return { + total: scored.length, + judged: scored.filter((s) => s.verdict !== "unknown").length, + falseConfirm: scored.filter((s) => s.verdict === "false-confirm").length, + missedIncident: scored.filter((s) => s.verdict === "missed-incident").length, + correctConfirm: scored.filter((s) => s.verdict === "correct-confirm").length, + correctDecline: scored.filter((s) => s.verdict === "correct-decline").length, + unknown: scored.filter((s) => s.verdict === "unknown").length, + runs: scored, + }; +} + +// ── CLI ────────────────────────────────────────────────────────────────────── + +function loadLabels(): IncidentLabel[] { + const here = dirname(fileURLToPath(import.meta.url)); + const raw = JSON.parse(readFileSync(resolve(here, "fixtures/deep-investigation-labels.json"), "utf-8")); + return raw.labels as IncidentLabel[]; +} + +function getFlag(name: string): string | undefined { + const i = process.argv.indexOf(name); + return i >= 0 ? process.argv[i + 1] : undefined; +} + +function printCard(card: Scorecard): void { + const pad = (n: number) => String(n).padStart(3); + console.log("\n=== Deep-investigation eval ==="); + console.log(`runs: ${card.total} (labeled ${card.labeled}, unlabeled ${card.unlabeled})`); + console.log(` ✓ correct ${pad(card.correct)} (${card.correctRate}% of labeled)`); + console.log(` ✗ confident-wrong ${pad(card.confidentWrong)} (${card.confidentWrongRate}% of labeled) ← target 0`); + console.log(` of which category-error ${pad(card.categoryError)}`); + console.log(` ◐ honest-inconclusive ${pad(card.honestInconclusive)}`); + console.log(` ? unexpected-inconclusive ${pad(card.unexpectedInconclusive)}`); + console.log("\nper service:"); + for (const [svc, ps] of Object.entries(card.perService)) { + console.log(` ${svc}: ${ps.correct}/${ps.runs} correct, ${ps.confidentWrong} confident-wrong`); + } + const wrong = card.runs.filter((r) => r.verdict === "confident-wrong"); + if (wrong.length) { + console.log("\nconfident-wrong confirms:"); + for (const r of wrong) { + console.log(` ✗ ${r.service}: "${r.rootCause}"${r.matchedWrongPattern ? ` [matched "${r.matchedWrongPattern}"]` : ""}`); + } + } +} + +function main(): void { + const resultsPath = getFlag("--results"); + if (!resultsPath) { + console.error("error: --results is required (a JSON array of run objects)"); + process.exit(2); + } + const runs = JSON.parse(readFileSync(resolve(resultsPath), "utf-8")) as DeepRun[]; + if (!Array.isArray(runs)) { + console.error("error: results file must be a JSON array"); + process.exit(2); + } + const labels = loadLabels(); + const card = scoreRuns(runs, labels); + printCard(card); + + // F2: ground-truth-anchored scoring (the real quality metric). Pass a map of + // service → "healthy"|"unhealthy"|"unknown" (produce it live with + // deep-investigation-groundtruth.mjs). + let gtFailed = false; + const gtPath = getFlag("--ground-truth"); + if (gtPath) { + const gt = JSON.parse(readFileSync(resolve(gtPath), "utf-8")) as Record; + const g = scoreRunsGroundTruth(runs, gt); + console.log("\n=== Ground-truth-anchored (vs actual service health) ==="); + console.log(`judged: ${g.judged}/${g.total} (unknown ground truth: ${g.unknown})`); + console.log(` ✗ false-confirm ${String(g.falseConfirm).padStart(3)} (confirmed a cause on a HEALTHY service) ← target 0`); + console.log(` ✗ missed-incident ${String(g.missedIncident).padStart(3)} (declined on a BROKEN service) ← target 0`); + console.log(` ✓ correct-confirm ${String(g.correctConfirm).padStart(3)}`); + console.log(` ✓ correct-decline ${String(g.correctDecline).padStart(3)}`); + for (const r of g.runs.filter((x) => x.verdict === "false-confirm" || x.verdict === "missed-incident")) { + console.log(` ${r.verdict === "false-confirm" ? "✗ false-confirm" : "✗ missed"}: ${r.service} (actually ${r.state}, run ${r.outcome})`); + } + const maxFC = getFlag("--max-false-confirm"); + if (maxFC !== undefined && g.falseConfirm > Number(maxFC)) { console.error(`\nGATE FAIL: false-confirm ${g.falseConfirm} > ${maxFC}`); gtFailed = true; } + const maxMI = getFlag("--max-missed"); + if (maxMI !== undefined && g.missedIncident > Number(maxMI)) { console.error(`GATE FAIL: missed-incident ${g.missedIncident} > ${maxMI}`); gtFailed = true; } + } + + if (process.argv.includes("--save")) { + const here = dirname(fileURLToPath(import.meta.url)); + const dir = resolve(here, "baselines"); + mkdirSync(dir, { recursive: true }); + const summary = { + correctRate: card.correctRate, + confidentWrongRate: card.confidentWrongRate, + correct: card.correct, + confidentWrong: card.confidentWrong, + categoryError: card.categoryError, + labeled: card.labeled, + perService: card.perService, + }; + const out = resolve(dir, "deep-latest.json"); + writeFileSync(out, JSON.stringify(summary, null, 2)); + console.log(`\nsaved baseline → ${out}`); + } + + const comparePath = getFlag("--compare"); + if (comparePath) { + const base = JSON.parse(readFileSync(resolve(comparePath), "utf-8")); + const d = (k: "correctRate" | "confidentWrongRate") => card[k] - (base[k] ?? 0); + console.log(`\nvs baseline ${comparePath}:`); + console.log(` correctRate: ${base.correctRate ?? "?"}% → ${card.correctRate}% (${d("correctRate") >= 0 ? "+" : ""}${d("correctRate")})`); + console.log(` confidentWrongRate ${base.confidentWrongRate ?? "?"}% → ${card.confidentWrongRate}% (${d("confidentWrongRate") >= 0 ? "+" : ""}${d("confidentWrongRate")})`); + } + + // Gates + let failed = false; + const maxCW = getFlag("--max-confident-wrong"); + if (maxCW !== undefined && card.confidentWrong > Number(maxCW)) { + console.error(`\nGATE FAIL: confident-wrong ${card.confidentWrong} > ${maxCW}`); + failed = true; + } + const maxCE = getFlag("--max-category-error"); + if (maxCE !== undefined && card.categoryError > Number(maxCE)) { + console.error(`GATE FAIL: category-error ${card.categoryError} > ${maxCE}`); + failed = true; + } + const minCorrect = getFlag("--min-correct"); + if (minCorrect !== undefined && card.correctRate < Number(minCorrect)) { + console.error(`GATE FAIL: correctRate ${card.correctRate}% < ${minCorrect}%`); + failed = true; + } + process.exit(failed || gtFailed ? 1 : 0); +} + +// Only run main() as a CLI, not when imported by tests. +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} diff --git a/src/eval/deep-investigation-groundtruth.mts b/src/eval/deep-investigation-groundtruth.mts new file mode 100644 index 00000000..baef1b09 --- /dev/null +++ b/src/eval/deep-investigation-groundtruth.mts @@ -0,0 +1,50 @@ +// Ground-truth fetcher for F2. Queries each labeled service's REAL health via +// grafana-mcp, using the investigation skills' declared healthySignal / +// failureSignal (so it's consistent with what the engine's health-gate + floor +// evaluate). Emits {service: "healthy"|"unhealthy"|"unknown"} JSON for +// `deep-eval.ts --ground-truth`. +// +// Usage (server NOT required — connects its own MCP client): +// npx tsx src/eval/deep-investigation-groundtruth.mts > gt.json +// (stackDir = the data/ whose providers.yaml + service set you ran) +import { readFileSync } from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parse as parseYaml } from "yaml"; +import { createMcpProvider } from "../mcp/provider.js"; +import { queryInstantValue } from "../server/prometheus-query.js"; +import { SkillStore } from "../skills/store.js"; + +const stackDir = process.argv[2] || "default"; +const prov = (parseYaml(readFileSync(`data/${stackDir}/providers.yaml`, "utf-8")) as any[]).find((p) => p.name === "grafana-mcp"); +if (!prov) { console.error(`no grafana-mcp provider in data/${stackDir}/providers.yaml`); process.exit(1); } +const provider = createMcpProvider({ name: prov.name, roles: prov.roles, mcpServer: prov.mcpServer } as any, 30000); + +const store = new SkillStore({ dir: "skills", maxPerQuery: 50, maxCharsPerSkill: 4000 }); +await store.loadAll(); +const skills = store.getAllForScope("investigation").filter((s) => s.healthySignal); + +const here = dirname(fileURLToPath(import.meta.url)); +const labels = JSON.parse(readFileSync(resolve(here, "fixtures/deep-investigation-labels.json"), "utf-8")).labels as Array<{ service: string }>; + +const out: Record = {}; +for (const { service } of labels) { + let state = "unknown"; + for (const sk of skills) { + const hv = await queryInstantValue([provider], sk.healthySignal!.replaceAll("$service", service)).catch(() => null); + if (hv === null) continue; // this skill's signal doesn't apply to this service + if (hv >= 1) { state = "healthy"; break; } + // healthySignal present but not healthy → confirm via failureSignal + if (sk.failureSignal) { + const fv = await queryInstantValue([provider], sk.failureSignal.replaceAll("$service", service)).catch(() => null); + state = fv !== null && fv >= 1 ? "unhealthy" : "unknown"; + } else { + state = "unhealthy"; + } + break; + } + out[service] = state; +} +console.log(JSON.stringify(out, null, 2)); +await provider.client.disconnect().catch(() => {}); +process.exit(0); diff --git a/src/eval/deep-investigation-run.mjs b/src/eval/deep-investigation-run.mjs new file mode 100644 index 00000000..a7dc5c4f --- /dev/null +++ b/src/eval/deep-investigation-run.mjs @@ -0,0 +1,118 @@ +// deep-investigation-run.mjs — live runner for the deep-investigation eval. +// +// Drives the orchestrator over a batch of incidents against a RUNNING server, +// records each outcome + confirmed cause, and writes a results JSON that +// deep-eval.ts scores. This makes the previously-manual batch reproducible. +// +// Per incident: (1) run a baseline investigation, (2) fire orchestrator_investigate +// on it, (3) auto-continue at operator-pauses (capped), (4) record the outcome. +// +// Usage (server must be running on :3000): +// node src/eval/deep-investigation-run.mjs [out.json] +// MAX_CONTINUES=3 node src/eval/deep-investigation-run.mjs incidents.json /tmp/runs.json +// npx tsx src/eval/deep-eval.ts --results /tmp/runs.json +// +// incidents.json: [{ "stack": "