diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..4037fdd27 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "type": "pwa-node", + "request": "launch", + "name": "Debug trueforge-core orchestration (current file)", + "cwd": "${workspaceFolder}/packages/trueforge-core", + "runtimeExecutable": "pnpm", + "runtimeArgs": ["test:orchestration", "--", "--runInBand", "--testTimeout", "0", "${file}"], + "console": "integratedTerminal", + "autoAttachChildProcesses": true, + "skipFiles": ["/**", "**/node_modules/**"], + "sourceMaps": true, + "resolveSourceMapLocations": ["${workspaceFolder}/packages/trueforge-core/**", "!**/node_modules/**"] + } + ] +} diff --git a/package.json b/package.json index 238f8811b..19070f0ec 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,8 @@ "test:frontend": "pnpm --filter frontend test", "test:chart-version": "bash tests/scripts/resolve-chart-version.test.sh", "test:trueforge-core": "pnpm --filter @truefoundry/trueforge-core test", + "test:trueforge-core:orchestration": "pnpm --filter @truefoundry/trueforge-core test:orchestration", + "test:trueforge-core:orchestration:debug": "pnpm --filter @truefoundry/trueforge-core test:orchestration:debug", "test:trueforge": "pnpm --filter @truefoundry/trueforge test", "test:local-sandbox:contract": "pnpm --filter @truefoundry/trueforge test:local-sandbox:contract", "smoke:local-sandbox": "pnpm --filter @truefoundry/trueforge smoke:local-sandbox", diff --git a/packages/trueforge-core/jest.config.cjs b/packages/trueforge-core/jest.config.cjs index 935b21b7e..31df920f6 100644 --- a/packages/trueforge-core/jest.config.cjs +++ b/packages/trueforge-core/jest.config.cjs @@ -37,5 +37,6 @@ module.exports = { roots: ['/tests'], testMatch: ['**/tests/**/*.test.ts'], // Compile-time suites are enforced by `tsc --noEmit`, not the Jest runner. - testPathIgnorePatterns: ['\\.compile\\.test\\.ts$'], + // Orchestration tests live under tests/orchestration and are run via jest.orchestration.config.cjs. + testPathIgnorePatterns: ['\\.compile\\.test\\.ts$', '/tests/orchestration/'], }; diff --git a/packages/trueforge-core/jest.orchestration.config.cjs b/packages/trueforge-core/jest.orchestration.config.cjs new file mode 100644 index 000000000..d042e3bbf --- /dev/null +++ b/packages/trueforge-core/jest.orchestration.config.cjs @@ -0,0 +1,37 @@ +/** @type {import('jest').Config} */ +module.exports = { + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': [ + '@swc/jest', + { + jsc: { + parser: { syntax: 'typescript', decorators: true }, + target: 'es2022', + }, + module: { type: 'commonjs' }, + sourceMaps: 'inline', + }, + ], + '^.+\\.js$': [ + '@swc/jest', + { + jsc: { + parser: { syntax: 'ecmascript' }, + target: 'es2022', + }, + module: { type: 'commonjs' }, + sourceMaps: 'inline', + }, + ], + }, + transformIgnorePatterns: [], + moduleNameMapper: { + '^(\\.{1,2}/.*)\\.js$': '$1', + }, + setupFilesAfterEnv: ['/tests/setup.ts'], + testTimeout: 60_000, + maxWorkers: 1, + roots: ['/tests/orchestration'], + testMatch: ['/tests/orchestration/**/*.test.ts'], +}; diff --git a/packages/trueforge-core/package.json b/packages/trueforge-core/package.json index b1f2a1c1a..ac0ba17b8 100644 --- a/packages/trueforge-core/package.json +++ b/packages/trueforge-core/package.json @@ -95,7 +95,9 @@ "build:pkg": "node scripts/write-dist-package-json.mjs", "build:check": "node scripts/check-dist.mjs", "typecheck": "pnpm run build:gen && tsc --noEmit", - "test": "pnpm run build:gen && jest --config jest.config.cjs", + "test": "pnpm run build:gen && jest --config jest.config.cjs && jest --config jest.orchestration.config.cjs", + "test:orchestration": "pnpm run build:gen && jest --config jest.orchestration.config.cjs", + "test:orchestration:debug": "pnpm run build:gen && node --inspect-brk ./node_modules/jest/bin/jest.js --config jest.orchestration.config.cjs --runInBand --testTimeout 0", "pack:dry": "pnpm pack --dry-run" }, "dependencies": { diff --git a/packages/trueforge-core/tests/orchestration/README.md b/packages/trueforge-core/tests/orchestration/README.md new file mode 100644 index 000000000..86cb0d025 --- /dev/null +++ b/packages/trueforge-core/tests/orchestration/README.md @@ -0,0 +1,387 @@ +# Orchestration tests + +End-to-end tests for `AgentThreadOrchestrator` and `AgentThread` in `@truefoundry/trueforge-core`. + +These tests wire the real orchestration loop with **mocked LLMs** and **no database**. They exist to learn and verify how a turn flows through the harness before adding persistence (`SessionHandle`), HTTP, or real model providers. + +## What we are testing + +| Layer | In scope | Out of scope | +| --------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------- | +| `AgentThreadOrchestrator.send` | Route input to threads, validate, append context | Postgres / Redis store writes | +| `AgentThreadOrchestrator.execute` | Run leaf threads, merge streams, spawn sub-agents, return terminal result | `TurnHandle.stream` persistence | +| `AgentThread` | LLM loop, tool execution, context mutations | Real OpenAI / Vercel AI calls | +| Sub-agent lifecycle | `create_sub_agent` tool → child thread → result back to parent | Full `SessionHandle` resolver / spec wiring | + +**Goal:** prove the orchestrator correctly coordinates one root thread (Program 1) and a root + dynamic child thread (Program 2). + +## Why this design + +Production creates the orchestrator inside `SessionHandle.createTurn`: + +```text +resolve definitions → build AgentThread map → new AgentThreadOrchestrator → send → persist → execute (via TurnHandle) +``` + +These orchestration tests **skip the store and session layer** and talk to the orchestrator directly. That keeps the surface area small while still exercising the same `send` / `execute` contract production uses. + +```mermaid +flowchart LR + subgraph production["Production path"] + SH[SessionHandle] + Store[(ISessionStore)] + OrchP[AgentThreadOrchestrator] + SH --> Store + SH --> OrchP + end + + subgraph orch["Orchestration tests"] + Test[Jest test] + OrchE[AgentThreadOrchestrator] + MockLLM[Mock ILLM] + Test --> OrchE + OrchE --> MockLLM + end + + OrchP -. same class .- OrchE +``` + +## Files + +| File | Role | +| -------------------------------- | -------------------------------------------------------------------- | +| `orchestration.test.ts` | **Program 1** - single root thread, text-only reply, full assertions | +| `orchestrationWithTools.test.ts` | **Program 2** - root delegates to sub-agent via `create_sub_agent` | +| `helpers.ts` | Mock LLM streams and approval-gated tools | +| `jest.orchestration.config.cjs` | Jest config scoped to this folder | + +## Core components under test + +### `AgentThread` + +One conversation thread. Holds: + +- **`definition`** - `modelClient` (`ILLM`), optional `instruction`, `toolSets`, etc. +- **`context`** - LLM message history (user, assistant, tool messages) +- **`send(messages)`** - append user input, approvals, or tool responses to context (no LLM call) +- **`execute({ signal })`** - run the state machine: LLM → tools → pause or done + +### `AgentThreadOrchestrator` + +Owns a `Map` and coordinates a turn: + +- **`send(batch)`** - fan out messages to the right threads, validate, delegate to each thread's `send` +- **`execute({ signal })`** - run **leaf** threads in parallel (up to 5), merge event streams, handle sub-agent creation/completion +- **`createDynamicSubAgentThread`** - factory callback invoked when the root calls `create_sub_agent`; must return a new `AgentThread` (not called at construction time) + +### `CreateDynamicSubAgentThread` + +```ts +(input: { + parentDefinition: AgentDefinition; + request: AgentInfo; // { type: 'dynamic', name, input, model? } + threadId: string; // orchestrator already minted this + parent: AgentParent; // { thread_id, tool_call_id } + signal: AbortSignal; +}) => Promise; +``` + +Pass the **function reference** to the orchestrator. Do not call it yourself. + +## Turn lifecycle: `send` then `execute` + +These are separate steps on purpose (same as production: send before commit, then execute). + +```mermaid +sequenceDiagram + participant Test + participant Orch as AgentThreadOrchestrator + participant Thread as AgentThread + participant LLM as Mock ILLM + + Test->>Orch: send([USER_MESSAGE]) + Orch->>Thread: send(messages) + Thread-->>Orch: AGENT_CONTEXT_APPEND + Orch-->>Test: yield append events + + Note over Test,LLM: send does NOT call the model + + Test->>Orch: execute({ signal }) + loop until AGENT_DONE or pause + Orch->>Thread: execute({ signal }) + Thread->>LLM: create(streaming) + LLM-->>Thread: chunks / tool_calls + Thread-->>Orch: model.message.delta, model.message, ... + Orch-->>Test: yield execution events + end + Orch-->>Test: return AgentThreadExecutionResult +``` + +**Important:** `send` returns an async generator. You must consume it with `for await`; otherwise the user message never lands in context. + +**Important:** `execute` also returns an async generator. The **return value** (final assistant output, required pauses, errors) is only available after the last `next()` when `done === true`. + +## Mock LLM helpers (`helpers.ts`) + +| Helper | Behavior | +| ------------------------- | ---------------------------------------------------------------------------- | +| `textReplyStream(text)` | One streaming chunk + stop completion with fixed text | +| `makeTextLLM(text)` | `ILLM` that always replies with `text` (used for child threads) | +| `createSubAgentStream()` | First root call: stream a `create_sub_agent` tool call | +| `makeRootLLM(finalReply)` | First `create()` → sub-agent tool call; every later call → `finalReply` text | + +Root and child threads use **different** `ILLM` instances so each can follow its own scripted sequence. + +--- + +## Program 1: text-only happy path + +**File:** `orchestration.test.ts` + +### Setup + +| Piece | Value | +| ----------------------------- | -------------------------------------------- | +| Root thread id | `"main"` | +| LLM | `makeTextLLM("hello from the mocked model")` | +| Tool sets | none | +| `createDynamicSubAgentThread` | rejects if ever called | +| Tracing | `NOOP_AGENT_TRACING` | +| Logger | silent (`makeSilentLogger`) | + +### Data flow + +```mermaid +flowchart TD + A["send: USER_MESSAGE 'hello'"] --> B["context: user message appended"] + B --> C["execute: llm-call-required"] + C --> D["Mock LLM streams text reply"] + D --> E["context: assistant message appended"] + E --> F["AGENT_DONE on root"] + F --> G["execute returns output + empty required_actions"] +``` + +### Expected event types + +**After `send`:** + +```text +internal.agent.context.append +``` + +**During `execute` (order may include duplicates / internal appends):** + +```text +model.message.delta +model.message +internal.agent.done ← last yielded event +``` + +**Must NOT appear:** + +```text +thread.created +tool.response +``` + +### Passing expectations (assertions) + +- `step.value.output.content` === `"hello from the mocked model"` +- `step.value.required_actions` === `[]` +- `step.value.root_agent_error` is undefined +- Root snapshot context contains user `"hello"` and assistant reply + +--- + +## Program 2: sub-agent delegation + +**File:** `orchestrationWithTools.test.ts` + +### Setup + +| Piece | Root thread | Child thread | +| ---------------- | ----------------------------- | ----------------------------------------------- | +| Thread id | `"thread_1"` (fixed) | minted by orchestrator at runtime | +| LLM | `makeRootLLM("How are you?")` | `makeTextLLM("hello from the child")` | +| Tool sets | `[new DynamicSubAgents(...)]` | `undefined` (no nested sub-agents) | +| Instruction | test setup string | `undefined` (harness adds `SUB_AGENT_IDENTITY`) | +| Initial messages | none | `[{ role: 'user', content: request.input }]` | +| Parent link | none | `{ thread_id, tool_call_id }` from orchestrator | + +`createSubAgentThread` is a top-level `CreateDynamicSubAgentThread` implementation (mirrors a simplified `SessionHandle.makeCreateDynamicSubAgentThread`). + +### Scripted LLM behavior + +1. **Root call 1** - model returns `create_sub_agent` with `{ name: 'worker', input: '...' }` +2. **Child call 1** - model returns `"hello from the child"` +3. **Root call 2** - model returns `"How are you?"` + +### Thread tree over time + +```mermaid +flowchart TD + subgraph phase1["After root LLM call 1"] + R1["thread_1 (root)
open create_sub_agent tool call"] + end + + subgraph phase2["After sub-agent created"] + R2["thread_1 (root)
waiting on tool call"] + C["child thread (leaf)
runs execute"] + R2 --- C + end + + subgraph phase3["After child AGENT_DONE"] + R3["thread_1 (root, leaf again)
tool result appended"] + end + + phase1 --> phase2 --> phase3 +``` + +Only **leaf** threads run. While the child exists, the root is paused (not a leaf). When the child finishes, the orchestrator: + +1. Yields `tool.response` on the parent +2. `send()`s the child's result into the parent as a tool message +3. Removes the child from the thread map +4. Resumes the root for LLM call 2 + +### Data flow + +```mermaid +sequenceDiagram + participant Test + participant Orch as Orchestrator + participant Root as thread_1 + participant Child as sub-agent + participant RootLLM as makeRootLLM + participant ChildLLM as makeTextLLM + + Test->>Orch: send(USER_MESSAGE) + Test->>Orch: execute() + + Root->>RootLLM: create() #1 + RootLLM-->>Root: create_sub_agent tool call + Root-->>Orch: internal.agent.create_subagent + Orch->>Orch: createSubAgentThread(...) + Orch-->>Test: thread.created + + Child->>ChildLLM: create() + ChildLLM-->>Child: "hello from the child" + Child-->>Orch: model.message, AGENT_DONE (child) + + Orch-->>Test: tool.response (parent) + Orch->>Root: send(tool message with child result) + + Root->>RootLLM: create() #2 + RootLLM-->>Root: "How are you?" + Root-->>Orch: model.message, AGENT_DONE (root) + Orch-->>Test: return { output: "How are you?", ... } +``` + +### Expected event types (from logging) + +Typical `execute` event sequence: + +```text +model.message / model.message.delta ← root tool call +internal.agent.context.append ← (internal, may repeat) +thread.created ← child registered +model.message / model.message.delta ← child reply +tool.response ← child result routed to parent +internal.agent.done ← child finished (thread_id = child) +model.message / model.message.delta ← root final reply +internal.agent.done ← root finished (last event) +``` + +`internal.agent.create_subagent` is handled inside the orchestrator and is **not** yielded to the test consumer. + +### Expected final state + +**`execute` return value:** + +| Field | Expected | +| ------------------ | --------------------------------------------------- | +| `output.content` | `"How are you?"` (root final reply, not child text) | +| `required_actions` | `[]` | +| `root_agent_error` | undefined | + +**Root thread context (after send + execute):** + +```text +1. user: "hello" +2. assistant: tool_call create_sub_agent (id: call-sub) +3. tool: "hello from the child" +4. assistant: "How are you?" +``` + +--- + +## Running tests + +From `packages/trueforge-core`: + +```bash +pnpm test:orchestration +``` + +Single file: + +```bash +pnpm test:orchestration -- orchestration.test.ts +pnpm test:orchestration -- orchestrationWithTools.test.ts +``` + +From repo root: + +```bash +pnpm test:trueforge-core:orchestration +``` + +Orchestration tests use `jest.orchestration.config.cjs` (`maxWorkers: 1`, 60s timeout). Unit tests under `tests/` (excluding `tests/orchestration/`) run separately via `jest.config.cjs`. + +Threads and the orchestrator still take a Winston logger (required by the runtime). These tests use `makeSilentLogger()` from `tests/core/harnessMocks.ts`, so the suite does not print turn flow. + +## Relationship to production + +| Orchestration test | Production equivalent | +| -------------------------------------- | ------------------------------------------------------ | +| `new AgentThread({ definition, ... })` | `SessionHandle.buildThreads` + resolver | +| `createSubAgentThread` callback | `SessionHandle.makeCreateDynamicSubAgentThread` | +| `orchestrator.send` + `execute` | `SessionHandle.createTurn` + `TurnHandle.stream` | +| In-memory `thread.toSnapshot()` | `ISessionStore.createTurn` / persisted context appends | +| `NOOP_AGENT_TRACING` | `resolver.createTracing()` | + +Production adds: store persistence, turn records, event folding for SSE, sandbox resolution, full builtin capabilities from `AgentSpec`, and MCP servers beyond `DynamicSubAgents`. + +## Planned coverage (not yet implemented) + +| Program | Scenario | +| ------- | -------------------------------------------------------------------------------------------------- | +| **3** | Pause on `tool.approval.required` or `tool.response.required`, then resume with `send` + `execute` | +| **4** | Reject user message while sub-agent is live (`InvalidAgentSendInputError`) | +| **5** | MCP auth required (`internal.mcp.auth_required` merge across parallel sub-agents) | + +## Quick reference: orchestrator inputs + +```ts +new AgentThreadOrchestrator({ + agentThreads: new Map([[rootThreadId, rootThread]]), + createDynamicSubAgentThread, // function reference, not a call + tracing: NOOP_AGENT_TRACING, + logger, +}); +``` + +Every turn: + +```ts +for await (const _ of orchestrator.send(input)) { + /* collect appends */ +} +const it = orchestrator.execute({ signal }); +let step = await it.next(); +while (!step.done) { + // step.value is a streamed execution event + step = await it.next(); +} +// step.value is AgentThreadExecutionResult +``` diff --git a/packages/trueforge-core/tests/orchestration/helpers/helpers.ts b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts new file mode 100644 index 000000000..1f1ad4b9f --- /dev/null +++ b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts @@ -0,0 +1,234 @@ +import type { ILLM } from '../../../src/core/llm/ILLM'; +import type { ExtendedChatCompletionChunk, RawAssistantMessageWithUsage } from '../../../src/core/llm/LLMTypes'; +import { getEmptyUsage } from '../../../src/core/llm/LLMTypes'; +import type { IToolSet, ToolSource } from '../../../src/core/mcp/IMCPServer'; +import { toolResultResponse } from '../../../src/core/mcp/IMCPServer'; +import { ToolSet } from '../../../src/core/mcp/ToolSet'; +import type { + AgentThreadExecutionEvent, + AgentThreadExecutionResult, + AgentThreadSendBatch, +} from '../../../src/core/runtime/AgentThread.types'; +import type { AgentThreadOrchestrator } from '../../../src/core/runtime/AgentThreadOrchestrator'; + +export const WRITE_NOTE_TOOL_NAME = 'write_note'; +export const WRITE_NOTE_CALL_ID = 'call-write'; +export const WRITE_NOTE_ARGUMENTS = JSON.stringify({ text: 'hello' }); +export const WRITE_NOTE_RESULT = 'note written'; + +/** One streamed chunk plus a stop completion. Used when the test needs a text reply and no tool calls. */ +// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O +export async function* textReplyStream( + text: string, +): AsyncGenerator { + yield { + id: 'chunk-text', + object: 'chat.completion.chunk', + created: 0, + model: 'test-model', + choices: [{ index: 0, delta: { role: 'assistant', content: text }, finish_reason: 'stop' }], + }; + return { + output: { role: 'assistant', content: text }, + usage: getEmptyUsage(), + finish_reason: 'stop', + }; +} + +// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O +export async function* createSubAgentStream() { + yield { + id: 'chunk-tool', + object: 'chat.completion.chunk', + created: 0, + model: 'test-model', + choices: [ + { + index: 0, + delta: { + role: 'assistant', + tool_calls: [ + { + index: 0, + id: 'call-sub', + type: 'function', + function: { + name: 'create_sub_agent', + arguments: JSON.stringify({ name: 'worker', input: 'do the delegated task' }), + }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + }; + + return { + output: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call-sub', + type: 'function', + function: { + name: 'create_sub_agent', + arguments: JSON.stringify({ name: 'worker', input: 'do the delegated task' }), + }, + }, + ], + }, + usage: getEmptyUsage(), + finish_reason: 'tool_calls', + }; +} + +/** ILLM that always streams `text` and then stops. */ +export function makeTextLLM(text: string): ILLM { + return { + create: jest.fn().mockImplementation(() => textReplyStream(text)), + createNonStream: jest.fn().mockImplementation(() => textReplyStream(text)), + }; +} + +export function makeRootLLM(finalReply: string): ILLM { + return { + create: jest + .fn() + .mockImplementationOnce(() => createSubAgentStream()) + .mockImplementation(() => textReplyStream(finalReply)), + createNonStream: jest.fn(), + }; +} + +// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O +export async function* writeNoteToolCallStream() { + yield { + id: 'chunk-write-note', + object: 'chat.completion.chunk', + created: 0, + model: 'test-model', + choices: [ + { + index: 0, + delta: { + role: 'assistant', + tool_calls: [ + { + index: 0, + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { + name: WRITE_NOTE_TOOL_NAME, + arguments: WRITE_NOTE_ARGUMENTS, + }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + }; + + return { + output: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { + name: WRITE_NOTE_TOOL_NAME, + arguments: WRITE_NOTE_ARGUMENTS, + }, + }, + ], + }, + usage: getEmptyUsage(), + finish_reason: 'tool_calls', + }; +} + +/** First create() requests write_note; later calls stream `finalReply`. */ +export function makeApprovalThenTextLLM(finalReply: string): ILLM { + return { + create: jest + .fn() + .mockImplementationOnce(() => writeNoteToolCallStream()) + .mockImplementation(() => textReplyStream(finalReply)), + createNonStream: jest.fn(), + }; +} + +function makeWriteNoteSource(): ToolSource { + return { + name: 'notes', + id: 'notes', + listTools: () => + Promise.resolve({ + result: { + tools: [ + { + name: WRITE_NOTE_TOOL_NAME, + description: 'Write a note', + inputSchema: { + type: 'object', + properties: { text: { type: 'string' } }, + }, + preload: true, + }, + ], + }, + wasInitialized: undefined, + }), + callTool: () => Promise.resolve(toolResultResponse({ text: WRITE_NOTE_RESULT })), + toolCallInfo: () => + Promise.resolve({ + type: 'mcp', + mcp_server_id: 'notes', + mcp_server_name: 'notes', + original_tool_name: WRITE_NOTE_TOOL_NAME, + }), + }; +} + +/** User tool set that pauses until write_note is approved. */ +export function makeApprovalGatedWriteNoteToolSet(): IToolSet { + return new ToolSet({ + source: makeWriteNoteSource(), + selectors: { + enableTools: ['@all'], + disableTools: [], + preloadTools: [], + requireApprovalForTools: [WRITE_NOTE_TOOL_NAME], + }, + preload: true, + }); +} + +/** Consume send() then execute(); return raw events and the generator result. */ +export async function runTurn(input: { + orchestrator: AgentThreadOrchestrator; + sendBatch: AgentThreadSendBatch; + signal?: AbortSignal | undefined; +}): Promise<{ events: AgentThreadExecutionEvent[]; result: AgentThreadExecutionResult }> { + for await (const _event of input.orchestrator.send(input.sendBatch)) { + void _event; + } + const events: AgentThreadExecutionEvent[] = []; + const iterator = input.orchestrator.execute({ + signal: input.signal ?? new AbortController().signal, + }); + let step = await iterator.next(); + while (!step.done) { + events.push(step.value); + step = await iterator.next(); + } + return { events, result: step.value }; +} + +export function llmCreateInputs(llm: ILLM): unknown[] { + return jest.mocked(llm).create.mock.calls.map(call => call[0]); +} diff --git a/packages/trueforge-core/tests/orchestration/orchestration.test.ts b/packages/trueforge-core/tests/orchestration/orchestration.test.ts new file mode 100644 index 000000000..af3aa5591 --- /dev/null +++ b/packages/trueforge-core/tests/orchestration/orchestration.test.ts @@ -0,0 +1,72 @@ +import { EventType } from '../../src/core/events/schema'; +import { AgentThread } from '../../src/core/runtime/AgentThread'; +import { InternalEventType } from '../../src/core/runtime/AgentThread.types'; +import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; +import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; +import { makeSilentLogger } from '../core/harnessMocks'; +import { llmCreateInputs, makeTextLLM, runTurn } from './helpers/helpers'; + +const THREAD_ID = 'main'; +const REPLY = 'hello from the mocked model'; +const INSTRUCTION = 'You are running in a test setup.'; + +/** One root thread, no tools: user message in, text reply out. */ +const EXPECTED_EVENTS = [ + { type: EventType.MODEL_MESSAGE, thread_id: THREAD_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: THREAD_ID, content: REPLY }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: THREAD_ID }, + { type: InternalEventType.AGENT_DONE, thread_id: THREAD_ID, status: 'done' }, +]; + +const OUTPUT = { + output: { thread_id: THREAD_ID, content: REPLY }, + required_actions: [], +}; + +const EXPECTED_LLM_INPUT = [ + { + stream: true, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + ], + }, +]; + +/** Root thread with a one-shot text LLM and no tool sets. */ +function makeTextLlmThread(): AgentThread { + return new AgentThread({ + threadId: THREAD_ID, + title: 'orchestration', + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + definition: { + modelClient: makeTextLLM(REPLY), + instruction: INSTRUCTION, + }, + }); +} + +describe('orchestration: mocked LLM and no tools', () => { + it('sends a user message and finishes the thread with a text reply', async () => { + const thread = makeTextLlmThread(); + // Orchestrator owns the thread map and fans send/execute across live threads. + // This case has only the root thread, so sub-agent creation must never run. + const orchestrator = new AgentThreadOrchestrator({ + agentThreads: new Map([[thread.threadId, thread]]), + createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in no-tool test')), + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }); + + const { events, result } = await runTurn({ + orchestrator, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + }); + + expect(events).toMatchObject(EXPECTED_EVENTS); + expect(result).toMatchObject(OUTPUT); + expect(result.root_agent_error).toBeUndefined(); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject(EXPECTED_LLM_INPUT); + }); +}); diff --git a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts new file mode 100644 index 000000000..de8d6ad9d --- /dev/null +++ b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts @@ -0,0 +1,168 @@ +import type { AgentDefinition } from '../../src/core'; +import { EventType } from '../../src/core/events/schema'; +import { AgentThread } from '../../src/core/runtime/AgentThread'; +import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; +import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator'; +import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; +import { makeSilentLogger } from '../core/harnessMocks'; +import { + llmCreateInputs, + makeApprovalGatedWriteNoteToolSet, + makeApprovalThenTextLLM, + runTurn, + WRITE_NOTE_ARGUMENTS, + WRITE_NOTE_CALL_ID, + WRITE_NOTE_RESULT, + WRITE_NOTE_TOOL_NAME, +} from './helpers/helpers'; + +const ROOT_ID = 'thread_root'; +const ROOT_FINAL = 'note saved'; +const INSTRUCTION = 'You are running in a test setup.'; + +const WRITE_NOTE_TOOLS = [ + { function: { name: 'call_tool' } }, + { function: { name: 'get_tool_info' } }, + { function: { name: 'get_tool_output_schema' } }, + { function: { name: 'list_tools' } }, + { function: { name: WRITE_NOTE_TOOL_NAME } }, +]; + +/** Pause on write_note approval, then resume after allow and finish. */ +const EXPECTED_PAUSE_EVENTS = [ + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { + type: EventType.TOOL_APPROVAL_REQUIRED, + thread_id: ROOT_ID, + tool_calls: [{ id: WRITE_NOTE_CALL_ID }], + }, +]; + +const PAUSE_OUTPUT = { + output: null, + required_actions: [ + { + type: EventType.TOOL_APPROVAL_REQUIRED, + thread_id: ROOT_ID, + tool_calls: [{ id: WRITE_NOTE_CALL_ID }], + }, + ], +}; + +const EXPECTED_PAUSE_LLM_INPUT = [ + { + stream: true, + tools: WRITE_NOTE_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + ], + }, +]; + +const EXPECTED_RESUME_EVENTS = [ + { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: WRITE_NOTE_CALL_ID }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, +]; + +const RESUME_OUTPUT = { + output: { thread_id: ROOT_ID, content: ROOT_FINAL }, + required_actions: [], +}; + +const EXPECTED_RESUME_LLM_INPUT = { + stream: true, + tools: WRITE_NOTE_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: WRITE_NOTE_CALL_ID, + type: 'function', + function: { name: WRITE_NOTE_TOOL_NAME, arguments: WRITE_NOTE_ARGUMENTS }, + }, + ], + }, + { role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: WRITE_NOTE_RESULT }, + ], +}; + +describe('orchestration: pause then resume on tool approval', () => { + it('pauses for write_note approval, then finishes after allow', async () => { + const thread = makeApprovalThread(); + const orchestrator = new AgentThreadOrchestrator({ + agentThreads: new Map([[thread.threadId, thread]]), + createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in approval test')), + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }); + + const paused = await runTurn({ + orchestrator, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + }); + expect(paused.events).toMatchObject(EXPECTED_PAUSE_EVENTS); + expect(paused.result).toMatchObject(PAUSE_OUTPUT); + expect(paused.result.root_agent_error).toBeUndefined(); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject(EXPECTED_PAUSE_LLM_INPUT); + + const resumed = await runTurn({ + orchestrator, + sendBatch: [ + { + type: EventType.USER_TOOL_APPROVAL, + thread_id: ROOT_ID, + tool_call_id: WRITE_NOTE_CALL_ID, + approval: { status: 'allow' }, + }, + ], + }); + expect(resumed.events).toMatchObject(EXPECTED_RESUME_EVENTS); + expect(resumed.result).toMatchObject(RESUME_OUTPUT); + expect(resumed.result.root_agent_error).toBeUndefined(); + expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject([ + ...EXPECTED_PAUSE_LLM_INPUT, + EXPECTED_RESUME_LLM_INPUT, + ]); + }); +}); + +function makeApprovalThread(): AgentThread { + const agentDefinition: AgentDefinition = { + modelClient: makeApprovalThenTextLLM(ROOT_FINAL), + instruction: INSTRUCTION, + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: [makeApprovalGatedWriteNoteToolSet()], + }; + + const agentThreadInput: AgentThreadConstructorInput = { + definition: agentDefinition, + threadId: ROOT_ID, + title: 'orchestration-approval', + parent: undefined, + agentInfo: undefined, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }; + + return new AgentThread(agentThreadInput); +} diff --git a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts new file mode 100644 index 000000000..e3ce2c38e --- /dev/null +++ b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts @@ -0,0 +1,202 @@ +import type { AgentDefinition, CreateDynamicSubAgentThread } from '../../src/core'; +import { DynamicSubAgents } from '../../src/core/capabilities/builtins/DynamicSubAgents'; +import { EventType } from '../../src/core/events/schema'; +import type { ILLM } from '../../src/core/llm/ILLM'; +import { AgentThread } from '../../src/core/runtime/AgentThread'; +import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types'; +import { + AgentThreadOrchestrator, + type AgentThreadOrchestratorInput, +} from '../../src/core/runtime/AgentThreadOrchestrator'; +import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing'; +import { makeSilentLogger } from '../core/harnessMocks'; +import { createSubAgentStream, llmCreateInputs, runTurn, textReplyStream } from './helpers/helpers'; + +const ROOT_ID = 'thread_root'; +const TOOL_CALL_ID = 'call-sub'; +const CHILD_REPLY = 'hello from the child'; +const ROOT_FINAL = 'How are you?'; +const INSTRUCTION = 'You are running in a test setup.'; +const CHILD_TASK = 'do the delegated task'; + +const ROOT_TOOLS = [ + { function: { name: 'call_tool' } }, + { function: { name: 'get_tool_info' } }, + { function: { name: 'get_tool_output_schema' } }, + { function: { name: 'list_tools' } }, + { function: { name: 'create_sub_agent' } }, +]; + +/** Root delegates via create_sub_agent; child result returns to parent; root finishes. */ +const EXPECTED_EVENTS = [ + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { + type: EventType.THREAD_CREATED, + title: 'worker', + parent: { thread_id: ROOT_ID, tool_call_id: TOOL_CALL_ID }, + }, + { type: EventType.MODEL_MESSAGE, thread_id: expect.any(String) }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: expect.any(String), content: CHILD_REPLY }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: expect.any(String) }, + { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: TOOL_CALL_ID }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { type: InternalEventType.AGENT_DONE, thread_id: expect.any(String), status: 'done' }, + { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID }, + { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL }, + { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID }, + { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' }, +]; + +const OUTPUT = { + output: { thread_id: ROOT_ID, content: ROOT_FINAL }, + required_actions: [], +}; + +const EXPECTED_ROOT_LLM_INPUT = [ + { + stream: true, + tools: ROOT_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + ], + }, + { + stream: true, + tools: ROOT_TOOLS, + messages: [ + { role: 'system', content: expect.stringContaining(INSTRUCTION) }, + { role: 'user', content: 'hello' }, + { + role: 'assistant', + content: null, + tool_calls: [ + { + id: TOOL_CALL_ID, + type: 'function', + function: { + name: 'create_sub_agent', + arguments: JSON.stringify({ name: 'worker', input: CHILD_TASK }), + }, + }, + ], + }, + { role: 'tool', tool_call_id: TOOL_CALL_ID, content: CHILD_REPLY }, + ], + }, +]; + +const EXPECTED_CHILD_LLM_INPUT = [ + { + stream: true, + messages: [ + { role: 'system', content: expect.stringContaining('sub-agent') }, + { role: 'user', content: CHILD_TASK }, + ], + }, +]; + +describe('orchestration: dynamic sub-agent', () => { + it('delegates via create_sub_agent, routes child result to parent, then finishes', async () => { + let agentThreadInput: AgentThreadConstructorInput = { + // AgentDefinition + definition: { + // This is an instance if ILLM + modelClient: { + create: jest + .fn() + .mockImplementationOnce(() => createSubAgentStream()) + .mockImplementation(() => textReplyStream(ROOT_FINAL)), + createNonStream: jest.fn(), + }, + instruction: INSTRUCTION, + // Undefined + messages: undefined, + modelParams: undefined, + responseFormat: undefined, + iterationLimit: undefined, + toolSets: [new DynamicSubAgents({ tracing: NOOP_AGENT_TRACING })], + }, + threadId: ROOT_ID, + title: 'orchestration-with-tools', + // Undefined + parent: undefined, + agentInfo: undefined, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + // Default + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }; + + let thread_1 = new AgentThread(agentThreadInput); + let childLLM: ILLM | undefined; + + const createSubAgentThread: CreateDynamicSubAgentThread = async ({ + parentDefinition, + request, + threadId, + parent, + }) => { + childLLM = { + create: jest.fn().mockImplementation(() => textReplyStream(CHILD_REPLY)), + createNonStream: jest.fn().mockImplementation(() => textReplyStream(CHILD_REPLY)), + }; + + const agentDefinition: AgentDefinition = { + modelClient: childLLM, + instruction: undefined, + messages: [{ role: 'user', content: request.input }], + modelParams: parentDefinition.modelParams, + responseFormat: undefined, + iterationLimit: parentDefinition.iterationLimit, + toolSets: undefined, + }; + return new AgentThread({ + definition: agentDefinition, + threadId, + title: request.name, + parent, + agentInfo: request, + context: undefined, + currentContextUsage: undefined, + preComputedCompletion: undefined, + sandbox: undefined, + capabilities: undefined, + capabilityState: undefined, + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }); + }; + + let orchestratorInput: AgentThreadOrchestratorInput = { + agentThreads: new Map([[thread_1.threadId, thread_1]]), + createDynamicSubAgentThread: createSubAgentThread, + tracing: NOOP_AGENT_TRACING, + logger: makeSilentLogger(), + }; + + const orchestrator = new AgentThreadOrchestrator(orchestratorInput); + + const { events, result } = await runTurn({ + orchestrator, + sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }], + }); + + expect(events).toMatchObject(EXPECTED_EVENTS); + expect(result).toMatchObject(OUTPUT); + expect(result.root_agent_error).toBeUndefined(); + expect(llmCreateInputs(thread_1.definition.modelClient)).toMatchObject(EXPECTED_ROOT_LLM_INPUT); + if (childLLM === undefined) { + throw new Error('expected child LLM to be created'); + } + expect(llmCreateInputs(childLLM)).toMatchObject(EXPECTED_CHILD_LLM_INPUT); + }); +}); diff --git a/packages/trueforge-core/tsconfig.json b/packages/trueforge-core/tsconfig.json index 85ca45efe..97562af5e 100644 --- a/packages/trueforge-core/tsconfig.json +++ b/packages/trueforge-core/tsconfig.json @@ -12,6 +12,6 @@ "openai/resources/chat": ["./node_modules/openai/resources/chat/index.d.ts"] } }, - "include": ["src/**/*", "tests/**/*", "tsup.config.ts", "jest.config.cjs"], + "include": ["src/**/*", "tests/**/*", "tsup.config.ts", "jest.config.cjs", "jest.orchestration.config.cjs"], "exclude": ["node_modules", "dist"] }