diff --git a/.changeset/e2e-contract-cli.md b/.changeset/e2e-contract-cli.md new file mode 100644 index 000000000..f0e3b922c --- /dev/null +++ b/.changeset/e2e-contract-cli.md @@ -0,0 +1,5 @@ +--- +'@truefoundry/trueforge': patch +--- + +Add a local end-to-end test command (`pnpm test:e2e`) that starts its own stack, runs session checks, and tears everything down. diff --git a/AGENTS.md b/AGENTS.md index 313e093e1..027d96d7c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ - CI package path filters, matrix package ids, and root scripts `test:*` in `.github/workflows/ci.yml` and root `package.json` MUST stay synchronized when a workspace package is added, renamed, or moved; the `store` filter sync rule lives in `packages/trueforge-core/src/agent-session/store/AGENTS.md`. - Release wiring MUST keep dist-free host development, `pnpm smoke`, and packed CJS/ESM consumers of `@truefoundry/trueforge-core` working without changes. - PRs that change published-package code (`packages/trueforge-core`, `packages/trueforge`, `packages/trueforge-ui`, `packages/trueforge-sdk`) or `packages/frontend` (ships inside `@truefoundry/trueforge`) MUST include a new `.changeset/*.md` file (`pnpm changeset`). Docs, CI/workflows, charts, and docker-compose changes do not. SDK regeneration already adds `@truefoundry/trueforge-sdk` via `pnpm changeset:sdk-regen`. -- Shared Postgres/Redis settings in `docker-compose.yml` and `docker-compose.dev.yml` (image versions, health checks, `env_file`) MUST stay synchronized; intentional differences (app services, data paths, project `name`, host ports, in-network `POSTGRES_HOST` / `REDIS_URL`) MUST stay explicit. `packages/trueforge/.env` is the host-dev + secrets source; `docker-compose.yml` may read it but MUST override container connectivity so host-dev localhost values are not used inside the smoke-test stack. +- Shared Postgres/Redis settings in `docker-compose.yml`, `docker-compose.dev.yml`, and `docker-compose.e2e.yml` (image versions, health checks) MUST stay synchronized; intentional differences (app services, data paths, project `name`, host ports, `env_file`, in-network `POSTGRES_HOST` / `REDIS_URL`) MUST stay explicit. Smoke and E2E server services MUST set `HOST: 0.0.0.0` so published ports and in-container `/healthz` probes reach the process. `packages/trueforge/.env` is the host-dev + secrets source; `packages/trueforge/e2e/.env` is the E2E stack + CLI source. Compose files MUST override container connectivity so host-dev localhost values are not used inside smoke or E2E stacks. - Changes to types or schemas MUST keep `packages/trueforge-core`, `packages/frontend`, `packages/trueforge`, and `patches` synchronized; they MUST NOT update only one affected layer. - TypeScript code MUST NOT use assertion escapes such as `as T`, `as unknown as T`, non-null `!`, or `as never` to silence type errors; implementations MUST use sound contracts, guards, or corrected types. - When catching an error and throwing another, the new error MUST set `{ cause: caught }` so the original failure is preserved for logs and debugging. diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml new file mode 100644 index 000000000..e8ce316b6 --- /dev/null +++ b/docker-compose.e2e.yml @@ -0,0 +1,90 @@ +# Full-stack E2E contract. Same topology as docker-compose.yml (smoke); isolated +# project name, host ports, and env_file so it does not collide with host-dev +# (8790/5432/6379) or smoke (8791/5433/6380). Postgres has no host bind mount so +# `scripts/e2e.sh` `down -v` discards the DB. Keep image versions and healthchecks +# in sync with docker-compose.yml / docker-compose.dev.yml. +name: trueforge-e2e + +services: + postgres: + image: postgres:17 + env_file: + - path: packages/trueforge/e2e/.env + required: true + # Host 5434 avoids conflict with docker-compose.dev.yml on 5432 and smoke on 5433. + ports: + - '5434:5432' + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"'] + interval: 5s + timeout: 5s + retries: 10 + restart: unless-stopped + + # Shared by all server replicas: carries executor peering (cross-replica + # turn cancel via request-reply). + redis: + image: redis:7-alpine + # Host 6381 avoids conflict with docker-compose.dev.yml on 6379 and smoke on 6380. + ports: + - '6381:6379' + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 5s + timeout: 3s + retries: 5 + restart: unless-stopped + + # Serves both the API and the UI on one port. + server: + build: + context: . + # From-source image; root Dockerfile is the npm-install prod recipe. + dockerfile: Dockerfile.dev + image: truefoundry-server:latest + ports: + # Host 8792 avoids conflict with host `pnpm dev` on 8790 and smoke on 8791. + - '8792:8790' + environment: + # Wins over env_file / image defaults so a host .env cannot flip this. + NODE_ENV: production + # Inside the container; host mapping above is 8792. + PORT: 8790 + # Wins over env_file / Node default `localhost` (::1 vs healthcheck 127.0.0.1). + HOST: 0.0.0.0 + # Host-facing origin for MCP OAuth callbacks (mapped port, not container PORT). + PUBLIC_BASE_URL: http://localhost:8792 + # Compose network hosts — do not use packages/trueforge/e2e/.env localhost values. + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + # Compose runs one replica but keeps peering on so the peered cancel path + # stays exercised. STANDALONE=false ⇒ Postgres + Redis. + STANDALONE: 'false' + REDIS_URL: redis://redis:6379 + # Postgres credentials (POSTGRES_USER, …) and later E2E secrets. + # `environment` above overrides any colliding keys from this file. + env_file: + - path: packages/trueforge/e2e/.env + required: true + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + # Probes GET /healthz (plain text OK!); node is already in the image. + healthcheck: + test: + [ + 'CMD', + 'node', + '-e', + "fetch('http://127.0.0.1:8790/healthz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))", + ] + interval: 5s + timeout: 5s + retries: 10 + start_period: 15s + # Above the app's default GRACEFUL_TIMEOUT_SECONDS (30) so SIGKILL does not + # cut off turn drain. + stop_grace_period: 35s + restart: unless-stopped diff --git a/package.json b/package.json index 8440ec19f..c4324c2e9 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "test:frontend": "pnpm --filter frontend test", "test:trueforge-core": "pnpm --filter @truefoundry/trueforge-core test", "test:trueforge": "pnpm --filter @truefoundry/trueforge test", + "test:e2e": "bash scripts/e2e.sh", "test:local-sandbox:contract": "pnpm --filter @truefoundry/trueforge test:local-sandbox:contract", "smoke:local-sandbox": "pnpm --filter @truefoundry/trueforge smoke:local-sandbox", "smoke:local-sandbox:lima": "pnpm --filter @truefoundry/trueforge smoke:local-sandbox:lima", diff --git a/packages/trueforge/e2e/.env.example b/packages/trueforge/e2e/.env.example new file mode 100644 index 000000000..6b6f0d47f --- /dev/null +++ b/packages/trueforge/e2e/.env.example @@ -0,0 +1,17 @@ +# Copy to `.env` (git-ignored). Used by docker-compose.e2e.yml (postgres + server) +# and by the E2E CLI. Do not point this file at host-dev ports. +# +# Compose maps the API to http://127.0.0.1:8792 (not 8790 / 8791). + +# --- Postgres (required for the e2e compose postgres service) --- +POSTGRES_USER=trueforge +POSTGRES_PASSWORD=trueforge +POSTGRES_DB=trueforge + +# --- CLI (model + Daytona) --- +# Model FQN `provider/model`, e.g. openai/gpt-4.1-mini +MODEL=openai/gpt-4.1-mini +MODEL_API_KEY= +DAYTONA_API_KEY= +# Per-turn wait ceiling in milliseconds (default 180000 when unset). +TEST_TIMEOUT_MS=180000 diff --git a/packages/trueforge/e2e/helpers.ts b/packages/trueforge/e2e/helpers.ts new file mode 100644 index 000000000..e820ea763 --- /dev/null +++ b/packages/trueforge/e2e/helpers.ts @@ -0,0 +1,731 @@ +/** + * Shared E2E helpers: SDK client, resource upserts, turn collection, + * stream-vs-listTurnEvents reconciliation, and the sequential runner. + */ +import { TrueForge, TrueForgeApi, TrueForgeError, isEventDelta, mergeEventDelta } from '@truefoundry/trueforge-sdk'; +import { randomUUID } from 'node:crypto'; + +/** Host URL of docker-compose.e2e.yml (API mapped to 8792) */ +export const E2E_BASE_URL = 'http://127.0.0.1:8792'; + +export const MCP_DEEPWIKI = 'deepwiki'; +export const MCP_LINEAR = 'linear'; +export const NAMED_AGENT = 'e2e-memory'; + +const WELL_KNOWN_PROVIDER_TYPES = Object.values(TrueForgeApi.CatalogWellKnownModelProviderType); + +export function requireEnv(key: string): string { + const value = process.env[key]; + if (value === undefined || value.trim() === '') { + throw new Error(`Missing required environment variable: ${key} (set it in packages/trueforge/e2e/.env)`); + } + return value.trim(); +} + +export function optionalEnv(key: string): string | undefined { + const value = process.env[key]; + return value !== undefined && value.trim() !== '' ? value.trim() : undefined; +} + +export function turnTimeoutMs(): number { + const raw = optionalEnv('TEST_TIMEOUT_MS'); + const parsed = raw !== undefined ? Number(raw) : Number.NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : 180_000; +} + +export function createClient(): TrueForge { + return new TrueForge({ + baseUrl: E2E_BASE_URL, + timeoutInSeconds: Math.ceil(turnTimeoutMs() / 1000), + }); +} + +export function baseAgentSpec(overrides: Partial = {}): TrueForgeApi.AgentSpec { + return { + model: { name: requireEnv('MODEL'), params: { temperature: 0 } }, + ...overrides, + }; +} + +export async function createInlineSession({ + client, + spec, +}: { + client: TrueForge; + spec: TrueForgeApi.AgentSpec; +}): Promise { + const response = await client.sessions.create({ agent: { spec } }); + return response.data; +} + +export async function createNamedAgentSession({ + client, + name, +}: { + client: TrueForge; + name: string; +}): Promise { + const response = await client.sessions.create({ agent: { name } }); + return response.data; +} + +export function userMessage(content: TrueForgeApi.UserMessageContent): TrueForgeApi.UserMessage { + return { type: 'user.message', content }; +} + +export function textFileContent({ + name, + text, + mime = 'text/plain', +}: { + name: string; + text: string; + mime?: string; +}): TrueForgeApi.FileContent { + const base64 = Buffer.from(text, 'utf8').toString('base64'); + return { type: 'file', name, data: `data:${mime};base64,${base64}` }; +} + +export function approveToolCall({ + threadId, + toolCallId, +}: { + threadId: string; + toolCallId: string; +}): TrueForgeApi.UserToolApprovalEvent { + return { type: 'user.tool_approval', threadId, toolCallId, approval: { status: 'allow' } }; +} + +export function denyToolCall({ + threadId, + toolCallId, + reason, +}: { + threadId: string; + toolCallId: string; + reason?: string; +}): TrueForgeApi.UserToolApprovalEvent { + return { + type: 'user.tool_approval', + threadId, + toolCallId, + approval: reason !== undefined ? { status: 'deny', reason } : { status: 'deny' }, + }; +} + +/** Unguessable token so a pass proves data flowed through the feature, not a model guess. */ +export function makeNonce(prefix = 'NONCE'): string { + return `${prefix}_${randomUUID().replace(/-/g, '')}`; +} + +export function httpStatusCode(error: unknown): number | undefined { + return error instanceof TrueForgeError ? error.statusCode : undefined; +} + +export function errorMessage(error: unknown): string { + if (!(error instanceof Error)) { + return String(error); + } + const base = error.stack ?? error.message; + return error.cause != null ? `${base}\ncaused by: ${errorMessage(error.cause)}` : base; +} + +export type TurnEvent = TrueForgeApi.TurnStreamingEvent; +export type PersistedTurnEvent = TrueForgeApi.SessionEvent; +export type TurnDoneEvent = Extract; +export type TurnCreatedEvent = Extract; +export type ActionRequiredEvent = TrueForgeApi.ActionRequiredEvent; + +export interface CollectedTurn { + events: TurnEvent[]; + turnId: string | undefined; + previousTurnId: string | null | undefined; + threadIds: string[]; + threadId: string | undefined; + finalText: string; + consolidatedText: string | undefined; + accumulatedText: string; + sandboxIds: string[]; + terminal: TurnDoneEvent | undefined; +} + +function messageText(content: TrueForgeApi.ModelMessageEventContent | null | undefined): string { + if (content === undefined || content === null) { + return ''; + } + if (typeof content === 'string') { + return content; + } + return content.map(part => (part.type === 'text' ? part.text : '')).join(''); +} + +/** Fold stream deltas into base `model.message` events. Keep lifecycle events (OSS persists them). */ +export function gatherEvents(events: TurnEvent[]): PersistedTurnEvent[] { + const gathered: PersistedTurnEvent[] = []; + const indexById = new Map(); + for (const event of events) { + if (isEventDelta(event)) { + const idx = indexById.get(event.id); + const base = idx !== undefined ? gathered[idx] : undefined; + if (base?.type === 'model.message') { + mergeEventDelta(base, event); + } + continue; + } + if (event.type === 'model.message') { + indexById.set(event.id, gathered.length); + gathered.push(structuredClone(event)); + } else { + gathered.push(event); + } + } + return gathered; +} + +export function requiredActions(turn: CollectedTurn): ActionRequiredEvent[] { + const state = turn.terminal?.state; + return state?.status === 'done' ? state.requiredActions : []; +} + +function isActionType( + action: ActionRequiredEvent, + type: T, +): action is Extract { + return action.type === type; +} + +export function requireAction({ + turn, + type, + label, +}: { + turn: CollectedTurn; + type: T; + label?: string; +}): Extract { + const actions = requiredActions(turn); + const found = actions.find(action => isActionType(action, type)); + if (found === undefined) { + const at = label !== undefined ? `[${label}] ` : ''; + throw new Error(`${at}expected a ${type} required action, got: ${actions.map(a => a.type).join(', ') || '(none)'}`); + } + return found; +} + +export function summarizeTurn(events: TurnEvent[]): CollectedTurn { + const deltasById = new Map(); + const threadIds = new Set(); + const sandboxIds: string[] = []; + let created: TurnCreatedEvent | undefined; + let terminal: TurnDoneEvent | undefined; + + for (const event of events) { + if ('threadId' in event && event.threadId) { + threadIds.add(event.threadId); + } + switch (event.type) { + case 'turn.created': + created = event; + break; + case 'turn.done': + terminal = event; + break; + case 'sandbox.created': + sandboxIds.push(event.sandboxId); + break; + case 'model.message.delta': + deltasById.set(event.id, (deltasById.get(event.id) ?? '') + (event.content ?? '')); + break; + } + } + + const output = terminal?.state.status === 'done' ? terminal.state.output : undefined; + const consolidatedText = output !== undefined && output !== null ? messageText(output.content) : undefined; + const accumulatedText = + output?.id !== undefined ? (deltasById.get(output.id) ?? '') : [...deltasById.values()].join(''); + + return { + events, + turnId: created?.turnId, + previousTurnId: created?.previousTurnId, + threadIds: [...threadIds], + threadId: [...threadIds][0], + finalText: consolidatedText ?? accumulatedText, + consolidatedText, + accumulatedText, + sandboxIds, + terminal, + }; +} + +export async function collectTurn({ + client, + sessionId, + input, + previousTurnId, +}: { + client: TrueForge; + sessionId: string; + input?: TrueForgeApi.TurnInputItem[]; + previousTurnId?: TrueForgeApi.PreviousTurnIdInput; +}): Promise { + const request: TrueForgeApi.CreateTurnSessionsStreamRequest = {}; + if (input !== undefined) { + request.input = input; + } + if (previousTurnId !== undefined) { + request.previousTurnId = previousTurnId; + } + const stream = await client.sessions.createTurnStream(sessionId, request); + const events: TurnEvent[] = []; + for await (const event of stream) { + events.push(event); + } + const turn = summarizeTurn(events); + await assertGatheredEventsMatchListEvents({ client, sessionId, turn }); + return turn; +} + +const IGNORED_EVENT_FIELDS = new Set(['createdAt']); + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(canonicalize); + } + if (isPlainObject(value)) { + const out: Record = {}; + for (const key of Object.keys(value).sort()) { + if (IGNORED_EVENT_FIELDS.has(key)) { + continue; + } + const v = value[key]; + out[key] = canonicalize(v); + } + return out; + } + return value; +} + +async function assertGatheredEventsMatchListEvents({ + client, + sessionId, + turn, +}: { + client: TrueForge; + sessionId: string; + turn: CollectedTurn; +}): Promise { + if (turn.turnId === undefined) { + throw new InvariantError('cannot reconcile listTurnEvents: turn.created is missing a turn id'); + } + const listed: PersistedTurnEvent[] = []; + const page = await client.sessions.listTurnEvents(sessionId, turn.turnId, { order: 'asc' }); + for await (const event of page) { + listed.push(event); + } + + const gathered = gatherEvents(turn.events); + const remaining = new Map(); + for (const event of gathered) { + const key = JSON.stringify(canonicalize(event)); + const bucket = remaining.get(key); + if (bucket !== undefined) { + bucket.push(event); + } else { + remaining.set(key, [event]); + } + } + + const missing: PersistedTurnEvent[] = []; + for (const event of listed) { + const bucket = remaining.get(JSON.stringify(canonicalize(event))); + if (bucket !== undefined && bucket.length > 0) { + bucket.pop(); + } else { + missing.push(event); + } + } + + const streamOnly = [...remaining.values()].flat(); + if (missing.length > 0 || streamOnly.length > 0) { + throw new InvariantError( + `gathered stream events do not reconcile with listTurnEvents (createdAt ignored).\n` + + `gathered=${String(gathered.length)} (${gathered.map(e => e.type).join(', ')})\n` + + `listed=${String(listed.length)} (${listed.map(e => e.type).join(', ')})\n` + + `persisted but never streamed:\n${missing.map(e => JSON.stringify(canonicalize(e))).join('\n') || '(none)'}\n` + + `streamed but never persisted:\n${streamOnly.map(e => JSON.stringify(canonicalize(e))).join('\n') || '(none)'}`, + ); + } +} + +export class InvariantError extends Error { + constructor(message: string) { + super(message); + this.name = 'InvariantError'; + } +} + +function invariant(condition: unknown, message: string): asserts condition { + if (!condition) { + throw new InvariantError(message); + } +} + +type DeepPartial = T extends (infer U)[] + ? DeepPartial[] + : T extends object + ? { [K in keyof T]?: DeepPartial } + : T; + +export type ExpectedRequiredAction = DeepPartial; + +export interface TurnInvariantOptions { + label?: string; + previousTurnId?: string | null; + expectSandbox?: boolean; + expectRequiredAction?: ExpectedRequiredAction; + allowMultipleThreads?: boolean; +} + +function partialMatch({ actual, expected }: { actual: unknown; expected: unknown }): boolean { + if (actual === expected) { + return true; + } + if (typeof expected !== 'object' || expected === null) { + return false; + } + if (typeof actual !== 'object' || actual === null) { + return false; + } + if (Array.isArray(expected)) { + return Array.isArray(actual) && expected.every((item, i) => partialMatch({ actual: actual[i], expected: item })); + } + if (!isPlainObject(actual) || !isPlainObject(expected)) { + return false; + } + return Object.entries(expected).every(([key, value]) => partialMatch({ actual: actual[key], expected: value })); +} + +export function assertTurnInvariants(turn: CollectedTurn, opts: TurnInvariantOptions = {}): void { + const at = opts.label !== undefined ? `[${opts.label}] ` : ''; + const { events } = turn; + + invariant(events.length > 0, `${at}turn produced no events`); + invariant( + events[0]?.type === 'turn.created', + `${at}first event must be turn.created, got ${String(events[0]?.type)}`, + ); + invariant( + events[events.length - 1]?.type === 'turn.done', + `${at}last event must be turn.done, got ${String(events[events.length - 1]?.type)}`, + ); + invariant(turn.turnId, `${at}turn.created is missing a turn_id`); + invariant(turn.terminal !== undefined, `${at}stream ended without a turn.done event`); + + if (opts.allowMultipleThreads !== true) { + invariant(turn.threadIds.length <= 1, `${at}events span multiple thread_ids: ${turn.threadIds.join(', ')}`); + } + + const createdIndex = new Map(); + events.forEach((event, i) => { + if (event.type === 'thread.created') { + createdIndex.set(event.threadId, i); + } + }); + events.forEach((event, i) => { + if (event.type === 'thread.done') { + const opened = createdIndex.get(event.threadId); + invariant( + opened === undefined || opened < i, + `${at}thread ${event.threadId} emitted thread.done before its thread.created within the turn`, + ); + } + }); + + const state = turn.terminal.state; + if (state.status === 'error') { + throw new InvariantError(`${at}turn ended in error: ${state.message}`); + } + if (state.status === 'cancelled') { + throw new InvariantError(`${at}turn was cancelled: ${state.reason}`); + } + + if (opts.previousTurnId !== undefined) { + invariant( + turn.previousTurnId === opts.previousTurnId, + `${at}expected previous_turn_id ${String(opts.previousTurnId)}, got ${String(turn.previousTurnId)}`, + ); + } + + if (opts.expectSandbox === true) { + invariant(turn.sandboxIds.length > 0, `${at}expected a sandbox.created event, none observed`); + } + + if (turn.consolidatedText !== undefined && turn.accumulatedText.length > 0) { + invariant( + turn.consolidatedText === turn.accumulatedText, + `${at}final assistant text does not match accumulated streamed deltas.\nfinal: ${JSON.stringify(turn.consolidatedText)}\naccumulated: ${JSON.stringify(turn.accumulatedText)}`, + ); + } + + const pending = requiredActions(turn); + const want = opts.expectRequiredAction; + if (want !== undefined) { + invariant( + pending.some(action => partialMatch({ actual: action, expected: want })), + `${at}expected a pending required action matching ${JSON.stringify(want)}, got: ${pending.map(a => a.type).join(', ') || '(none)'}`, + ); + } else { + invariant( + pending.length === 0, + `${at}turn ended with ${String(pending.length)} unresolved required_action(s): ${pending.map(a => a.type).join(', ')}`, + ); + } +} + +export type TrackTurnOptions = Omit; + +export class SessionTracker { + readonly sessionId: string; + private lastTurnId: string | null = null; + private sandboxId: string | undefined; + private readonly openThreadIds = new Set(); + private readonly closedThreadIds = new Set(); + + constructor(sessionId: string) { + this.sessionId = sessionId; + } + + record(turn: CollectedTurn, opts: TrackTurnOptions = {}): string { + assertTurnInvariants(turn, { ...opts, previousTurnId: this.lastTurnId }); + + const at = opts.label !== undefined ? `[${opts.label}] ` : ''; + for (const sandboxId of turn.sandboxIds) { + if (this.sandboxId === undefined) { + this.sandboxId = sandboxId; + } else { + invariant( + sandboxId === this.sandboxId, + `${at}sandbox id changed across turns: expected ${this.sandboxId}, saw ${sandboxId} (persistence broken)`, + ); + } + } + + for (const event of turn.events) { + if (event.type === 'thread.created') { + invariant( + !this.openThreadIds.has(event.threadId) && !this.closedThreadIds.has(event.threadId), + `${at}thread ${event.threadId} was created more than once`, + ); + this.openThreadIds.add(event.threadId); + } else if (event.type === 'thread.done') { + invariant( + this.openThreadIds.has(event.threadId), + `${at}thread.done for ${event.threadId} without a preceding open thread.created`, + ); + if (event.state.status === 'error') { + throw new InvariantError(`${at}thread ${event.threadId} ended in error: ${event.state.error}`); + } + this.openThreadIds.delete(event.threadId); + this.closedThreadIds.add(event.threadId); + } + } + + invariant(turn.turnId, `${at}turn is missing a turn id after invariants`); + this.lastTurnId = turn.turnId; + return turn.turnId; + } + + assertAllThreadsClosed(label?: string): void { + const at = label !== undefined ? `[${label}] ` : ''; + const open = [...this.openThreadIds]; + invariant(open.length === 0, `${at}threads created but never completed (no thread.done): ${open.join(', ')}`); + } +} + +export interface TestCase { + name: string; + run: () => Promise; +} + +interface TestResult { + name: string; + ok: boolean; + ms: number; + error?: unknown; +} + +const GREEN = '\x1b[32m'; +const RED = '\x1b[31m'; +const DIM = '\x1b[2m'; +const RESET = '\x1b[0m'; + +export async function runTests({ tests, filter }: { tests: TestCase[]; filter?: string | undefined }): Promise { + const selected = filter !== undefined ? tests.filter(t => t.name.includes(filter)) : tests; + + if (selected.length === 0) { + console.error( + filter !== undefined + ? `${RED}No tests match filter "${filter}"${RESET} — check for a typo or stale name (nothing was run).` + : `${RED}No tests registered${RESET} (nothing was run).`, + ); + return 1; + } + + console.log(`Running ${String(selected.length)} E2E test(s)...\n`); + + const results: TestResult[] = []; + for (const test of selected) { + const start = Date.now(); + try { + await test.run(); + const ms = Date.now() - start; + results.push({ name: test.name, ok: true, ms }); + console.log(`${GREEN}PASS${RESET} ${test.name} ${DIM}(${String(ms)}ms)${RESET}`); + } catch (error) { + const ms = Date.now() - start; + results.push({ name: test.name, ok: false, ms, error }); + console.log(`${RED}FAIL${RESET} ${test.name} ${DIM}(${String(ms)}ms)${RESET}`); + console.log(`${DIM}${errorMessage(error)}${RESET}\n`); + } + } + + const passed = results.filter(r => r.ok).length; + const failed = results.length - passed; + console.log(`\n${'-'.repeat(48)}`); + console.log(`${String(passed)} passed, ${String(failed)} failed, ${String(results.length)} total`); + if (failed > 0) { + console.log( + `${RED}Failed:${RESET} ${results + .filter(r => !r.ok) + .map(r => r.name) + .join(', ')}`, + ); + } + + return failed > 0 ? 1 : 0; +} + +function parseModelFqn(name: string): { providerType: string; modelName: string } { + const slash = name.indexOf('/'); + if (slash <= 0 || slash === name.length - 1 || name.includes('/', slash + 1)) { + throw new Error(`MODEL must be a fully qualified "provider/model", got: ${name}`); + } + return { providerType: name.slice(0, slash), modelName: name.slice(slash + 1) }; +} + +function isWellKnownProviderType(value: string): value is TrueForgeApi.CatalogWellKnownModelProviderType { + return WELL_KNOWN_PROVIDER_TYPES.some(type => type === value); +} + +function wellKnownProviderManifest({ + type, + apiKey, + modelName, +}: { + type: TrueForgeApi.CatalogWellKnownModelProviderType; + apiKey: string; + modelName: string; +}): TrueForgeApi.ModelProviderManifest { + const auth = { apiKey }; + const models = [{ modelId: modelName, name: modelName, properties: {} }]; + switch (type) { + case 'openai': + return { type: 'openai', auth, models }; + case 'anthropic': + return { type: 'anthropic', auth, models }; + case 'google-gemini': + return { type: 'google-gemini', auth, models }; + case 'fireworks': + return { type: 'fireworks', auth, models }; + case 'zai': + return { type: 'zai', auth, models }; + case 'moonshot': + return { type: 'moonshot', auth, models }; + case 'alibaba': + return { type: 'alibaba', auth, models }; + case 'together': + return { type: 'together', auth, models }; + } +} + +async function upsertNamedAgent({ client, spec }: { client: TrueForge; spec: TrueForgeApi.AgentSpec }): Promise { + const listed = await client.agents.list(); + const existing = listed.data.find(agent => agent.name === NAMED_AGENT); + if (existing !== undefined) { + await client.agents.update(existing.id, { manifest: spec }); + return; + } + try { + await client.agents.create({ name: NAMED_AGENT, manifest: spec }); + } catch (error) { + if (!(error instanceof TrueForgeApi.ConflictError)) { + throw error; + } + const retry = await client.agents.list(); + const created = retry.data.find(agent => agent.name === NAMED_AGENT); + if (created === undefined) { + throw error; + } + await client.agents.update(created.id, { manifest: spec }); + } +} + +/** Idempotent settings + named agent used by later cases. */ +export async function upsertE2eResources(client: TrueForge): Promise { + const fqn = requireEnv('MODEL'); + const { providerType, modelName } = parseModelFqn(fqn); + if (!isWellKnownProviderType(providerType)) { + throw new Error( + `MODEL provider "${providerType}" is not a well-known type (${WELL_KNOWN_PROVIDER_TYPES.join(', ')})`, + ); + } + + await client.settings.modelProviders.createOrUpdate({ + manifest: wellKnownProviderManifest({ + type: providerType, + apiKey: requireEnv('MODEL_API_KEY'), + modelName, + }), + }); + + await client.settings.mcpServers.createOrUpdate({ + manifest: { + type: 'remote', + name: MCP_DEEPWIKI, + url: 'https://mcp.deepwiki.com/mcp', + description: 'Read documentation and ask questions about any public GitHub repository.', + }, + }); + await client.settings.mcpServers.createOrUpdate({ + manifest: { + type: 'remote', + name: MCP_LINEAR, + url: 'https://mcp.linear.app/mcp', + description: 'Search, read, and create Linear issues.', + auth: { type: 'dcr' }, + }, + }); + + await client.settings.sandboxProviders.createOrUpdate({ + manifest: { + type: 'daytona', + auth: { apiKey: requireEnv('DAYTONA_API_KEY') }, + execTimeoutMs: 60_000, + autoStopIntervalInMinutes: 5, + autoArchiveIntervalInMinutes: 60, + autoDeleteIntervalInMinutes: 43_200, + }, + }); + + await upsertNamedAgent({ + client, + spec: baseAgentSpec({ + instructions: 'You are a terse assistant. Follow instructions exactly and keep replies short.', + }), + }); +} diff --git a/packages/trueforge/e2e/run.ts b/packages/trueforge/e2e/run.ts new file mode 100644 index 000000000..5632005ba --- /dev/null +++ b/packages/trueforge/e2e/run.ts @@ -0,0 +1,41 @@ +/** + * E2E runner: load `e2e/.env` (real environment variables win), upsert settings, + * run registered scenarios (optionally `--only `), exit non-zero on failure + * or when the filter matches nothing. + */ +import { existsSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createClient, errorMessage, runTests, upsertE2eResources } from './helpers'; +import { tests } from './scenarios'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +function loadDotEnv(path: string): void { + if (!existsSync(path)) { + return; + } + const preexisting = { ...process.env }; + process.loadEnvFile(path); + Object.assign(process.env, preexisting); +} + +function parseOnlyFilter(argv: string[]): string | undefined { + const idx = argv.indexOf('--only'); + const value = idx !== -1 ? argv[idx + 1] : undefined; + return value !== undefined && value.trim() !== '' ? value : undefined; +} + +async function main(): Promise { + loadDotEnv(resolve(HERE, '.env')); + const client = createClient(); + console.log('Upserting E2E resources (model, MCP, sandbox, named agent)...'); + await upsertE2eResources(client); + const exitCode = await runTests({ tests, filter: parseOnlyFilter(process.argv) }); + process.exitCode = exitCode; +} + +main().catch((err: unknown) => { + console.error(errorMessage(err)); + process.exitCode = 1; +}); diff --git a/packages/trueforge/e2e/scenarios.ts b/packages/trueforge/e2e/scenarios.ts new file mode 100644 index 000000000..fef495892 --- /dev/null +++ b/packages/trueforge/e2e/scenarios.ts @@ -0,0 +1,383 @@ +/** + * Sequential E2E scenarios. Each proves one v1 flow against the compose stack. + */ +import { type TrueForge, type TrueForgeApi } from '@truefoundry/trueforge-sdk'; +import { + MCP_DEEPWIKI, + MCP_LINEAR, + NAMED_AGENT, + SessionTracker, + approveToolCall, + baseAgentSpec, + collectTurn, + createClient, + createInlineSession, + createNamedAgentSession, + denyToolCall, + httpStatusCode, + makeNonce, + requireAction, + textFileContent, + userMessage, + type TestCase, +} from './helpers'; + +async function runMemoryRecall({ + client, + session, + label, +}: { + client: TrueForge; + session: TrueForgeApi.Session; + label: string; +}): Promise { + const nonce = makeNonce('SESSION'); + const tracker = new SessionTracker(session.id); + + const turn1 = await collectTurn({ + client, + sessionId: session.id, + input: [userMessage(`Remember this exact code for later: ${nonce}. Reply with just "ok".`)], + }); + const turn1Id = tracker.record(turn1, { label: `${label} turn1` }); + + const turn2 = await collectTurn({ + client, + sessionId: session.id, + input: [userMessage('What exact code did I ask you to remember? Reply with only the code.')], + previousTurnId: turn1Id, + }); + tracker.record(turn2, { label: `${label} turn2` }); + + if (!turn2.finalText.includes(nonce)) { + throw new Error( + `turn 2 did not recall the nonce from turn 1 context.\nexpected to contain: ${nonce}\ngot: ${turn2.finalText}`, + ); + } +} + +/** Inline agent: a later turn recalls a nonce stored in session context. */ +const sessionMemoryTest: TestCase = { + name: 'session_memory', + run: async () => { + const client = createClient(); + const session = await createInlineSession({ + client, + spec: baseAgentSpec({ + instructions: 'You are a terse assistant. Follow instructions exactly and keep replies short.', + }), + }); + await runMemoryRecall({ client, session, label: 'session_memory' }); + }, +}; + +/** Named agent (`e2e-memory`): same recall check on a catalog agent, not an inline spec. */ +const namedAgentMemoryTest: TestCase = { + name: 'named_agent_memory', + run: async () => { + const client = createClient(); + const session = await createNamedAgentSession({ client, name: NAMED_AGENT }); + await runMemoryRecall({ client, session, label: 'named_agent_memory' }); + }, +}; + +/** Preloaded Linear DCR MCP: connecting surfaces `mcp.auth_required` with a non-empty auth URL. */ +const mcpAuthRequiredTest: TestCase = { + name: 'mcp_auth_required', + run: async () => { + const client = createClient(); + const session = await createInlineSession({ + client, + spec: baseAgentSpec({ + instructions: 'You have access to an MCP server. Keep replies short.', + mcpServers: [{ name: MCP_LINEAR, preload: true }], + }), + }); + const tracker = new SessionTracker(session.id); + const turn = await collectTurn({ + client, + sessionId: session.id, + input: [userMessage('List the tools available from your MCP server.')], + }); + tracker.record(turn, { + label: 'mcp_auth_required', + expectRequiredAction: { type: 'mcp.auth_required', mcpServers: [{ name: MCP_LINEAR }] }, + }); + + const auth = requireAction({ turn, type: 'mcp.auth_required', label: 'mcp_auth_required' }); + const server = auth.mcpServers.find(s => s.name === MCP_LINEAR) ?? auth.mcpServers[0]; + if (server === undefined || server.authUrl.trim() === '') { + throw new Error( + `mcp.auth_required did not include an authUrl for ${MCP_LINEAR}. servers: ${JSON.stringify(auth.mcpServers)}`, + ); + } + }, +}; + +/** Pending `ask_user_question`: a follow-up user message without an answer is rejected with HTTP 422. */ +const unresolvedRequiredActionTest: TestCase = { + name: 'unresolved_required_action', + run: async () => { + const client = createClient(); + const session = await createInlineSession({ + client, + spec: baseAgentSpec({ + instructions: + 'Before doing anything, you MUST gather required details from the user by calling the ask_user_question ' + + 'tool. Never guess or assume the answer. Keep replies short.', + config: { askUserQuestions: { enabled: true } }, + }), + }); + const tracker = new SessionTracker(session.id); + const turn1 = await collectTurn({ + client, + sessionId: session.id, + input: [ + userMessage( + 'Book me a meeting room for tomorrow. First use the ask_user_question tool to ask which office building ' + + 'I want the room in — do not proceed until I answer.', + ), + ], + }); + const turn1Id = tracker.record(turn1, { + label: 'unresolved turn1', + expectRequiredAction: { type: 'tool.response_required' }, + }); + + let thrown: unknown; + try { + await collectTurn({ + client, + sessionId: session.id, + input: [userMessage('Actually, never mind the question — just book any room.')], + previousTurnId: turn1Id, + }); + } catch (error) { + thrown = error; + } + if (thrown === undefined) { + throw new Error('expected turn 2 to be rejected for leaving the pending question unresolved, but it succeeded'); + } + const statusCode = httpStatusCode(thrown); + if (statusCode !== 422) { + throw new Error( + `expected turn 2 to be rejected with HTTP 422 (unprocessable send while a question is pending), ` + + `but got ${statusCode === undefined ? 'a non-HTTP failure' : `HTTP ${String(statusCode)}`}`, + { cause: thrown }, + ); + } + }, +}; + +/** Cancel as soon as `turn.created` is seen, without blocking the SSE read, then both stream and `getTurn` are `cancelled`. */ +const turnCancellationTest: TestCase = { + name: 'turn_cancellation', + run: async () => { + const client = createClient(); + const session = await createInlineSession({ + client, + spec: baseAgentSpec({ + instructions: 'You are a verbose assistant. When asked for a long answer, keep going and never stop early.', + }), + }); + + const stream = await client.sessions.createTurnStream(session.id, { + input: [ + userMessage( + 'Write an extremely long, exhaustive essay of at least 3000 words about the full history of computing. ' + + 'Do not summarize and do not stop early.', + ), + ], + }); + + const events: TrueForgeApi.TurnStreamingEvent[] = []; + let cancelInFlight: Promise | undefined; + for await (const event of stream) { + events.push(event); + if (cancelInFlight === undefined && event.type === 'turn.created') { + // Do not await here: blocking the SSE consumer lets a fast turn finish as `done` before cancel is applied. + cancelInFlight = client.sessions.cancel(session.id); + } + } + if (cancelInFlight === undefined) { + throw new Error('streamed turn.created is missing; cancel was never requested.'); + } + await cancelInFlight; + + const terminal = events.at(-1); + if (terminal?.type !== 'turn.done') { + throw new Error(`expected the stream to end with turn.done, got ${terminal?.type ?? '(no events)'}.`); + } + if (terminal.state.status !== 'cancelled') { + throw new Error(`expected the streamed turn.done state to be "cancelled", got "${terminal.state.status}".`); + } + + const created = events.find(event => event.type === 'turn.created'); + if (created === undefined) { + throw new Error('streamed turn.created is missing; cannot verify cancel via getTurn.'); + } + const listed = await client.sessions.getTurn(session.id, created.turnId); + if (listed.data.state.status !== 'cancelled') { + throw new Error(`getTurn reports turn ${created.turnId} as "${listed.data.state.status}", expected "cancelled".`); + } + }, +}; + +const SUBAGENT_INSTRUCTIONS = + 'When asked to look something up, you MUST delegate the work to a sub-agent via the create_sub_agent ' + + 'tool rather than calling the tool yourself. The sub-agent must make exactly one tool call. Keep replies short.'; + +const SUBAGENT_TASK = + 'Create a sub-agent and instruct it to call ONLY the deepwiki `ask_question` tool exactly once, ' + + 'with repoName "facebook/react" and question "What is this repository about?". ' + + 'It must not call any other tool.'; + +async function startApprovalFlow(scenario: string) { + const client = createClient(); + const session = await createInlineSession({ + client, + spec: baseAgentSpec({ + instructions: SUBAGENT_INSTRUCTIONS, + config: { dynamicSubAgents: { enabled: true } }, + mcpServers: [{ name: MCP_DEEPWIKI, requireApprovalForTools: ['@all'] }], + }), + }); + const tracker = new SessionTracker(session.id); + const turn1 = await collectTurn({ + client, + sessionId: session.id, + input: [userMessage(SUBAGENT_TASK)], + }); + const turn1Id = tracker.record(turn1, { + label: `${scenario} turn1`, + expectRequiredAction: { type: 'tool.approval_required' }, + allowMultipleThreads: true, + }); + const approval = requireAction({ turn: turn1, type: 'tool.approval_required', label: `${scenario} turn1` }); + const toolCall = approval.toolCalls[0]; + if (toolCall === undefined) { + throw new Error('tool.approval_required action carried no tool calls'); + } + return { client, session, tracker, turn1Id, threadId: approval.threadId, toolCallId: toolCall.id }; +} + +/** Sub-agent MCP tool with `@all` approval: allowing the call runs the tool and finishes the thread. */ +const subagentToolApprovalAllowTest: TestCase = { + name: 'subagent_tool_approval_allow', + run: async () => { + const { client, session, tracker, turn1Id, threadId, toolCallId } = await startApprovalFlow('approve'); + const turn2 = await collectTurn({ + client, + sessionId: session.id, + input: [approveToolCall({ threadId, toolCallId })], + previousTurnId: turn1Id, + }); + tracker.record(turn2, { label: 'approve turn2', allowMultipleThreads: true }); + const executed = turn2.events.some(e => e.type === 'tool.response' && e.toolCallId === toolCallId); + if (!executed) { + throw new Error(`approved tool call ${toolCallId} did not execute in turn 2 (no matching tool.response event).`); + } + if (!turn2.events.some(e => e.type === 'thread.done')) { + throw new Error( + 'expected the sub-agent thread to complete (thread.done) in turn 2 after approval, none observed.', + ); + } + tracker.assertAllThreadsClosed('approve'); + }, +}; + +/** Same approval pause: denying the call returns a denial `tool.response` and still closes the thread. */ +const subagentToolApprovalDenyTest: TestCase = { + name: 'subagent_tool_approval_deny', + run: async () => { + const { client, session, tracker, turn1Id, threadId, toolCallId } = await startApprovalFlow('deny'); + const turn2 = await collectTurn({ + client, + sessionId: session.id, + input: [denyToolCall({ threadId, toolCallId, reason: 'denied by e2e test' })], + previousTurnId: turn1Id, + }); + tracker.record(turn2, { label: 'deny turn2', allowMultipleThreads: true }); + const response = turn2.events.find(e => e.type === 'tool.response' && e.toolCallId === toolCallId); + if (response?.type !== 'tool.response') { + throw new Error(`denied tool call ${toolCallId} produced no tool.response in turn 2.`); + } + if (!response.content.includes('denied')) { + throw new Error(`expected a denial tool.response for ${toolCallId}, got content: ${response.content}`); + } + if (!turn2.events.some(e => e.type === 'thread.done')) { + throw new Error('expected the sub-agent thread to complete (thread.done) in turn 2 after denial, none observed.'); + } + tracker.assertAllThreadsClosed('deny'); + }, +}; + +const UPLOAD_NAME = 'upload.txt'; +const UPLOAD_MARKER = 'Distinctive marker line: MAGENTA-OTTER-7731-CANYON-BELL'; + +/** Uploaded sandbox file is unread on turn 1, then read on turn 2 from the same sandbox (not a new one). */ +const sandboxPersistenceTest: TestCase = { + name: 'sandbox_persistence', + run: async () => { + const client = createClient(); + const session = await createInlineSession({ + client, + spec: baseAgentSpec({ + instructions: 'You have a code sandbox. Use it to read files exactly as asked. Keep replies short.', + config: { sandbox: { enabled: true } }, + }), + }); + const tracker = new SessionTracker(session.id); + const turn1 = await collectTurn({ + client, + sessionId: session.id, + input: [ + userMessage([ + { + type: 'text', + text: + `A text file named ${UPLOAD_NAME} has been uploaded to your sandbox. ` + + `DO NOT READ IT NOW. YOU MUST REPLY WITH JUST OK.`, + }, + textFileContent({ + name: UPLOAD_NAME, + text: `Agent-harness sandbox upload fixture.\n${UPLOAD_MARKER}\n`, + }), + ]), + ], + }); + const turn1Id = tracker.record(turn1, { label: 'sandbox turn1', expectSandbox: true }); + const turn2 = await collectTurn({ + client, + sessionId: session.id, + input: [ + userMessage( + `Print the exact contents of the ${UPLOAD_NAME} file that was uploaded to you earlier. Output it verbatim.`, + ), + ], + previousTurnId: turn1Id, + }); + tracker.record(turn2, { label: 'sandbox turn2' }); + if (turn2.sandboxIds.length > 0) { + throw new Error( + `expected turn 2 to reuse the persisted sandbox, but it provisioned a new one: ${turn2.sandboxIds.join(', ')}`, + ); + } + if (!turn2.finalText.includes(UPLOAD_MARKER)) { + throw new Error( + `turn 2 did not read the uploaded file back from the persisted sandbox.\nexpected to contain: ${UPLOAD_MARKER}\ngot: ${turn2.finalText}`, + ); + } + }, +}; + +export const tests: TestCase[] = [ + sessionMemoryTest, + mcpAuthRequiredTest, + unresolvedRequiredActionTest, + turnCancellationTest, + subagentToolApprovalAllowTest, + subagentToolApprovalDenyTest, + sandboxPersistenceTest, + namedAgentMemoryTest, +]; diff --git a/packages/trueforge/e2e/tsconfig.json b/packages/trueforge/e2e/tsconfig.json new file mode 100644 index 000000000..86ebbd547 --- /dev/null +++ b/packages/trueforge/e2e/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["**/*.ts"] +} diff --git a/packages/trueforge/package.json b/packages/trueforge/package.json index a1def7185..ec1995c15 100644 --- a/packages/trueforge/package.json +++ b/packages/trueforge/package.json @@ -48,12 +48,13 @@ "test:eventsub": "jest --config jest.eventsub.config.cjs", "test:store:postgres": "jest --config jest.store.postgres.config.cjs", "test:store:sqlite": "jest --config jest.store.sqlite.config.cjs", + "e2e": "tsx e2e/run.ts", "test": "pnpm run build:gen && NODE_OPTIONS='--conditions=trueforge-dev' node --env-file=.env.test ./node_modules/jest/bin/jest.js --config jest.unit.config.cjs", "test:local-sandbox:contract": "pnpm run build:gen && NODE_OPTIONS='--conditions=trueforge-dev' node --env-file=.env.test ./node_modules/jest/bin/jest.js --config jest.local-sandbox.contract.config.cjs", "smoke:local-sandbox": "pnpm run build:gen && NODE_OPTIONS='--conditions=trueforge-dev' jest --config jest.local-sandbox.smoke.config.cjs --runInBand --forceExit tests/sandbox/local/smoke.test.ts", "smoke:local-sandbox:lima": "bash scripts/local-sandbox/smoke-lima.sh", "probe:loopback": "pnpm exec tsx scripts/local-sandbox/probe-loopback.ts", - "typecheck": "pnpm run build:gen && tsc --noEmit && tsc --noEmit -p tests/db/tsconfig.json && tsc --noEmit -p tests/unit/tsconfig.json" + "typecheck": "pnpm run build:gen && tsc --noEmit && tsc --noEmit -p tests/db/tsconfig.json && tsc --noEmit -p tests/unit/tsconfig.json && tsc --noEmit -p e2e/tsconfig.json" }, "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.71", @@ -79,6 +80,7 @@ "devDependencies": { "@swc/core": "^1.11.0", "@swc/jest": "^0.2.37", + "@truefoundry/trueforge-sdk": "workspace:*", "@types/better-sqlite3": "^7.6.13", "@types/jest": "^29.5.14", "@types/node": "^24.12.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f3f53f440..937ca8ee6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -176,6 +176,9 @@ importers: '@swc/jest': specifier: ^0.2.37 version: 0.2.39(@swc/core@1.15.46) + '@truefoundry/trueforge-sdk': + specifier: workspace:* + version: link:../trueforge-sdk '@types/better-sqlite3': specifier: ^7.6.13 version: 7.6.13 diff --git a/scripts/e2e.sh b/scripts/e2e.sh new file mode 100755 index 000000000..88780c721 --- /dev/null +++ b/scripts/e2e.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Local E2E contract: ephemeral Compose stack (isolated from host-dev and smoke), +# then the package CLI. Stack is always removed on exit (including failures). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +COMPOSE=(docker compose -f docker-compose.e2e.yml) +ENV_FILE="$ROOT/packages/trueforge/e2e/.env" +ENV_EXAMPLE="$ROOT/packages/trueforge/e2e/.env.example" +E2E_BASE_URL="${E2E_BASE_URL:-http://127.0.0.1:8792}" +STARTED_STACK=0 + +cleanup() { + if [[ "$STARTED_STACK" -eq 1 ]]; then + "${COMPOSE[@]}" down -v >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT INT TERM + +if ! command -v docker >/dev/null 2>&1; then + echo "error: docker is required to run E2E tests" >&2 + exit 1 +fi + +if [[ ! -f "$ENV_FILE" ]]; then + echo "error: missing $ENV_FILE" >&2 + echo "Copy $ENV_EXAMPLE to $ENV_FILE and fill in values." >&2 + exit 1 +fi + +echo "Starting E2E stack (API ${E2E_BASE_URL})..." +if ! "${COMPOSE[@]}" up --build --wait; then + echo "error: failed to start docker-compose.e2e.yml (server unhealthy, or ports 8792/5434/6381 in use)" >&2 + "${COMPOSE[@]}" logs --tail=200 server >&2 || true + STARTED_STACK=1 + exit 1 +fi +STARTED_STACK=1 + +node -e " +Promise.all([ + fetch('${E2E_BASE_URL}/healthz'), + fetch('${E2E_BASE_URL}/'), +]).then(async ([health, ui]) => { + if (!health.ok) throw new Error('healthz returned ' + health.status); + const html = await ui.text(); + if (!ui.ok || !html.includes('id=\"root\"')) throw new Error('UI did not return its app shell'); + console.log('healthz and UI OK'); +}); +" + +echo "Running E2E cases..." +pnpm --filter @truefoundry/trueforge e2e -- "$@"