+ ))}
+
+
+
+ );
+ }
+
const requestHeadersCopy = record.request.headers.map(([name, value]) => `${name}: ${value}`).join('\n');
const responseHeadersCopy = record.response.headers.map(([name, value]) => `${name}: ${value}`).join('\n');
const collectedCopyText = collected?.result === null || collected?.result === undefined
diff --git a/apps/web/src/components/requests/run-render.ts b/apps/web/src/components/requests/run-render.ts
new file mode 100644
index 0000000000..9679c0f54c
--- /dev/null
+++ b/apps/web/src/components/requests/run-render.ts
@@ -0,0 +1,46 @@
+import { errorMessage } from '../../lib/error-message';
+import type { DumpEvent } from '@floway-dev/gateway/dump-types';
+
+// A pipelined turn is recorded as its whole run: one NDJSON line per event, in
+// the order the run emitted them. A line is also one SSE `data:` payload, so
+// what this reads out of a stored record is what a live observer would be handed
+// frame by frame.
+
+export interface RenderedRunEvent {
+ /** The event's own kind — `stage.entered`, `object`, `stream.frame`, … */
+ type: string;
+ /** Which stage, object or stream it is about. Ids, not prose. */
+ subject: string | null;
+ text: string;
+ parseError: string | null;
+}
+
+// Only three of the six name a stage, and each of the other three names its own
+// namespace, so the subject is read off the event rather than looked up.
+const subjectOf = (event: DumpEvent): string | null => {
+ switch (event.type) {
+ case 'stage.entered': return event.name;
+ case 'stage.leaved': return `#${event.stageId}`;
+ case 'stage.log': return event.level;
+ case 'object': return `#${event.fromObjectId}`;
+ case 'stream.frame':
+ case 'stream.end': return `#${event.streamId}`;
+ }
+};
+
+// A line the gateway wrote and this cannot parse is shown as it was stored, with
+// the failure named: a record that renders as nothing would read as an empty run.
+export const renderRunEvents = (ndjson: string): RenderedRunEvent[] =>
+ ndjson.split('\n').filter(line => line.length > 0).map(line => {
+ try {
+ const event = JSON.parse(line) as DumpEvent;
+ return {
+ type: event.type,
+ subject: subjectOf(event),
+ text: JSON.stringify(event, null, 2),
+ parseError: null,
+ };
+ } catch (error) {
+ return { type: '', subject: null, text: line, parseError: errorMessage(error) };
+ }
+ });
diff --git a/apps/web/src/i18n/locales/en.ts b/apps/web/src/i18n/locales/en.ts
index 54c44b93e2..4e8add8de4 100644
--- a/apps/web/src/i18n/locales/en.ts
+++ b/apps/web/src/i18n/locales/en.ts
@@ -957,6 +957,8 @@ const en = {
apiKeysLink: 'API Keys',
request: 'Request',
requestBody: 'Request body',
+ run: 'Run',
+ noRunEvents: 'This run recorded no events.',
response: 'Response',
responseBody: 'Response body',
noRequestBody: 'No request body',
diff --git a/apps/web/src/i18n/locales/zh-Hans.ts b/apps/web/src/i18n/locales/zh-Hans.ts
index 0cad703f1d..5958791d8c 100644
--- a/apps/web/src/i18n/locales/zh-Hans.ts
+++ b/apps/web/src/i18n/locales/zh-Hans.ts
@@ -910,6 +910,8 @@ const zhHansCN = {
apiKeysLink: 'API 密钥',
request: '请求',
requestBody: '请求体',
+ run: '运行过程',
+ noRunEvents: '这次运行没有记录到任何事件。',
response: '响应',
responseBody: '响应体',
noRequestBody: '没有请求体',
diff --git a/apps/web/src/routes/dashboard-monitor-requests.tsx b/apps/web/src/routes/dashboard-monitor-requests.tsx
index aaf1054b3e..f7caa264f6 100644
--- a/apps/web/src/routes/dashboard-monitor-requests.tsx
+++ b/apps/web/src/routes/dashboard-monitor-requests.tsx
@@ -66,7 +66,9 @@ export async function clientLoader({ request }: Route.ClientLoaderArgs): Promise
]);
const record = recordResult?.data ?? null;
const collectKind = record ? detectCollectKind(record.meta.path) : null;
- const streamEvents = record?.response.body.type === 'stream' ? record.response.body.events : [];
+ // Only an edge-shaped record carries a captured frame log to collect; a run
+ // records its stream inside its own events and the detail panel reads it there.
+ const streamEvents = record?.shape === 'edge' && record.response.body.type === 'stream' ? record.response.body.events : [];
const collected = collectKind && streamEvents.length ? await collectStream(collectKind, streamEvents) : null;
return {
collected,
diff --git a/packages/gateway/__tests__/control-plane/dump_test.ts b/packages/gateway/__tests__/control-plane/dump_test.ts
index 6bc9dd9f0d..2bebb7dd7c 100644
--- a/packages/gateway/__tests__/control-plane/dump_test.ts
+++ b/packages/gateway/__tests__/control-plane/dump_test.ts
@@ -3,7 +3,7 @@ import { test } from 'vitest';
import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts';
import type { DumpStore } from '../../src/dump/store-contract.ts';
import type { DumpMetadata, DumpRecord, StoredDumpRecord } from '../../src/dump/types.ts';
-import { fakeMeta as baseFakeMeta, fakeRecord as baseFakeRecord, installDumpStubs } from '../dump/test-fixtures.ts';
+import { fakeMeta as baseFakeMeta, fakeRecord as baseFakeRecord, fakeRunRecord, installDumpStubs } from '../dump/test-fixtures.ts';
import { requestApp, setupAppTest } from '../test-utils/app.ts';
import { assertEquals, assertExists } from '@floway-dev/test-utils';
@@ -80,6 +80,41 @@ test('GET /api/dump/keys/:keyId/records/:recordId returns the rehydrated record'
assertEquals(body.meta.id, '01HZZ0000000000000000000XX');
});
+// One list, both shapes — the metadata is common, so the dashboard's list says
+// nothing about which mechanism served a turn — and a detail fetch hands each
+// one back as what it is.
+test('GET /api/dump/keys/:keyId/records serves a run record beside an edge one', async () => {
+ const { repo, apiKey } = await setupAppTest();
+ await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 });
+ const stubs = installDumpStubs(initDumpStore, initDumpBroker);
+ stubs.seed(apiKey.id, fakeRecord('01HZZ0000000000000000EDGE', 1000));
+ stubs.seed(apiKey.id, fakeRunRecord(
+ [
+ { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: { 'request.payload': { model: 'm' } } },
+ { type: 'stage.leaved', stageId: 1, facts: { 'response.http.status': 200 } },
+ ],
+ { id: '01HZZ00000000000000000RUN', completedAt: 2000, startedAt: 1999 },
+ ));
+
+ const listed = await requestApp(`/api/dump/keys/${apiKey.id}/records`, { headers: { 'x-api-key': apiKey.key } });
+ assertEquals(listed.status, 200);
+ const { records } = await listed.json() as { records: DumpMetadata[] };
+ assertEquals(records.map(meta => meta.id), ['01HZZ00000000000000000RUN', '01HZZ0000000000000000EDGE']);
+
+ const run = await requestApp(`/api/dump/keys/${apiKey.id}/records/01HZZ00000000000000000RUN`, { headers: { 'x-api-key': apiKey.key } });
+ const runBody = await run.json() as DumpRecord;
+ if (runBody.shape !== 'run') throw new Error(`expected the run shape, got ${runBody.shape}`);
+ // The object space rides with the events that reference it, so the `object`
+ // event carrying the payload arrives before the entry that points at it.
+ assertEquals(runBody.events.trimEnd().split('\n').map(line => (JSON.parse(line) as { type: string }).type),
+ ['object', 'stage.entered', 'stage.leaved']);
+
+ const edge = await requestApp(`/api/dump/keys/${apiKey.id}/records/01HZZ0000000000000000EDGE`, { headers: { 'x-api-key': apiKey.key } });
+ const edgeBody = await edge.json() as DumpRecord;
+ if (edgeBody.shape !== 'edge') throw new Error(`expected the edge shape, got ${edgeBody.shape}`);
+ assertEquals(edgeBody.request.method, 'POST');
+});
+
test('GET /api/dump/keys/:keyId/records/:recordId 404s on unknown id', async () => {
const { repo, apiKey } = await setupAppTest();
await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 });
diff --git a/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts b/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts
index f10fb3bf80..947a24c4bd 100644
--- a/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts
+++ b/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts
@@ -4,9 +4,11 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
import { mountAlphaSearchRoutes } from '../../../src/data-plane/alpha-search/routes.ts';
import { resolveConfiguredWebSearchProvider } from '../../../src/data-plane/tools/web-search/provider.ts';
import type { WebSearchConfig, WebSearchFetchPageRequest, WebSearchFetchPageResult, WebSearchProvider, WebSearchProviderRequest, WebSearchProviderResult } from '../../../src/data-plane/tools/web-search/types.ts';
+import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts';
import { type AuthVars, authMiddleware } from '../../../src/middleware/auth.ts';
import { internalErrorResponse } from '../../../src/middleware/internal-error-response.ts';
-import { buildCustomUpstreamRecord, setupAppTest } from '../../test-utils/app.ts';
+import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts';
+import { buildCustomUpstreamRecord, flushAsyncWork, setupAppTest } from '../../test-utils/app.ts';
import { withMockedFetch } from '@floway-dev/test-utils';
// Real provider construction (`createTavilyWebSearchProvider` etc.) hits the
@@ -333,6 +335,35 @@ describe('/alpha/search data plane', () => {
});
});
+ // The endpoint is served by the pipeline that was written for it, which is what a run record
+ // says and nothing else does: the handler it replaced opened no run at all, so a turn on this
+ // path produced no record however much retention the key had.
+ describe('the run it is served by', () => {
+ it('records the whole run, stage by stage', async () => {
+ const { apiKey, repo } = await setupAppTest({ webSearchConfig: TAVILY_CONFIG });
+ await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 });
+ const stub = makeStubProvider();
+ mockResolveConfigured.mockReturnValue({ type: 'enabled', provider: 'tavily', impl: stub.provider });
+ const dumpStubs = installDumpStubs(initDumpStore, initDumpBroker);
+
+ const response = await postSearch(buildAlphaSearchApp(), apiKey.key, { commands: { search_query: [{ q: 'Floway' }] } });
+ expect(response.status).toBe(200);
+ await response.json();
+ await flushAsyncWork();
+
+ expect(dumpStubs.stored).toHaveLength(1);
+ const record = runRecordOf(dumpStubs.stored[0]?.record);
+ expect(record.meta.path).toBe(SEARCH_PATH);
+ expect(record.meta.status).toBe(200);
+ // The chain the operator's configuration assembled: the edge, settlement, and the local
+ // ending's two stages.
+ const entered = eventsOf(record)
+ .filter(event => event.type === 'stage.entered')
+ .map(event => event.name);
+ expect(entered).toEqual(['emitAlphaSearch', 'writeSettlement', 'parseSearchOperations', 'executeSearchOperations']);
+ });
+ });
+
describe('provider not configured', () => {
it('surfaces disabled search as in-band output text (contract-shaped 200)', async () => {
const { apiKey, repo } = await setupAppTest();
diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/serve_test.ts
index 93653e7194..ac38b981c5 100644
--- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/serve_test.ts
+++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/serve_test.ts
@@ -705,7 +705,7 @@ test('alias whose targets have no kind-matching binding surfaces as the regular
// callback so the http.ts catch can build an internal-error result carrying
// the correct upstream, and `recordFailedRequest` lands a row rather than
// short-circuiting on missing telemetry. Passthrough's equivalent regression
-// lives in passthrough-serve_test.ts (R3 fix 303c4e89).
+// lived in passthrough-serve_test.ts, which went with that surface (R3 fix 303c4e89).
test('mid-attempt throw stamps telemetry with the throwing candidate, not the previous one', async () => {
installRepo();
const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), {
diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts
index 1fe9b1e322..a0cd1ad248 100644
--- a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts
+++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts
@@ -8,7 +8,7 @@ import { KEEP_ALIVE_EVENT_TYPE } from '../../../../src/data-plane/chat/openai-re
import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS } from '../../../../src/data-plane/shared/sse.ts';
import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts';
import { initBackgroundSchedulerResolver } from '../../../../src/runtime/background.ts';
-import { installDumpStubs } from '../../../dump/test-fixtures.ts';
+import { edgeRecordOf, installDumpStubs } from '../../../dump/test-fixtures.ts';
import { FakeTime } from '../../../test-time.ts';
import { buildCodexUpstreamRecord, codexModels, copilotModels, flushAsyncWork, setupAppTest, sseResponse, sseOpenAIResponsesResponse } from '../../../test-utils/app.ts';
import { trackBackground } from '../../../test-utils/background-tracker.ts';
@@ -252,9 +252,10 @@ test('OpenAI Responses WebSocket starts capturing on the next turn when dump ret
const stored = dumps.stored[0];
assertExists(stored);
assertEquals(stored.keyId, apiKey.id);
- assertEquals(stored.record.request.method, 'WS');
- assertEquals(stored.record.request.path, '/v1/responses');
- assertEquals(JSON.parse(new TextDecoder().decode(stored.record.request.body)), {
+ const request = edgeRecordOf(stored.record).request;
+ assertEquals(request.method, 'WS');
+ assertEquals(request.path, '/v1/responses');
+ assertEquals(JSON.parse(new TextDecoder().decode(request.body)), {
type: 'response.create',
event_id: 'capture-after-enable',
response: {
diff --git a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts
new file mode 100644
index 0000000000..3bf5712372
--- /dev/null
+++ b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts
@@ -0,0 +1,170 @@
+// OpenAI Audio Transcriptions' pipeline, assembled. `compose` derives the entry contract and
+// rejects an array that cannot work, so most of what this file establishes is established
+// by the module importing at all — the assembly runs at load. What is worth writing down is
+// the entry contract it derives, the keys it cannot see, and the one runtime property this
+// family's code is shaped around that no declaration expresses.
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { openaiAudioTranscriptionServePipeline } from '../../src/data-plane/openai-audio/pipeline.ts';
+import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts';
+import { initRepo } from '../../src/repo/index.ts';
+import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts';
+import { isOwned, move, run } from '@floway-dev/pipeline';
+import type { SseFrame } from '@floway-dev/protocols/common';
+import { directFetcher, type ModelCandidate, type ProviderCallResult } from '@floway-dev/provider';
+import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils';
+
+vi.mock('../../src/data-plane/providers/resolution.ts', async importOriginal => ({
+ ...(await importOriginal()),
+ enumerateModelCandidates: vi.fn(),
+}));
+
+let live: readonly ModelCandidate[] = [];
+
+const candidate = (callOpenAIAudioTranscriptions: () => Promise): ModelCandidate => {
+ const endpoints = { openaiAudioTranscriptions: {} };
+ return {
+ provider: {
+ upstreamId: 'up_a', kind: 'custom', name: 'up_a', inboundHeaderAllowlist: [],
+ disabledPublicModelIds: [], modelPrefix: null, modelsCache: null,
+ instance: stubProvider({ callOpenAIAudioTranscriptions }),
+ },
+ model: stubInternalModel(
+ { id: 'whisper-1', kind: 'transcription', endpoints, providerModels: { up_a: stubProviderModel({ id: 'whisper-1', endpoints }) } },
+ 'up_a',
+ ),
+ fetcher: directFetcher,
+ } as unknown as ModelCandidate;
+};
+
+const resolves = (candidates: readonly ModelCandidate[]): void => {
+ live = candidates;
+ vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never);
+};
+
+const sse = (...events: readonly string[]): Response =>
+ new Response(events.map(event => `data: ${event}\n\n`).join(''), { status: 200, headers: { 'content-type': 'text/event-stream' } });
+
+const serve = async (responseFormat = 'json') => await run(
+ openaiAudioTranscriptionServePipeline,
+ move({
+ 'ingress.openaiAudioTranscription.responseFormat': responseFormat,
+ 'ingress.http.headers': [] as readonly (readonly [string, string])[],
+ 'request.openaiAudioTranscription.form': [{ name: 'model', value: 'whisper-1' }],
+ 'serve.model': 'whisper-1',
+ }) as never,
+ {
+ gateway: mockGatewayCtx({ wantsStream: responseFormat === 'stream' }),
+ background: () => {},
+ rememberCandidates: () => {},
+ resolveAttempt: (selector: { readonly upstreamId: string }) => {
+ const found = live.find(c => c.provider.upstreamId === selector.upstreamId);
+ if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`);
+ return found;
+ },
+ } as never,
+);
+
+beforeEach(() => {
+ vi.mocked(enumerateModelCandidates).mockReset();
+ initRepo({
+ usage: { record: async () => {} },
+ performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} },
+ } as never);
+});
+
+describe('the OpenAI Audio Transcriptions pipeline', () => {
+ it('assembles, and asks its caller for what the descending stages need', () => {
+ expect([...openaiAudioTranscriptionServePipeline.entryNeeds].sort()).toEqual([
+ 'ingress.openaiAudioTranscription.responseFormat',
+ 'serve.model',
+ ]);
+ });
+
+ // The ending stage reads `request.openaiAudioTranscription.form` and `ingress.http.headers`,
+ // and the entry contract mentions neither. That is not this family's defect: a stage whose
+ // only trait is `return` declares no request side at all, by ruling — "when it
+ // short-circuits, only `provides`" — so assembly cannot see what an ending stage reads.
+ //
+ // It bites harder here than it does for a family whose edge needs the request payload to
+ // render: this family's edge does not, so the payload the whole endpoint exists to send is
+ // among the keys assembly cannot ask for. A caller who omits it reaches the deepest stage
+ // before failing. The type layer still catches it at the definition site, which is why this
+ // is a gap and not a break.
+ it('cannot see the request payload, because only a return-only stage reads it', () => {
+ expect(openaiAudioTranscriptionServePipeline.entryNeeds).not.toContain('request.openaiAudioTranscription.form');
+ expect(openaiAudioTranscriptionServePipeline.entryNeeds).not.toContain('ingress.http.headers');
+ });
+
+ it('names the entry key a caller did not bring, before any stage runs', async () => {
+ await expect(run(openaiAudioTranscriptionServePipeline, move({ 'serve.model': 'whisper-1' }) as never, {}))
+ .rejects.toThrow('run(openaiAudioTranscriptionServe): openaiAudioTranscriptionServe needs');
+ });
+
+ // The streamed answer is a wrapper around its generator rather than the generator itself.
+ // That was once load-bearing: the runner detected ownership structurally, an async
+ // generator carries `Symbol.asyncDispose`, and the sweep would have called it — which for
+ // a generator is `return()`, cancelling the iteration rather than draining it.
+ //
+ // The runner now claims ownership through `own()` instead of detecting it, so a bare
+ // generator is safe in the record. The wrapper stays because it says which thing is the
+ // resource: the upstream body at `response.http.body`, and nothing else here.
+ it('renders a transcription the upstream answered with, as a 200', async () => {
+ resolves([candidate(async () => ({
+ response: Response.json({ text: 'hello there' }),
+ modelKey: 'whisper-key',
+ } as ProviderCallResult))]);
+
+ const { facts } = await serve();
+
+ expect(facts['response.http.status']).toBe(200);
+ expect(facts['response.openaiAudioTranscription.rendered']).toMatchObject({ text: 'hello there' });
+ });
+
+ // A refusal reaches the client with the status the upstream gave it, and in the words the
+ // upstream used — the same statement every other family makes.
+ it('answers an upstream refusal with its own status and words', async () => {
+ resolves([candidate(async () => ({
+ response: new Response(JSON.stringify({ error: { message: 'too large' } }), {
+ status: 413, headers: { 'content-type': 'application/json' },
+ }),
+ modelKey: 'whisper-key',
+ } as ProviderCallResult))]);
+
+ const { facts } = await serve();
+
+ expect(facts['response.http.status']).toBe(413);
+ expect(facts['response.openaiAudioTranscription.rendered']).toEqual({ error: { message: 'too large' } });
+ });
+
+ // The reader stops at the terminal event. An upstream that holds the connection open past
+ // it would otherwise hold the client's stream open with it, which is what the replaced
+ // surface avoided by cancelling the moment the transcript was complete.
+ it('stops reading a stream at its terminal event', async () => {
+ const done = JSON.stringify({ type: 'transcript.text.done', text: 'hi', usage: { type: 'tokens', input_tokens: 3, output_tokens: 1 } });
+ const after = JSON.stringify({ type: 'transcript.text.delta', delta: 'never read' });
+ resolves([candidate(async () => ({
+ response: sse(JSON.stringify({ type: 'transcript.text.delta', delta: 'hi' }), done, after),
+ modelKey: 'whisper-key',
+ } as ProviderCallResult))]);
+
+ const { facts, drain } = await serve('json');
+ const frames: SseFrame[] = [];
+ for await (const frame of facts['response.openaiAudioTranscription.rendered'] as AsyncIterable) frames.push(frame);
+ await drain();
+
+ expect(frames).toHaveLength(2);
+ expect(frames.map(frame => JSON.parse(frame.data) as { type: string }).map(event => event.type))
+ .toEqual(['transcript.text.delta', 'transcript.text.done']);
+ });
+
+ it('keeps the events view clear of what answers to release', () => {
+ // Neither the generator nor the view around it is a resource. Whether the host marks a
+ // generator disposable varies — Node 24 does, Node 22 does not — and the run's answer
+ // does not, because ownership is claimed rather than read off the value.
+ const generator = (async function* () { yield 1; })();
+ expect(isOwned(generator)).toBe(false);
+ expect(isOwned({ [Symbol.asyncIterator]: () => generator })).toBe(false);
+ });
+});
diff --git a/packages/gateway/__tests__/data-plane/openai-audio/http_test.ts b/packages/gateway/__tests__/data-plane/openai-audio/http_test.ts
index ae8934926d..51a12605bf 100644
--- a/packages/gateway/__tests__/data-plane/openai-audio/http_test.ts
+++ b/packages/gateway/__tests__/data-plane/openai-audio/http_test.ts
@@ -1,5 +1,7 @@
import { test, vi } from 'vitest';
+import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts';
+import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts';
import type { InMemoryRepo } from '../../repo/memory.ts';
import { flushAsyncWork, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts';
import type { ModelPricing } from '@floway-dev/protocols/common';
@@ -150,6 +152,44 @@ test('/v1/audio/transcriptions forwards VTT verbatim and records request-only us
assertEquals(usage.metrics, []);
});
+// A subtitle document is carried, so every convention the upstream did not follow survives:
+// the ordinal it started at, the line ending it chose, and the fraction digits it wrote.
+// Reading the cues and writing them back would have normalized all three.
+test('/v1/audio/transcriptions forwards SubRip with the numbering and line endings the upstream chose', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await registerAudioModel(repo);
+ const document = '7\r\n00:00:00,0 --> 00:00:01,5\r\nhello\r\n';
+ await withMockedFetch(
+ () => new Response(document, { headers: { 'content-type': 'application/x-subrip' } }),
+ async () => {
+ const response = await requestApp('/v1/audio/transcriptions', {
+ method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['response_format', 'srt']]),
+ });
+ assertEquals(response.headers.get('content-type'), 'application/x-subrip');
+ assertEquals(await response.text(), document);
+ },
+ );
+});
+
+// The document never becomes a string on the way through, so an upstream that answered under
+// a charset nobody here asked about is forwarded rather than mangled: decoding these bytes as
+// UTF-8 and encoding them again would replace both of them with U+FFFD.
+test('/v1/audio/transcriptions forwards a document the gateway never decoded', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await registerAudioModel(repo);
+ const latin1 = new Uint8Array([0x63, 0x61, 0x66, 0xE9, 0x20, 0xE0, 0x20, 0x31]);
+ await withMockedFetch(
+ () => new Response(latin1, { headers: { 'content-type': 'text/plain; charset=iso-8859-1' } }),
+ async () => {
+ const response = await requestApp('/v1/audio/transcriptions', {
+ method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['response_format', 'text']]),
+ });
+ assertEquals(response.headers.get('content-type'), 'text/plain; charset=iso-8859-1');
+ assertEquals(new Uint8Array(await response.arrayBuffer()), latin1);
+ },
+ );
+});
+
test('/v1/audio/transcriptions skips JSON parsing for text responses without warning', async () => {
const { apiKey, repo } = await setupAppTest();
await registerAudioModel(repo);
@@ -331,9 +371,11 @@ test('/v1/audio/transcriptions preserves token usage unpriced when the model is
test('/v1/audio/transcriptions streams through transcript.text.done without adding Chat termination', async () => {
const { apiKey, repo } = await setupAppTest();
+ await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 });
await registerAudioModel(repo, {
entries: [{ rates: { input_audio_tokens: '0.000001', output_tokens: '0.000001' } }],
});
+ const dumpStubs = installDumpStubs(initDumpStore, initDumpBroker);
await withMockedFetch(
() => new Response([
'data: {"type":"transcript.text.delta","delta":"hel"}',
@@ -362,6 +404,16 @@ test('/v1/audio/transcriptions streams through transcript.text.done without addi
const [performance] = await repo.performance.listAll();
assertEquals(performance.neutral, 1);
assertEquals(performance.errorsNoOutput, 0);
+
+ // This family's stream is bare protocol events rather than protocol frames, so the record is
+ // told how one becomes a frame — and until it was, a streamed transcription was recorded as
+ // stage boundaries with the transcript missing entirely.
+ const events = eventsOf(runRecordOf(dumpStubs.stored[0]?.record));
+ const frames = events
+ .filter(event => event.type === 'stream.frame')
+ .flatMap(event => (event as unknown as { frames: readonly unknown[] }).frames);
+ assertEquals(frames.length, 2);
+ assertEquals(events.filter(event => event.type === 'stream.end'), [{ type: 'stream.end', streamId: 1 }]);
});
test('/v1/audio/transcriptions preserves a terminal stream event with malformed usage', async () => {
diff --git a/packages/gateway/__tests__/data-plane/openai-audio/usage_test.ts b/packages/gateway/__tests__/data-plane/openai-audio/usage_test.ts
deleted file mode 100644
index d55bd38f6a..0000000000
--- a/packages/gateway/__tests__/data-plane/openai-audio/usage_test.ts
+++ /dev/null
@@ -1,91 +0,0 @@
-import { test } from 'vitest';
-
-import { openaiAudioTranscriptionUsageMeasurement } from '../../../src/data-plane/openai-audio/usage.ts';
-import { assertEquals, assertThrows } from '@floway-dev/test-utils';
-
-test('audio transcription usage preserves duration seconds as a base-unit metric', () => {
- assertEquals(openaiAudioTranscriptionUsageMeasurement({
- usage: { type: 'duration', seconds: 91 },
- duration: 91.8,
- }), {
- quantities: { input_audio_seconds: '91' },
- pricingFacts: {},
- dumpTokenUsage: null,
- });
-});
-
-test('audio transcription usage reads Whisper verbose JSON duration', () => {
- assertEquals(openaiAudioTranscriptionUsageMeasurement({
- task: 'transcribe',
- duration: 91.8,
- text: 'hello',
- }), {
- quantities: { input_audio_seconds: '91.8' },
- pricingFacts: {},
- dumpTokenUsage: null,
- });
-});
-
-test('audio transcription usage maps text and audio input token details to disjoint metrics', () => {
- assertEquals(openaiAudioTranscriptionUsageMeasurement({
- usage: {
- type: 'tokens',
- input_tokens: 14,
- input_token_details: { text_tokens: 4, audio_tokens: 10 },
- output_tokens: 45,
- total_tokens: 59,
- },
- }), {
- quantities: { input_tokens: '4', input_audio_tokens: '10', output_tokens: '45' },
- pricingFacts: { inputTokens: 14 },
- dumpTokenUsage: { input: 14, output: 45 },
- });
-});
-
-test('audio transcription usage keeps aggregate input tokens general when details are absent', () => {
- assertEquals(openaiAudioTranscriptionUsageMeasurement({
- usage: { type: 'tokens', input_tokens: 14, output_tokens: 45, total_tokens: 59 },
- }).quantities, { input_tokens: '14', output_tokens: '45' });
-});
-
-test('audio transcription usage accepts partial details and leaves unclassified input general', () => {
- for (const [input_token_details, quantities] of [
- [{}, { input_tokens: '14', output_tokens: '45' }],
- [{ text_tokens: 4 }, { input_tokens: '14', output_tokens: '45' }],
- [{ audio_tokens: 10 }, { input_tokens: '4', input_audio_tokens: '10', output_tokens: '45' }],
- ] as const) {
- assertEquals(openaiAudioTranscriptionUsageMeasurement({
- usage: { type: 'tokens', input_tokens: 14, input_token_details, output_tokens: 45, total_tokens: 59 },
- }).quantities, quantities);
- }
-});
-
-test('audio transcription usage without a recognized metric is request-only', () => {
- for (const body of [
- { usage: { seconds: 10 } },
- { usage: { type: 'future_metric', samples: 10 } },
- ]) {
- assertEquals(openaiAudioTranscriptionUsageMeasurement(body), {
- quantities: {}, pricingFacts: {}, dumpTokenUsage: null,
- });
- }
-});
-
-test('audio transcription usage rejects malformed declared metrics', () => {
- for (const [body, message] of [
- [{ duration: '10' }, 'duration must be'],
- [{ usage: null }, 'usage must be an object'],
- [{ usage: 'tokens' }, 'usage must be an object'],
- [{ usage: { type: 'duration' } }, 'duration usage.seconds'],
- [{ usage: { type: 'duration', seconds: '10' } }, 'duration usage.seconds'],
- [{ usage: { type: 'tokens', input_tokens: -1, output_tokens: 45, total_tokens: 44 } }, 'token usage.input_tokens'],
- [{ usage: { type: 'tokens', input_tokens: 14, output_tokens: Number.NaN, total_tokens: 59 } }, 'token usage.output_tokens'],
- [{ usage: { type: 'tokens', input_tokens: 14, output_tokens: 45, total_tokens: '59' } }, 'token usage.total_tokens'],
- [{ usage: { type: 'tokens', input_tokens: 14, output_tokens: 45, total_tokens: 58 } }, 'total_tokens must equal'],
- [{ usage: { type: 'tokens', input_tokens: 14, input_token_details: null, output_tokens: 45, total_tokens: 59 } }, 'input_token_details must be an object'],
- [{ usage: { type: 'tokens', input_tokens: 14, input_token_details: { text_tokens: 4, audio_tokens: '10' }, output_tokens: 45, total_tokens: 59 } }, 'audio_tokens must be'],
- [{ usage: { type: 'tokens', input_tokens: 14, input_token_details: { text_tokens: 6, audio_tokens: 9 }, output_tokens: 45, total_tokens: 59 } }, 'input_token_details must not exceed'],
- ] as const) {
- assertThrows(() => openaiAudioTranscriptionUsageMeasurement(body), Error, message);
- }
-});
diff --git a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts
new file mode 100644
index 0000000000..2bdc7e6a9a
--- /dev/null
+++ b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts
@@ -0,0 +1,343 @@
+// OpenAI Completions' pipeline, assembled and run. `compose` derives the entry contract and
+// rejects an array that cannot work, so the assembly succeeding is itself most of what a test
+// of the wiring would say — what is written down here is the entry contract, the two things the
+// assembly cannot see, and one run of each shape the answer can take, because this is the
+// first family whose answer can be a stream and none of that is visible in a declaration.
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { openaiCompletionsServePipeline } from '../../src/data-plane/openai-completions/pipeline.ts';
+import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts';
+import { initRepo } from '../../src/repo/index.ts';
+import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts';
+import { isDeferred, move, run } from '@floway-dev/pipeline';
+import type { SseFrame } from '@floway-dev/protocols/common';
+import { directFetcher, type ModelCandidate, type ProviderCallResult, type UpstreamCallOptions } from '@floway-dev/provider';
+import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils';
+
+vi.mock('../../src/data-plane/providers/resolution.ts', async importOriginal => ({
+ ...(await importOriginal()),
+ enumerateModelCandidates: vi.fn(),
+}));
+
+/** The live candidates the resolver hands back. They never enter the record: a candidate
+ * carries the provider's instance, its fetcher and its models cache, and freezing those is
+ * what putting one in the record would do. What travels is the selector. */
+let live: readonly ModelCandidate[] = [];
+
+const resolves = (candidates: readonly ModelCandidate[]): void => {
+ live = candidates;
+ vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] });
+};
+
+const resolveAttempt = (selector: { readonly upstreamId: string }): ModelCandidate => {
+ const found = live.find(candidate => candidate.provider.upstreamId === selector.upstreamId);
+ if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`);
+ return found;
+};
+
+const candidate = (
+ upstream: string,
+ callOpenAICompletions: (model: unknown, body: unknown, signal: AbortSignal | undefined, opts: UpstreamCallOptions) => Promise,
+): ModelCandidate => {
+ const endpoints = { openaiChatCompletions: {}, openaiCompletions: {} };
+ return {
+ provider: {
+ upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [],
+ disabledPublicModelIds: [], modelPrefix: null, modelsCache: null,
+ instance: stubProvider({ callOpenAICompletions }),
+ },
+ model: stubInternalModel({ id: 'text-model', endpoints, providerModels: { [upstream]: stubProviderModel({ id: 'text-model', endpoints }) } }, upstream),
+ fetcher: directFetcher,
+ };
+};
+
+const sse = (...events: readonly string[]): Response =>
+ new Response(events.map(event => `data: ${event}\n\n`).join(''), { status: 200, headers: { 'content-type': 'text/event-stream' } });
+
+const chunk = (text: string): string =>
+ JSON.stringify({ id: 'cmpl_1', object: 'text_completion', created: 1, model: 'text-model', choices: [{ index: 0, text }] });
+
+const usageChunk = JSON.stringify({
+ id: 'cmpl_1', object: 'text_completion', created: 1, model: 'text-model', choices: [],
+ usage: { prompt_tokens: 5, completion_tokens: 7, total_tokens: 12 },
+});
+
+// Settlement is part of every serve pipeline now, and it writes. A test that drives a whole
+// pipeline therefore needs somewhere for the row to go — the point being that the write
+// happens at all, which is what "a run that measured rather than generated still writes"
+// means and what nothing was checking before.
+const recorded: { usage: unknown[]; performance: unknown[] } = { usage: [], performance: [] };
+
+beforeEach(() => {
+ recorded.usage = [];
+ recorded.performance = [];
+ initRepo({
+ usage: { record: async (row: unknown) => { recorded.usage.push(row); } },
+ performance: {
+ recordNeutral: async (dims: unknown) => { recorded.performance.push(dims); },
+ recordZeroOutputError: async (dims: unknown) => { recorded.performance.push(dims); },
+ },
+ } as never);
+});
+
+const serve = async (facts: Record) => await run(
+ openaiCompletionsServePipeline,
+ move(facts) as never,
+ {
+ gateway: mockGatewayCtx({ wantsStream: facts['ingress.openaiCompletions.wantsStream'] === true }),
+ background: () => {},
+ rememberCandidates: () => {},
+ resolveAttempt,
+ } as never,
+);
+
+const entryFacts = (overrides: Record = {}) => ({
+ 'ingress.openaiCompletions.wantsStream': true,
+ 'ingress.openaiCompletions.wantsUsageChunk': false,
+ 'ingress.http.headers': [],
+ 'request.openaiCompletions.payload': { model: 'text-model', prompt: 'hello', stream: true },
+ 'serve.model': 'text-model',
+ ...overrides,
+});
+
+const collect = async (rendered: unknown): Promise => {
+ const frames: SseFrame[] = [];
+ for await (const frame of rendered as AsyncIterable) frames.push(frame);
+ return frames;
+};
+
+beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); });
+
+describe('the OpenAI Completions pipeline', () => {
+ it('assembles, and asks its caller for what the descending stages need', () => {
+ expect([...openaiCompletionsServePipeline.entryNeeds].sort()).toEqual([
+ 'ingress.openaiCompletions.wantsStream',
+ 'ingress.openaiCompletions.wantsUsageChunk',
+ 'request.openaiCompletions.payload',
+ 'serve.model',
+ ]);
+ });
+
+ // `callOpenAICompletionsUpstream` reads `ingress.http.headers`, and the entry contract does
+ // not mention it. That is not this family's defect: a stage whose only trait is `return`
+ // declares no request side at all, by ruling — "when it short-circuits, only `provides`" —
+ // so assembly cannot see what an ending stage reads, and every family's ending stage reads
+ // something. A caller who omits that key gets a runtime failure at the deepest stage
+ // instead of the assembly error the entry contract exists to give it.
+ it('cannot see what an ending stage reads, because a return-only stage declares no needs', () => {
+ expect(openaiCompletionsServePipeline.entryNeeds).not.toContain('ingress.http.headers');
+ });
+
+ it('renders the upstream frames as SSE, hiding the usage chunk the client did not ask for', async () => {
+ let sent: Record | undefined;
+ resolves([candidate('up_a', async (_model, body) => {
+ sent = body as Record;
+ return { response: sse(chunk('he'), chunk('llo'), usageChunk, '[DONE]'), modelKey: 'text-model-key' };
+ })]);
+
+ const { facts, drain } = await serve(entryFacts());
+
+ // The answer comes back before the drain runs, which is what lets a streaming family
+ // hand its stream on: the frames are still there to read.
+ expect(await collect(facts['response.openaiCompletions.rendered'])).toEqual([
+ { type: 'sse', event: undefined, data: chunk('he') },
+ { type: 'sse', event: undefined, data: chunk('llo') },
+ { type: 'sse', event: undefined, data: '[DONE]' },
+ ]);
+ // Metering is the gateway's, not the client's: the upstream is always asked for the
+ // chunk the client is not shown.
+ expect(sent).toMatchObject({ stream_options: { include_usage: true } });
+ expect(sent).not.toHaveProperty('model');
+ await drain();
+ });
+
+ it('says the upstream was called and reported nothing until the frames run out', async () => {
+ resolves([candidate('up_a', async () => ({ response: sse(chunk('hi'), usageChunk, '[DONE]'), modelKey: 'text-model-key' }))]);
+
+ const { facts, drain } = await serve(entryFacts());
+
+ // What is known when the ending stage hands up: an entity, and no quantities. The
+ // numbers arrive with the last chunk, which is after this run has answered.
+ expect(facts['response.usage.billable']).toEqual([
+ { identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null }, quantities: {} },
+ ]);
+ // And the reading is declared as this run's own unfinished work rather than started and
+ // forgotten, which is what puts it in front of teardown: a family that hands up a reading
+ // that never settles is reported instead of silently never billing.
+ expect(isDeferred(facts['response.openaiCompletions.streamedUsage'])).toBe(true);
+ await collect(facts['response.openaiCompletions.rendered']);
+ expect(await facts['response.openaiCompletions.streamedUsage']).toMatchObject({
+ failed: false,
+ billable: [
+ {
+ identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null },
+ quantities: { input_tokens: '5', output_tokens: '7' },
+ },
+ ],
+ });
+ await drain();
+ });
+
+ it('shows the usage chunk to a client that asked for it', async () => {
+ resolves([candidate('up_a', async () => ({ response: sse(chunk('hi'), usageChunk, '[DONE]'), modelKey: 'text-model-key' }))]);
+
+ const { facts, drain } = await serve(entryFacts({ 'ingress.openaiCompletions.wantsUsageChunk': true }));
+
+ expect((await collect(facts['response.openaiCompletions.rendered'])).map(frame => frame.data)).toEqual([chunk('hi'), usageChunk, '[DONE]']);
+ await drain();
+ });
+
+ it('serializes a non-streaming answer from the value it parsed, and bills what it read', async () => {
+ const body = { id: 'cmpl_1', object: 'text_completion', created: 1, model: 'text-model', choices: [{ index: 0, text: 'hi', finish_reason: 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 7, total_tokens: 12 } };
+ resolves([candidate('up_a', async () => ({ response: Response.json(body), modelKey: 'text-model-key' }))]);
+
+ const { facts, drain } = await serve(entryFacts({
+ 'ingress.openaiCompletions.wantsStream': false,
+ 'request.openaiCompletions.payload': { model: 'text-model', prompt: 'hello' },
+ }));
+
+ expect(facts['response.openaiCompletions.rendered']).toEqual(body);
+ expect(facts['response.openaiCompletions.streamedUsage']).toBeNull();
+ expect(facts['response.usage.billable']).toEqual([
+ {
+ identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null },
+ quantities: { input_tokens: '5', output_tokens: '7' },
+ },
+ ]);
+ await drain();
+ });
+
+ // Ownership is claimed, never detected, and until the claim was made the whole mechanism
+ // was inert: `failover` declared it consumes the body, `drain()` existed, and no family
+ // ever marked a body — so a losing attempt's connection stayed open and the winner's was
+ // never drained. This is the property, not the absence of the bug.
+ it('drains the losing attempt-s body at the fork, and the winner-s at the drain', async () => {
+ const drained: string[] = [];
+ const body = (label: string, chunks: readonly string[]): ReadableStream => {
+ let index = 0;
+ return new ReadableStream({
+ pull: controller => {
+ if (index < chunks.length) { controller.enqueue(new TextEncoder().encode(chunks[index]!)); index += 1; return; }
+ drained.push(label);
+ controller.close();
+ },
+ });
+ };
+ resolves([
+ candidate('up_a', async () => ({
+ response: new Response(body('loser', []), { status: 429, headers: { 'content-type': 'application/json' } }),
+ modelKey: 'k',
+ })),
+ candidate('up_b', async () => ({
+ response: new Response(body('winner', [`data: ${chunk('hi')}\n\n`, 'data: [DONE]\n\n']), {
+ status: 200, headers: { 'content-type': 'text/event-stream' },
+ }),
+ modelKey: 'k',
+ })),
+ ]);
+
+ const { facts, drain } = await serve(entryFacts());
+ // The client's stream is still live: the run answered before anything was drained.
+ expect(facts['response.openaiCompletions.rendered']).toBeDefined();
+ await drain();
+ expect(drained).toContain('winner');
+ });
+
+ // Settlement is above the fork, so a run bills once however many candidates it tried —
+ // and it is unconditional, so a run that reached no upstream still writes a row that names
+ // no billed entity. Nothing asserted either until the review found the stage was composed
+ // into no pipeline at all.
+ it('writes exactly one usage row per run, however many candidates it tried', async () => {
+ resolves([
+ candidate('up_a', async () => ({ response: Response.json({ error: 'nope' }, { status: 429 }), modelKey: 'k' })),
+ candidate('up_b', async () => ({
+ response: Response.json({ id: 'c', choices: [], usage: { prompt_tokens: 7, completion_tokens: 2 } }),
+ modelKey: 'k',
+ })),
+ ]);
+ const { drain } = await serve(entryFacts({
+ 'ingress.openaiCompletions.wantsStream': false,
+ 'request.openaiCompletions.payload': { model: 'text-model', prompt: 'hello' },
+ }));
+ await drain();
+ expect(recorded.usage).toHaveLength(1);
+ expect(recorded.performance).toHaveLength(1);
+ });
+
+ // A stream states its usage in the chunk that ends it, which is after the run has answered.
+ // Settling in the stage as well would write the row twice — once for the entity that had
+ // reported nothing, once for what the stream turned out to say — so the pipeline hands the
+ // numbers up as a promise and the epilogue is what writes them.
+ it('defers a stream-s settlement to the promise it hands up', async () => {
+ resolves([candidate('up_b', async () => ({ response: sse(chunk('hi'), usageChunk, '[DONE]'), modelKey: 'k' }))]);
+
+ const { facts, drain } = await serve(entryFacts());
+ await drain();
+
+ expect(recorded.usage).toHaveLength(0);
+ const streamed = facts['response.openaiCompletions.streamedUsage'];
+ expect(streamed).not.toBeNull();
+ const outcome = await streamed!;
+ expect(outcome.billable).toHaveLength(1);
+ expect(outcome.billable[0]!.quantities).toMatchObject({ input_tokens: '5', output_tokens: '7' });
+ // The stream reached its terminator, so the turn produced what it said it would.
+ expect(outcome.failed).toBe(false);
+ });
+
+ // A run that reached no upstream bills nothing and samples nothing:
+ // `recordPerformance` returns early without an attempt's telemetry, and there is no
+ // attempt. Settlement still runs — it is unconditional — and finds an
+ // empty billed set, which is how "we did not call an upstream" is said.
+ it('bills nothing and samples nothing when no upstream was reached', async () => {
+ vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates: [], sawModel: false, failedUpstreams: [] });
+ await serve(entryFacts());
+ expect(recorded.usage).toHaveLength(0);
+ expect(recorded.performance).toHaveLength(0);
+ });
+
+ it('fails a refusal over to the next candidate, and renders the last one it got', async () => {
+ const tried: string[] = [];
+ resolves([
+ candidate('up_a', async () => {
+ tried.push('up_a');
+ return { response: Response.json({ error: { message: 'slow down' } }, { status: 429 }), modelKey: 'text-model-key' };
+ }),
+ candidate('up_b', async () => {
+ tried.push('up_b');
+ return { response: Response.json({ error: { message: 'no' } }, { status: 400 }), modelKey: 'text-model-key' };
+ }),
+ ]);
+
+ const { facts, drain } = await serve(entryFacts({
+ 'ingress.openaiCompletions.wantsStream': false,
+ 'request.openaiCompletions.payload': { model: 'text-model', prompt: 'hello' },
+ }));
+
+ expect(tried).toEqual(['up_a', 'up_b']);
+ // Every candidate failed, so the last failure is the base — the client sees the status
+ // an upstream actually returned, and the words that upstream used, rather than a
+ // synthesized envelope quoting its serialized body back as a message.
+ expect(facts['response.http.status']).toBe(400);
+ expect(facts['response.openaiCompletions.rendered']).toEqual({ error: { message: 'no' } });
+ await drain();
+ });
+
+ // Nothing in this family's own declarations says it must hand the upstream's body up, and
+ // nothing in assembly checks it: `failover` declares `provides: ['response.http.body']` on
+ // the way up, and that declaration is checked by the runner, at runtime, against whatever
+ // the stage below it handed on. A family whose ending stage parses the body and keeps it —
+ // which is what "no body is forwarded verbatim" invites — composes cleanly and then throws
+ // on its first request. The cast is the other half of the same story: the key rides in the
+ // record without being in the pipeline's exit type, because the path that refuses before
+ // any upstream is dialed has no body to hand up.
+ it('hands the upstream body up, because failover declares it provides one', async () => {
+ resolves([candidate('up_a', async () => ({ response: sse(chunk('hi'), '[DONE]'), modelKey: 'text-model-key' }))]);
+
+ const { facts, drain } = await serve(entryFacts());
+
+ expect((facts as Record)['response.http.body']).toBeInstanceOf(ReadableStream);
+ await collect(facts['response.openaiCompletions.rendered']);
+ await drain();
+ });
+});
diff --git a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts
index dc89209099..5edead4868 100644
--- a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts
+++ b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts
@@ -1,8 +1,8 @@
-import { test } from 'vitest';
+import { test, vi } from 'vitest';
import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts';
import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts';
-import { installDumpStubs } from '../../dump/test-fixtures.ts';
+import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts';
import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts';
import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot';
import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils';
@@ -311,15 +311,45 @@ test('/v1/completions non-streaming records usage row, performance neutral row (
assertEquals(performance[0]?.errorsNoOutput, 0);
assertEquals(dumpStubs.stored.length, 1);
- const dump = dumpStubs.stored[0]!.record;
+ const dump = runRecordOf(dumpStubs.stored[0]?.record);
assertEquals(dump.meta.path, '/v1/completions');
assertEquals(dump.meta.status, 200);
assertEquals(dump.meta.model, 'davinci-002');
assertEquals(dump.meta.inputTokens, 7);
assertEquals(dump.meta.outputTokens, 2);
- // Non-streaming: the upstream sent a one-shot JSON, so the dump
- // captures the bytes (not a frame log).
- assertEquals(dump.response.body.type, 'bytes');
+ // The shape follows the endpoint: a pipelined turn is recorded as its whole run, so what
+ // is stored is the event stream rather than the two edges. The metadata is common to both,
+ // which is what lets the dashboard list them together.
+ assertEquals(runRecordOf(dumpStubs.stored[0]!.record).shape, 'run');
+});
+
+// A stream that stops before its terminator did not produce what it said it would, and the
+// performance row has to say so — a neutral row would report a turn that never finished as
+// one that did.
+test('/v1/completions a stream that never terminated is recorded as a failed request', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await registerOpenAICompletionsUpstream(repo);
+
+ await withMockedFetch(
+ () => Promise.resolve(new Response('data: {"id":"c","object":"text_completion","created":1,"model":"davinci-002","choices":[{"index":0,"text":"hi"}]}\n\n', {
+ status: 200, headers: { 'content-type': 'text/event-stream' },
+ })),
+ async () => {
+ const response = await requestApp('/v1/completions', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
+ body: JSON.stringify({ model: 'davinci-002', prompt: 'hello', stream: true }),
+ });
+ assertEquals(response.status, 200);
+ await response.text();
+ },
+ );
+
+ await flushAsyncWork();
+
+ const performance = await repo.performance.listAll();
+ assertEquals(performance.length, 1);
+ assertEquals(performance[0]?.errorsNoOutput, 1);
});
test('/v1/completions streaming records usage row, performance neutral row (text_completion operation, no TTFT/TPOT), and a frame-log dump record', async () => {
@@ -353,32 +383,80 @@ test('/v1/completions streaming records usage row, performance neutral row (text
assertEquals(performance[0]?.errorsNoOutput, 0);
assertEquals(dumpStubs.stored.length, 1);
- const dump = dumpStubs.stored[0]!.record;
+ const dump = runRecordOf(dumpStubs.stored[0]?.record);
assertEquals(dump.meta.path, '/v1/completions');
assertEquals(dump.meta.status, 200);
assertEquals(dump.meta.model, 'davinci-002');
assertEquals(dump.meta.inputTokens, 4);
assertEquals(dump.meta.outputTokens, 2);
- // Streaming: dump stores the protocol frames the gateway saw from
- // upstream BEFORE transformFrame ran. The fixture stream emits two
- // content events, one usage-only event (which the client did not opt
- // into and so it was stripped from the forwarded stream), and a done
- // terminator.
- assertEquals(dump.response.body.type, 'stream');
- if (dump.response.body.type === 'stream') {
- const frames = dump.response.body.events.map(e => e.frame);
- assertEquals(frames.length, 4);
- assertEquals(frames[0]?.type, 'event');
- assertEquals(frames[1]?.type, 'event');
- assertEquals(frames[2]?.type, 'event');
- // Upstream's usage chunk is preserved in the dump even though it was
- // stripped from the client-facing stream.
- const usageFrame = frames[2];
- if (usageFrame?.type === 'event') {
- const event = usageFrame.event as { choices: unknown[]; usage: { prompt_tokens: number } };
- assertEquals(event.choices.length, 0);
- assertEquals(event.usage.prompt_tokens, 4);
- }
- assertEquals(frames[3]?.type, 'done');
+ // The frames the gateway saw from upstream, before the edge decided which of them the
+ // client is shown — recorded as `stream.frame` events in the run's own stream. The fixture
+ // emits two content events, one usage-only event (which this client did not opt into and
+ // so was stripped from what it received), and a done terminator.
+ const events = eventsOf(runRecordOf(dumpStubs.stored[0]!.record));
+ const frames = events
+ .filter(event => event.type === 'stream.frame')
+ .flatMap(event => (event as unknown as { frames: readonly unknown[] }).frames);
+ assertEquals(frames.length, 4);
+ // The usage chunk the client did not opt into is still in the run's own record: what the
+ // gateway saw is not narrowed to what it forwarded. The encoder shares repeated values, so
+ // what is asserted is that the numbers are in the stream, not the shape they took in it.
+ const stored = new TextDecoder().decode(runRecordOf(dumpStubs.stored[0]!.record).events);
+ assertEquals(stored.includes('"prompt_tokens":4'), true);
+
+ // The stream is named, and every frame says which stream it belongs to — the id is
+ // allocated per stream rather than fixed, so a run that opened two would keep them apart.
+ const streamIds = new Set(events.filter(event => event.type === 'stream.frame').map(event => event.streamId));
+ assertEquals([...streamIds], [1]);
+ // And the record of it is complete. A client that stopped reading would leave the frames it
+ // did get and no terminator, which is how a reader tells the two apart.
+ assertEquals(events.filter(event => event.type === 'stream.end'), [{ type: 'stream.end', streamId: 1 }]);
+ // The facts that hold the stream point at it: the answer the ending provided, and the
+ // framing the edge built for the client over the same frames.
+ assertEquals(stored.includes('"$stream":1'), true);
+});
+
+// A run that threw is the turn that most needs explaining and used to be the one that left
+// nothing behind: the record is closed at the seam, so an exception escaping past it lost the
+// whole thing — no row, no events, no reason. It is answered there now, with the same envelope
+// the app's own handler writes.
+test('/v1/completions a run that threw is recorded, with the reason and a debuggable body', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 });
+ await registerOpenAICompletionsUpstream(repo);
+ const dumpStubs = installDumpStubs(initDumpStore, initDumpBroker);
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+ // Read inside `resolveCandidates`, which is a stage — so the throw happens with the run
+ // open and a record already accumulating.
+ repo.modelAliases.getByName = () => Promise.reject(new Error('alias lookup exploded'));
+
+ try {
+ const response = await requestApp('/v1/completions', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
+ body: JSON.stringify({ model: 'davinci-002', prompt: 'hello' }),
+ });
+
+ assertEquals(response.status, 500);
+ const body = await response.json() as { error: { type: string; message: string; stack?: string } };
+ assertEquals(body.error.type, 'internal_error');
+ assertEquals(body.error.message, 'alias lookup exploded');
+ assertExists(body.error.stack);
+ } finally {
+ errorSpy.mockRestore();
}
+
+ await flushAsyncWork();
+
+ assertEquals(dumpStubs.stored.length, 1);
+ const dump = runRecordOf(dumpStubs.stored[0]?.record);
+ assertEquals(dump.meta.status, 500);
+ // The model the client asked for survives an outright failure, and the reason is on the row
+ // rather than only in the answer the client happened to receive.
+ assertEquals(dump.meta.model, 'davinci-002');
+ assertEquals(dump.meta.error, { kind: 'failed', reason: 'alias lookup exploded' });
+ // And the stages the run did get through are in it, which is what makes the record worth
+ // keeping: it says how far the turn got before it threw.
+ assertEquals(eventsOf(dump).some(event => event.type === 'stage.entered'), true);
});
diff --git a/packages/gateway/__tests__/data-plane/openai-embeddings-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-embeddings-pipeline_test.ts
new file mode 100644
index 0000000000..d9e19c3a28
--- /dev/null
+++ b/packages/gateway/__tests__/data-plane/openai-embeddings-pipeline_test.ts
@@ -0,0 +1,42 @@
+// OpenAI Embeddings' pipeline, assembled. `compose` derives the entry contract and rejects an
+// array that cannot work, so most of what this file establishes is established by the
+// assembly succeeding at all — and what is worth writing down is the entry contract it
+// derives, including the two keys it cannot see.
+
+import { describe, expect, it } from 'vitest';
+
+import { openaiEmbeddingsServePipeline } from '../../src/data-plane/openai-embeddings/pipeline.ts';
+import { move, run } from '@floway-dev/pipeline';
+
+describe('the OpenAI Embeddings pipeline', () => {
+ it('assembles, and asks its caller for what the descending stages need', () => {
+ expect([...openaiEmbeddingsServePipeline.entryNeeds].sort()).toEqual([
+ 'ingress.openaiEmbeddings.encodingFormat',
+ 'serve.model',
+ ]);
+ });
+
+ // Rerank records the same hole against `ingress.http.headers`; OpenAI Embeddings shows it
+ // twice over, and the second one is the sharper case. A stage whose only trait is `return`
+ // declares no request side at all, by ruling — "when it short-circuits, only `provides`"
+ // — so assembly cannot see what an ending stage reads. `callOpenAIEmbeddingsUpstream` reads
+ // both the headers and the request payload, and neither reaches the entry contract.
+ //
+ // Rerank's edge happened to need `request.rerank.canonical` for rendering, which put it
+ // in the contract for an unrelated reason. This family's edge needs only the encoding,
+ // so nothing above the ending stage names the payload — and the pipeline that cannot run
+ // without it does not ask for it. Written as a test rather than a comment because the
+ // hole has a consequence: a caller who omits either key gets a runtime failure at the
+ // deepest stage instead of an assembly error, and the entry contract exists to stop
+ // exactly that. The type layer still catches it at the definition site, which is why
+ // this is a gap and not a break.
+ it('cannot see what an ending stage reads, because a return-only stage declares no needs', () => {
+ expect(openaiEmbeddingsServePipeline.entryNeeds).not.toContain('request.openaiEmbeddings.canonical');
+ expect(openaiEmbeddingsServePipeline.entryNeeds).not.toContain('ingress.http.headers');
+ });
+
+ it('names the entry key a caller did not bring, before any stage runs', async () => {
+ await expect(run(openaiEmbeddingsServePipeline, move({ 'serve.model': 'text-embedding-3-small' }) as never, {}))
+ .rejects.toThrow('run(openaiEmbeddingsServe): openaiEmbeddingsServe needs');
+ });
+});
diff --git a/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts b/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts
new file mode 100644
index 0000000000..b06b87e761
--- /dev/null
+++ b/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts
@@ -0,0 +1,153 @@
+// What a run does when something other than the answer goes wrong. Each of these was a
+// behaviour of the passthrough serve that /v1/embeddings used to run through, and each is
+// kept here against the pipeline that replaced it — two unchanged, one deliberately not.
+
+import { test, vi } from 'vitest';
+
+import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts';
+import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot';
+import { jsonResponse, withMockedFetch, assertEquals } from '@floway-dev/test-utils';
+
+const registerOpenAIEmbeddingsUpstream = async (
+ repo: Awaited>['repo'],
+): Promise => {
+ await repo.upstreams.deleteAll();
+ clearInProcessCopilotTokenCache();
+ await repo.upstreams.save(buildCustomUpstreamRecord({
+ id: 'up_embeddings',
+ name: 'Embedding Provider',
+ sortOrder: 100,
+ config: {
+ baseUrl: 'https://embeddings.example.com',
+ authStyle: 'bearer',
+ ingressHeadersRules: [],
+ apiKey: 'sk-embeddings',
+ endpoints: {},
+ },
+ }));
+};
+
+const upstreamModels = () => jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] });
+
+const askForOpenAIEmbeddings = async (key: string): Promise => await requestApp('/v1/embeddings', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-api-key': key },
+ body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }),
+});
+
+// The answer does not depend on the row. Settlement hands the write to the background rather
+// than awaiting it, so a repository that rejects cannot turn an upstream's 2xx into a 502 —
+// and the failure is still reported, because a write nobody hears about is one nobody fixes.
+test('a usage write that fails leaves the answer alone and still reports', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await registerOpenAIEmbeddingsUpstream(repo);
+
+ repo.usage.record = () => Promise.reject(new Error('simulated SQL write failure'));
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+ try {
+ await withMockedFetch(
+ request => {
+ const url = new URL(request.url);
+ if (url.pathname === '/v1/models') return upstreamModels();
+ if (url.pathname === '/v1/embeddings') {
+ return jsonResponse({
+ object: 'list',
+ model: 'custom-embed-model',
+ data: [{ object: 'embedding', index: 0, embedding: [0.5] }],
+ usage: { prompt_tokens: 3, total_tokens: 3 },
+ });
+ }
+ throw new Error(`Unhandled fetch ${request.url}`);
+ },
+ async () => {
+ const response = await askForOpenAIEmbeddings(apiKey.key);
+
+ assertEquals(response.status, 200);
+ const body = await response.json() as { data: { embedding: number[] }[] };
+ assertEquals(body.data[0].embedding, [0.5]);
+ await flushAsyncWork();
+ },
+ );
+
+ assertEquals(errorSpy.mock.calls.some(call => String(call[0]).includes('usage')), true);
+ } finally {
+ errorSpy.mockRestore();
+ }
+});
+
+// A protocol that requires JSON and did not get JSON has no answer to serve, so the gateway
+// says so itself — it cannot claim to have served a request whose answer it never read.
+// Forwarding the body verbatim under the upstream's 200 would report success for a turn
+// nothing in this gateway could read.
+test('a 2xx body the OpenAI Embeddings protocol cannot read is refused rather than forwarded', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await registerOpenAIEmbeddingsUpstream(repo);
+
+ await withMockedFetch(
+ request => {
+ const url = new URL(request.url);
+ if (url.pathname === '/v1/models') return upstreamModels();
+ if (url.pathname === '/v1/embeddings') {
+ return new Response(new Uint8Array([0xde, 0xad, 0xbe, 0xef]), {
+ status: 200,
+ headers: { 'content-type': 'application/octet-stream' },
+ });
+ }
+ throw new Error(`Unhandled fetch ${request.url}`);
+ },
+ async () => {
+ const response = await askForOpenAIEmbeddings(apiKey.key);
+
+ assertEquals(response.status, 502);
+ assertEquals(response.headers.get('content-type'), 'application/json');
+ const body = await response.json() as { error: { message: string } };
+ assertEquals(body.error.message.includes('the OpenAI Embeddings protocol cannot read'), true);
+ await flushAsyncWork();
+ },
+ );
+
+ // An upstream that was called and reported nothing still counts as called: the row names
+ // the request, and a reading we could not parse is, from here, no reading rather than zero.
+ const usage = await repo.usage.listAll();
+ assertEquals(usage.length, 1);
+ assertEquals(usage[0].requests, 1);
+ assertEquals(usage[0].metrics, []);
+});
+
+// The last failure is the one the client is answered with, and an upstream that refused in
+// its own words is answered in them — with the status and the headers that carry what a
+// client does next.
+test('when every candidate refuses the client gets the last upstream-s own refusal', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await repo.upstreams.deleteAll();
+ clearInProcessCopilotTokenCache();
+ for (const [id, host, order] of [['up_a', 'up-a.example.com', 100], ['up_b', 'up-b.example.com', 200]] as const) {
+ await repo.upstreams.save(buildCustomUpstreamRecord({
+ id, name: id, sortOrder: order,
+ config: { baseUrl: `https://${host}`, authStyle: 'bearer', ingressHeadersRules: [], apiKey: 'sk-x', endpoints: {} },
+ }));
+ }
+
+ await withMockedFetch(
+ request => {
+ const url = new URL(request.url);
+ if (url.pathname === '/v1/models') return upstreamModels();
+ if (url.hostname === 'up-a.example.com') return new Response('first upstream unavailable', { status: 503 });
+ if (url.hostname === 'up-b.example.com') {
+ return new Response(JSON.stringify({ error: { message: 'rate limited' } }), {
+ status: 429, headers: { 'content-type': 'application/json', 'retry-after': '17' },
+ });
+ }
+ throw new Error(`Unhandled fetch ${request.url}`);
+ },
+ async () => {
+ const response = await askForOpenAIEmbeddings(apiKey.key);
+
+ assertEquals(response.status, 429);
+ assertEquals(response.headers.get('retry-after'), '17');
+ assertEquals(await response.json(), { error: { message: 'rate limited' } });
+ await flushAsyncWork();
+ },
+ );
+});
diff --git a/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts
new file mode 100644
index 0000000000..08f1d8aff5
--- /dev/null
+++ b/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts
@@ -0,0 +1,66 @@
+// OpenAI Images' pipeline, assembled. `compose` derives the entry contract and rejects an array
+// that cannot work, so most of what this file establishes is established by the assembly
+// succeeding at all — and what is worth writing down is the contract it derives and the keys it
+// cannot see.
+
+import { describe, expect, it } from 'vitest';
+
+import { openaiImagesServePipeline, type OpenAIImagesServeEntry } from '../../src/data-plane/openai-images/pipeline.ts';
+import type { CanonicalOpenAIImagesRequest } from '@floway-dev/protocols/openai-images';
+
+const generations: CanonicalOpenAIImagesRequest = {
+ operation: 'generations',
+ parameters: { prompt: 'a shiba in space' },
+};
+
+const edits: CanonicalOpenAIImagesRequest = {
+ operation: 'edits',
+ images: [{ kind: 'reference', reference: { file_id: 'file-source' } }],
+ parameters: { prompt: 'replace the sky' },
+};
+
+describe('the OpenAI Images pipeline', () => {
+ it('assembles both endpoints as one array, and asks its caller for what the descending stages need', () => {
+ expect([...openaiImagesServePipeline(generations).entryNeeds].sort()).toEqual(['serve.model']);
+ expect([...openaiImagesServePipeline(edits).entryNeeds].sort()).toEqual(['serve.model']);
+ });
+
+ // `callOpenAIImagesUpstream` reads `request.openaiImages.canonical`,
+ // `ingress.openaiImages.wantsStream` and `ingress.http.headers`, and the derived contract
+ // mentions none of them. That is not this family's defect: a stage whose only trait is
+ // `return` declares no request side at all, by ruling — "when it short-circuits, only
+ // `provides`" — so assembly cannot see what an ending stage reads, and every family's ending
+ // stage reads something.
+ //
+ // Written as a test rather than a comment because the hole has a consequence: a caller who
+ // omits any of them gets a runtime failure at the deepest stage instead of the assembly error
+ // the entry contract exists to produce.
+ it('cannot see what its ending stage reads, because a return-only stage declares no needs', () => {
+ const derived = openaiImagesServePipeline(generations).entryNeeds;
+ expect(derived).not.toContain('request.openaiImages.canonical');
+ expect(derived).not.toContain('ingress.openaiImages.wantsStream');
+ expect(derived).not.toContain('ingress.http.headers');
+ });
+
+ // What covers that hole, and the reason it is a gap rather than a break: the entry type names
+ // all four, so the caller `entryNeeds` would have let through does not compile.
+ it('names every key a caller must bring in its entry type', () => {
+ const entry: OpenAIImagesServeEntry = {
+ 'ingress.http.headers': [['content-type', 'application/json']],
+ 'ingress.openaiImages.wantsStream': false,
+ 'request.openaiImages.canonical': generations,
+ 'serve.model': 'gpt-image-1',
+ };
+ expect(Object.keys(entry).sort()).toEqual([
+ 'ingress.http.headers',
+ 'ingress.openaiImages.wantsStream',
+ 'request.openaiImages.canonical',
+ 'serve.model',
+ ]);
+
+ // @ts-expect-error — dropping one of them is a compile error, which is the statement this
+ // makes; the assertion below only keeps the binding from being unused.
+ const incomplete: OpenAIImagesServeEntry = { 'serve.model': 'gpt-image-1' };
+ expect(Object.keys(incomplete)).toEqual(['serve.model']);
+ });
+});
diff --git a/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts b/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts
new file mode 100644
index 0000000000..d9f1273728
--- /dev/null
+++ b/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts
@@ -0,0 +1,281 @@
+import { test, vi } from 'vitest';
+
+import type { InMemoryRepo } from '../../repo/memory.ts';
+import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts';
+import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot';
+import { withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils';
+
+// The upstream is declared rather than discovered: both OpenAI Images endpoints are named on
+// one manual model row, so nothing in these cases depends on the id heuristics that infer a
+// catalog's capabilities.
+const registerOpenAIImagesModel = async (repo: InMemoryRepo): Promise => {
+ await repo.upstreams.deleteAll();
+ clearInProcessCopilotTokenCache();
+ await repo.upstreams.save(buildCustomUpstreamRecord({
+ id: 'up_images',
+ name: 'Image Provider',
+ sortOrder: 100,
+ config: {
+ baseUrl: 'https://images.example.com',
+ authStyle: 'bearer',
+ ingressHeadersRules: [],
+ apiKey: 'sk-images',
+ endpoints: {},
+ modelsFetch: { enabled: false },
+ models: [{
+ upstreamModelId: 'gpt-image-2-upstream',
+ publicModelId: 'gpt-image-2',
+ kind: 'image',
+ endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} },
+ }],
+ },
+ }));
+};
+
+const PARTIAL = 'event: image_generation.partial_image\ndata: {"type":"image_generation.partial_image","b64_json":"UDA=","partial_image_index":0}\n\n';
+const COMPLETED = 'event: image_generation.completed\ndata: {"type":"image_generation.completed","b64_json":"RklO","usage":{"total_tokens":100,"input_tokens":50,"output_tokens":50,"input_tokens_details":{"text_tokens":10,"image_tokens":40}}}\n\n';
+
+const sseResponse = (body: string, headers: Record = {}): Response =>
+ new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream', ...headers } });
+
+test('/v1/images/generations answers a streaming request with the upstream events as SSE', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await registerOpenAIImagesModel(repo);
+ let forwarded: Record | undefined;
+
+ await withMockedFetch(
+ async request => {
+ const url = new URL(request.url);
+ if (url.hostname === 'images.example.com' && url.pathname === '/v1/images/generations') {
+ forwarded = await request.json() as Record;
+ return sseResponse(PARTIAL + COMPLETED, { 'x-image-trace': 'trace-stream', 'set-cookie': 'upstream-session=secret' });
+ }
+ throw new Error(`Unhandled fetch ${request.url}`);
+ },
+ async () => {
+ const response = await requestApp('/v1/images/generations', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
+ body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space', stream: true, partial_images: 1 }),
+ });
+ assertEquals(response.status, 200);
+ assertEquals(response.headers.get('content-type'), 'text/event-stream');
+ // Vendor traces stay visible; an upstream session does not.
+ assertEquals(response.headers.get('x-image-trace'), 'trace-stream');
+ assertEquals(response.headers.get('set-cookie'), null);
+
+ const stream = await response.text();
+ assertEquals(stream.includes('event: image_generation.partial_image'), true);
+ assertEquals(stream.includes('"partial_image_index":0'), true);
+ assertEquals(stream.includes('event: image_generation.completed'), true);
+ assertEquals(stream.includes('"b64_json":"RklO"'), true);
+ // This protocol ends at its completed event; the sentinel the chat dialects use is not
+ // part of it and is not invented here.
+ assertEquals(stream.includes('[DONE]'), false);
+ },
+ );
+
+ // The flag rides to the upstream exactly as the client wrote it — the gateway asks for the
+ // stream the client asked for, having no usage chunk of its own to turn on.
+ assertExists(forwarded);
+ assertEquals(forwarded.stream, true);
+ assertEquals(forwarded.partial_images, 1);
+ assertEquals(forwarded.model, 'gpt-image-2-upstream');
+});
+
+test('/v1/images/generations answers a non-streaming request with the JSON object', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await registerOpenAIImagesModel(repo);
+
+ await withMockedFetch(
+ request => {
+ const url = new URL(request.url);
+ if (url.hostname === 'images.example.com' && url.pathname === '/v1/images/generations') {
+ return Promise.resolve(Response.json({ data: [{ b64_json: 'aGVsbG8=' }], usage: { input_tokens: 10, output_tokens: 50 } }));
+ }
+ throw new Error(`Unhandled fetch ${request.url}`);
+ },
+ async () => {
+ const response = await requestApp('/v1/images/generations', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
+ body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space', stream: false }),
+ });
+ assertEquals(response.status, 200);
+ assertEquals(response.headers.get('content-type'), 'application/json');
+ assertEquals(await response.json(), { data: [{ b64_json: 'aGVsbG8=' }], usage: { input_tokens: 10, output_tokens: 50 } });
+ },
+ );
+});
+
+test('/v1/images/generations bills a stream from the usage its completed event reported, once', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await registerOpenAIImagesModel(repo);
+
+ await withMockedFetch(
+ () => Promise.resolve(sseResponse(PARTIAL + COMPLETED)),
+ async () => {
+ const response = await requestApp('/v1/images/generations', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
+ body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space', stream: true }),
+ });
+ assertEquals(response.status, 200);
+ await response.text();
+ },
+ );
+
+ await flushAsyncWork();
+ const rows = await repo.usage.listAll();
+ assertEquals(rows.length, 1);
+ // The numbers arrive with the completed event, long after the run answered, so a row that
+ // carries them is a row settled from the promise the run handed up. Settling in the stage as
+ // well would have counted the request twice.
+ assertEquals(rows[0]?.requests, 1);
+ assertEquals(rows[0]?.metrics.map(row => ({ metric: row.metric, quantity: row.quantity })), [
+ { metric: 'input_tokens', quantity: '10' },
+ { metric: 'input_image_tokens', quantity: '40' },
+ { metric: 'output_tokens', quantity: '50' },
+ ]);
+ const [performance] = await repo.performance.listAll();
+ assertEquals(performance?.requests, 1);
+});
+
+test('/v1/images/generations answers a refused stream in the upstream status and words', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await registerOpenAIImagesModel(repo);
+
+ await withMockedFetch(
+ () => Promise.resolve(new Response(JSON.stringify({ error: { message: 'Rate limit reached for images.', code: 'rate_limit_exceeded' } }), {
+ status: 429,
+ headers: { 'content-type': 'application/json', 'retry-after': '7', 'x-error-trace': 'trace-refused' },
+ })),
+ async () => {
+ const response = await requestApp('/v1/images/generations', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
+ body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space', stream: true }),
+ });
+ // A request that asked to stream but was refused is answered as the refusal it is.
+ assertEquals(response.status, 429);
+ assertEquals(response.headers.get('content-type'), 'application/json');
+ assertEquals(response.headers.get('retry-after'), '7');
+ assertEquals(response.headers.get('x-error-trace'), 'trace-refused');
+ assertEquals(await response.json(), { error: { message: 'Rate limit reached for images.', code: 'rate_limit_exceeded' } });
+ },
+ );
+
+ await flushAsyncWork();
+ const [performance] = await repo.performance.listAll();
+ assertEquals(performance?.errorsNoOutput, 1);
+});
+
+test('/v1/images/generations completes and cancels an upstream kept open after the completed event', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await registerOpenAIImagesModel(repo);
+ let upstreamCancelled = false;
+ const encoder = new TextEncoder();
+
+ await withMockedFetch(
+ () => Promise.resolve(new Response(new ReadableStream({
+ start(controller) {
+ controller.enqueue(encoder.encode(PARTIAL));
+ controller.enqueue(encoder.encode(COMPLETED));
+ // Deliberately never closed: the image is done and the connection is not.
+ },
+ cancel() {
+ upstreamCancelled = true;
+ },
+ }), { headers: { 'content-type': 'text/event-stream' } })),
+ async () => {
+ const response = await requestApp('/v1/images/generations', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
+ body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space', stream: true }),
+ });
+ const stream = await response.text();
+ assertEquals(stream.includes('event: image_generation.completed'), true);
+ },
+ );
+
+ assertEquals(upstreamCancelled, true);
+});
+
+test('/v1/images/generations fails a stream that ended without a completed event', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await registerOpenAIImagesModel(repo);
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+ try {
+ await withMockedFetch(
+ () => Promise.resolve(sseResponse(PARTIAL)),
+ async () => {
+ const response = await requestApp('/v1/images/generations', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
+ body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space', stream: true }),
+ });
+ // The answer had already begun, so what the client keeps is what it was already sent.
+ assertEquals(response.status, 200);
+ const stream = await response.text();
+ assertEquals(stream.includes('event: image_generation.partial_image'), true);
+ assertEquals(stream.includes('image_generation.completed'), false);
+ },
+ );
+ // An upstream that sent partial images and no image never answered the request, and the
+ // run says so rather than letting a truncated stream read as a complete one.
+ assertEquals(
+ errorSpy.mock.calls.some(call => call.some(arg => arg instanceof Error && arg.message === 'OpenAI Images stream ended without a completed event.')),
+ true,
+ );
+ } finally {
+ errorSpy.mockRestore();
+ }
+});
+
+test('/v1/images/edits streams a multipart request, whose stream field arrives as text', async () => {
+ const { apiKey, repo } = await setupAppTest();
+ await registerOpenAIImagesModel(repo);
+ let upstreamForm: FormData | undefined;
+
+ await withMockedFetch(
+ async request => {
+ const url = new URL(request.url);
+ if (url.hostname === 'images.example.com' && url.pathname === '/v1/images/edits') {
+ upstreamForm = await request.formData();
+ return sseResponse(
+ 'event: image_edit.partial_image\ndata: {"type":"image_edit.partial_image","b64_json":"UDA=","partial_image_index":0}\n\n'
+ + 'event: image_edit.completed\ndata: {"type":"image_edit.completed","b64_json":"RklO","usage":{"total_tokens":20,"input_tokens":8,"output_tokens":12,"input_tokens_details":{"text_tokens":3,"image_tokens":5}}}\n\n',
+ );
+ }
+ throw new Error(`Unhandled fetch ${request.url}`);
+ },
+ async () => {
+ const form = new FormData();
+ form.append('model', 'gpt-image-2');
+ form.append('prompt', 'replace the sky');
+ form.append('stream', 'true');
+ form.append('image', new Blob([new Uint8Array([1, 2, 3])], { type: 'image/png' }), 'photo.png');
+ const response = await requestApp('/v1/images/edits', {
+ method: 'POST', headers: { 'x-api-key': apiKey.key }, body: form,
+ });
+ assertEquals(response.status, 200);
+ assertEquals(response.headers.get('content-type'), 'text/event-stream');
+ const stream = await response.text();
+ assertEquals(stream.includes('event: image_edit.partial_image'), true);
+ assertEquals(stream.includes('event: image_edit.completed'), true);
+ },
+ );
+
+ assertExists(upstreamForm);
+ assertEquals(upstreamForm.get('stream'), 'true');
+
+ await flushAsyncWork();
+ const rows = await repo.usage.listAll();
+ assertEquals(rows.length, 1);
+ assertEquals(rows[0]?.metrics.map(row => ({ metric: row.metric, quantity: row.quantity })), [
+ { metric: 'input_tokens', quantity: '3' },
+ { metric: 'input_image_tokens', quantity: '5' },
+ { metric: 'output_tokens', quantity: '12' },
+ ]);
+});
diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts
new file mode 100644
index 0000000000..7d1fce498b
--- /dev/null
+++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts
@@ -0,0 +1,32 @@
+// Every family assembles. Assembly is a check over declarations, and a family whose
+// pipeline is built from the request assembles only when something builds one — so a
+// pipeline that could never run can sit in the tree looking fine. This test builds all of
+// them, which is what turns "the declarations disagree" into a failure at `pnpm test`
+// rather than on the first request that reaches the route.
+
+import { describe, expect, it } from 'vitest';
+
+import { searchServePipeline } from '../../../src/data-plane/alpha-search/pipeline.ts';
+import { openaiAudioTranscriptionServePipeline } from '../../../src/data-plane/openai-audio/pipeline.ts';
+import { openaiCompletionsServePipeline } from '../../../src/data-plane/openai-completions/pipeline.ts';
+import { openaiEmbeddingsServePipeline } from '../../../src/data-plane/openai-embeddings/pipeline.ts';
+import { openaiImagesServePipeline } from '../../../src/data-plane/openai-images/pipeline.ts';
+import { rerankServePipeline } from '../../../src/data-plane/rerank/pipeline.ts';
+
+const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] = [
+ ['OpenAI Embeddings', () => openaiEmbeddingsServePipeline],
+ ['rerank', () => rerankServePipeline({ sourceProtocol: 'cohere-v2', raw: {}, query: 'q', documents: ['a'] } as never)],
+ ['OpenAI Images Generations', () => openaiImagesServePipeline({ operation: 'generations', parameters: {} } as never)],
+ ['OpenAI Images Edits', () => openaiImagesServePipeline({ operation: 'edits', images: [], parameters: {} } as never)],
+ ['OpenAI Completions', () => openaiCompletionsServePipeline],
+ ['OpenAI Audio Transcriptions', () => openaiAudioTranscriptionServePipeline],
+ ['alpha search', () => searchServePipeline({ kind: 'search' } as never)],
+];
+
+describe('every family assembles', () => {
+ for (const [name, build] of FAMILIES) {
+ it(`${name} composes into a pipeline`, () => {
+ expect(build().name).toBeTypeOf('string');
+ });
+ }
+});
diff --git a/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts
new file mode 100644
index 0000000000..ec18347f71
--- /dev/null
+++ b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts
@@ -0,0 +1,88 @@
+// Rerank's pipeline, assembled. `compose` derives the entry contract and rejects an array
+// that cannot work, so most of what this file establishes is established by the assembly
+// succeeding at all — and what is worth writing down is the entry contract it derives,
+// including the one key it cannot see.
+
+import { describe, expect, it } from 'vitest';
+
+import { failover } from '../../src/data-plane/pipeline/stages.ts';
+import { rerankServePipeline } from '../../src/data-plane/rerank/pipeline.ts';
+import { move, run } from '@floway-dev/pipeline';
+import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank';
+
+const request: CanonicalRerankRequest = {
+ sourceProtocol: 'cohere-v2',
+ raw: {},
+ query: 'what is a pipeline',
+ documents: ['one', 'two'],
+};
+
+describe('the rerank pipeline', () => {
+ it('assembles, and asks its caller for what the descending stages need', () => {
+ expect([...rerankServePipeline(request).entryNeeds].sort()).toEqual([
+ 'ingress.rerank.sourceProtocol',
+ 'request.rerank.canonical',
+ 'serve.model',
+ ]);
+ });
+
+ // `callRerankUpstream` reads `ingress.http.headers`, and the entry contract does not
+ // mention it. That is not this family's defect: a stage whose only trait is `return`
+ // declares no request side at all, by ruling — "when it short-circuits, only `provides`"
+ // — so assembly cannot see what an ending stage reads, and every family's ending stage
+ // reads something.
+ //
+ // Written as a test rather than a comment because the hole has a consequence: a caller
+ // who omits that key gets a runtime failure at the deepest stage instead of an assembly
+ // error, and the entry contract exists to stop exactly that. The type layer still
+ // catches it at the definition site, which is why this is a gap and not a break.
+ it('cannot see what an ending stage reads, because a return-only stage declares no needs', () => {
+ expect(rerankServePipeline(request).entryNeeds).not.toContain('ingress.http.headers');
+ });
+
+ // `failover` used to declare that it provides `response.http.body` for every family. Three
+ // of the six never produce one — they read their answer to the end — so those pipelines
+ // composed cleanly and then threw on the first real request, at the deepest stage, with a
+ // message about a key their author had never written down.
+ //
+ // What a fork owns is a statement only the family can make, so it makes it. This asserts
+ // the shape of that statement rather than the absence of a bug, because the absence of a
+ // bug is what every one of these tests asserted before and none of them caught it.
+ it('claims nothing on the way up when a family reads its answer to the end', () => {
+ const reading = failover({ failed: () => false, owns: [] });
+ expect(reading.through?.response.consumes).toEqual([]);
+ expect(reading.through?.response.provides).toEqual([]);
+
+ // And a family that streams claims the key it streams at, in both directions: every
+ // attempt's is the fork's to release, and the one it adopts rides up with ownership.
+ const streaming = failover({ failed: () => false, owns: ['response.http.body'] });
+ expect(streaming.through?.response.consumes).toEqual(['response.http.body']);
+ expect(streaming.through?.response.provides).toEqual(['response.http.body']);
+ });
+
+ // A live handle is never a fact, and the test for that is whether it can be rendered into
+ // the dump. A `ModelCandidate` cannot: it carries the provider's instance, its fetcher and
+ // its models cache. Putting one in the record deep-freezes all three, and the SWR cache
+ // refresh the provider does on its own schedule then breaks — throwing under a module,
+ // which every file here is, and failing silently anywhere that is not strict.
+ it('carries a selector, so freezing the record cannot reach a live handle', () => {
+ const instance = { cache: null as unknown };
+ const candidate = { provider: { upstreamId: 'u', instance, modelsCache: { at: 1 } } };
+
+ // What the record actually holds: data, and nothing that answers to a call.
+ move({ 'route.attempt': { upstreamId: 'u', modelId: 'm', flags: [] } });
+ expect(Object.isFrozen(instance)).toBe(false);
+ expect(Object.isFrozen(candidate.provider.modelsCache)).toBe(false);
+
+ // And what putting the candidate itself there would have done, so the difference is not
+ // hypothetical — this is the shape the six families carried until the selector split.
+ move({ 'route.candidate': candidate } as never);
+ expect(Object.isFrozen(instance)).toBe(true);
+ expect(() => { instance.cache = { refreshed: true }; }).toThrow(TypeError);
+ });
+
+ it('names the entry key a caller did not bring, before any stage runs', async () => {
+ await expect(run(rerankServePipeline(request), move({ 'serve.model': 'rerank-v3' }) as never, {}))
+ .rejects.toThrow('run(rerankServe): rerankServe needs');
+ });
+});
diff --git a/packages/gateway/__tests__/data-plane/rerank-status_test.ts b/packages/gateway/__tests__/data-plane/rerank-status_test.ts
new file mode 100644
index 0000000000..0294b84032
--- /dev/null
+++ b/packages/gateway/__tests__/data-plane/rerank-status_test.ts
@@ -0,0 +1,104 @@
+// The two families that read their answer to the end, driven end to end. Until the review,
+// neither could express a status at all: an upstream 429, a resolver's 404 and a 400 all
+// reached the client as a 200 carrying an error envelope, which is not a difference the
+// no-passthrough ruling asks for — declining to forward a body is not declining to forward
+// a status.
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts';
+import { rerankServePipeline } from '../../src/data-plane/rerank/pipeline.ts';
+import { initRepo } from '../../src/repo/index.ts';
+import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts';
+import { move, run } from '@floway-dev/pipeline';
+import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank';
+import { directFetcher, type ModelCandidate, type ProviderRerankCallResult } from '@floway-dev/provider';
+import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils';
+
+vi.mock('../../src/data-plane/providers/resolution.ts', () => ({
+ enumerateModelCandidates: vi.fn(),
+}));
+
+let live: readonly ModelCandidate[] = [];
+
+const candidate = (upstream: string, callRerank: () => Promise): ModelCandidate => {
+ const endpoints = { rerank: {} };
+ return {
+ provider: {
+ upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [],
+ disabledPublicModelIds: [], modelPrefix: null, modelsCache: null,
+ instance: stubProvider({ callRerank }),
+ },
+ model: stubInternalModel(
+ { id: 'rr', endpoints, providerModels: { [upstream]: stubProviderModel({ id: 'rr', endpoints, rerankTarget: { protocol: 'cohere-v2' } }) } },
+ upstream,
+ ),
+ fetcher: directFetcher,
+ } as unknown as ModelCandidate;
+};
+
+const resolves = (candidates: readonly ModelCandidate[]): void => {
+ live = candidates;
+ vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never);
+};
+
+const request: CanonicalRerankRequest = {
+ sourceProtocol: 'cohere-v2', raw: {}, query: 'q', documents: ['a', 'b'],
+};
+
+const serve = async () => await run(
+ rerankServePipeline(request),
+ move({
+ 'ingress.rerank.sourceProtocol': 'cohere-v2',
+ 'ingress.http.headers': [] as readonly (readonly [string, string])[],
+ 'request.rerank.canonical': request,
+ 'serve.model': 'rr',
+ }) as never,
+ {
+ gateway: mockGatewayCtx({ wantsStream: false }),
+ background: () => {},
+ rememberCandidates: () => {},
+ resolveAttempt: (selector: { readonly upstreamId: string }) => {
+ const found = live.find(c => c.provider.upstreamId === selector.upstreamId);
+ if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`);
+ return found;
+ },
+ } as never,
+);
+
+beforeEach(() => {
+ initRepo({
+ usage: { record: async () => {} },
+ performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} },
+ } as never);
+});
+
+describe('a rerank answer carries its status', () => {
+ it('serves a success as 200', async () => {
+ resolves([candidate('up_a', async () => ({
+ response: Response.json({ results: [{ index: 0, relevance_score: 0.9 }], meta: { billed_units: { search_units: 1 } } }),
+ modelKey: 'rr-key',
+ target: { protocol: 'cohere-v2' },
+ } as ProviderRerankCallResult))]);
+ const { facts } = await serve();
+ expect(facts['response.http.status']).toBe(200);
+ });
+
+ // The case that reached the client as a 200 before: a refusal the client has to be able
+ // to act on, and a retry-after only means something alongside the status that implies it.
+ it('serves an upstream refusal with the upstream-s own status', async () => {
+ resolves([candidate('up_a', async () => ({
+ response: new Response('slow down', { status: 429 }),
+ modelKey: 'rr-key',
+ target: { protocol: 'cohere-v2' },
+ } as ProviderRerankCallResult))]);
+ const { facts } = await serve();
+ expect(facts['response.http.status']).toBe(429);
+ });
+
+ it('serves the resolver-s own refusal with the status it chose', async () => {
+ vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates: [], sawModel: false, failedUpstreams: [] } as never);
+ const { facts } = await serve();
+ expect(facts['response.http.status']).toBe(404);
+ });
+});
diff --git a/packages/gateway/__tests__/data-plane/rerank/serve_test.ts b/packages/gateway/__tests__/data-plane/rerank/serve_test.ts
index ef4e067cc6..b1bd422dbc 100644
--- a/packages/gateway/__tests__/data-plane/rerank/serve_test.ts
+++ b/packages/gateway/__tests__/data-plane/rerank/serve_test.ts
@@ -516,7 +516,10 @@ test('cross-protocol success still validates result items before rendering', asy
assertEquals(performance.errorsNoOutput, 1);
});
-test('same-protocol malformed JSON is forwarded as request-only usage', async () => {
+// Rerank is a JSON protocol, so a body that is not JSON at all is no answer to serve and the
+// gateway says so itself rather than passing on something it never read. Forwarding those
+// bytes verbatim under the upstream's 200 would report success for a turn nothing could read.
+test('same-protocol answer that is not JSON is refused rather than forwarded', async () => {
const { apiKey, repo } = await setupAppTest();
await saveRerankUpstream(repo, { protocol: 'jina-v1' });
@@ -528,12 +531,15 @@ test('same-protocol malformed JSON is forwarded as request-only usage', async ()
headers: requestHeaders(apiKey.key),
body: JSON.stringify({ model: 'public-reranker', query: 'query', documents: ['one'] }),
});
- assertEquals(response.status, 200);
- assertEquals(await response.text(), '{not-json');
+ assertEquals(response.status, 502);
+ const body = await response.json() as { error: { message: string } };
+ assertEquals(body.error.message.includes('the rerank protocol cannot read'), true);
},
);
await flushAsyncWork();
+ // Called, and reported nothing this reader could make sense of — which still counts as a
+ // request against the upstream that was dialled.
const [usage] = await repo.usage.listAll();
assertEquals(usage.requests, 1);
assertEquals(usage.metrics, []);
diff --git a/packages/gateway/__tests__/data-plane/search-pipeline_test.ts b/packages/gateway/__tests__/data-plane/search-pipeline_test.ts
new file mode 100644
index 0000000000..924fc51495
--- /dev/null
+++ b/packages/gateway/__tests__/data-plane/search-pipeline_test.ts
@@ -0,0 +1,60 @@
+// Search's pipeline, assembled. `compose` derives the entry contract and rejects an array
+// that cannot work, so most of what this file establishes is established by the assembly
+// succeeding at all — and what is worth writing down is the two contracts it derives, one per
+// ending, plus the one it cannot derive at all.
+
+import { describe, expect, it } from 'vitest';
+
+import { searchServePipeline } from '../../src/data-plane/alpha-search/pipeline.ts';
+import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts';
+import { move, run } from '@floway-dev/pipeline';
+
+const pinned = { kind: 'upstream', upstreamId: 'up_alpha', model: 'gpt-search' } as const;
+
+describe('the search pipeline', () => {
+ it('assembles the local ending, and asks its caller for what the descending stages need', () => {
+ expect([...searchServePipeline({ kind: 'local' }).entryNeeds].sort()).toEqual([
+ 'request.search.alphaSearch',
+ ]);
+ });
+
+ // The pinned ending is one `return`-only stage, and such a stage declares no request side
+ // at all — by ruling, when it short-circuits there is only `provides`. So assembly sees
+ // nothing it reads and derives an empty contract, even though the stage cannot run without
+ // `request.search.alphaSearch` and `ingress.http.headers`. It is the same hole every
+ // family's ending stage has; here it swallows the family's whole entry contract, because
+ // the ending is the only stage below the edge.
+ //
+ // Written as a test rather than a comment because the hole has a consequence: a caller who
+ // omits either key gets a runtime failure at the deepest stage instead of an assembly
+ // error, and the entry contract exists to stop exactly that. The type layer still catches
+ // it at the definition site, which is why this is a gap and not a break.
+ it('cannot see what the pinned ending reads, because a return-only stage declares no needs', () => {
+ expect(searchServePipeline(pinned).entryNeeds).toEqual([]);
+ });
+
+ it('names the entry key a caller did not bring, before any stage runs', async () => {
+ await expect(run(searchServePipeline({ kind: 'local' }), move({}) as never, {}))
+ .rejects.toThrow('run(searchServe): searchServe needs request.search.alphaSearch');
+ });
+
+ // A request with nothing to run reaches no backend at all: the parse stage answers instead
+ // of descending, and what the assembly guarantees — that a short-circuit covers what the
+ // edge needs — is what makes the run produce a rendered body, a status and a billed set
+ // anyway. Settlement is unconditional, so the run still writes; a search that ran locally
+ // simply names no billed entity.
+ it('answers in band when there is nothing to run, and the edge renders that answer', async () => {
+ const { facts } = await run(
+ searchServePipeline({ kind: 'local' }),
+ move({ 'request.search.alphaSearch': { commands: {} } }),
+ { gateway: mockGatewayCtx({ wantsStream: false }), background: () => {} } as never,
+ );
+ expect(facts['response.http.status']).toBe(200);
+ expect(facts['response.search.rendered']).toEqual({
+ encrypted_output: null,
+ output: 'No web search commands were provided. Populate at least one of `search_query`, `open`, or `find`.',
+ });
+ // Nothing was called that a model prices, and an empty set is how that is said.
+ expect(facts['response.usage.billable']).toEqual([]);
+ });
+});
diff --git a/packages/gateway/__tests__/data-plane/shared/custom-ingress-header-rules_test.ts b/packages/gateway/__tests__/data-plane/shared/custom-ingress-header-rules_test.ts
index 7d4f37ec33..d88e4551ad 100644
--- a/packages/gateway/__tests__/data-plane/shared/custom-ingress-header-rules_test.ts
+++ b/packages/gateway/__tests__/data-plane/shared/custom-ingress-header-rules_test.ts
@@ -152,7 +152,11 @@ const CASES: RouteCase[] = [
name: '/v1/completions reaches a Completions upstream',
path: '/v1/completions',
contentType: 'application/json',
- body: () => JSON.stringify({ model: 'completions-model', prompt: 'hi' }),
+ // Asks to stream, because the upstream below answers with one. A
+ // non-streaming request whose upstream replies `text/event-stream` has no
+ // body this protocol can read, and the pipeline answers 502 rather than
+ // serving something it never parsed.
+ body: () => JSON.stringify({ model: 'completions-model', prompt: 'hi', stream: true }),
upstreamPath: '/v1/completions',
upstreamResponse: () => new Response(
'data: {"id":"cmpl_1","object":"text_completion","created":1,"model":"completions-model","choices":[{"index":0,"text":"ok","finish_reason":"stop"}]}\n\ndata: [DONE]\n\n',
diff --git a/packages/gateway/__tests__/data-plane/shared/passthrough-attempt_test.ts b/packages/gateway/__tests__/data-plane/shared/passthrough-attempt_test.ts
deleted file mode 100644
index b4e51d530c..0000000000
--- a/packages/gateway/__tests__/data-plane/shared/passthrough-attempt_test.ts
+++ /dev/null
@@ -1,118 +0,0 @@
-import { Hono } from 'hono';
-import { test } from 'vitest';
-
-import { passthroughAttempt, type PassthroughAttemptResult } from '../../../src/data-plane/shared/passthrough-attempt.ts';
-import type { AuthVars } from '../../../src/middleware/auth.ts';
-import { mockGatewayCtx } from '../../test-utils/gateway-ctx.ts';
-import { assertEquals, assertExists, stubModelCandidate, stubProvider } from '@floway-dev/test-utils';
-
-test('passthroughAttempt applies the selected provider ingress policy', async () => {
- let observed: Headers | undefined;
- const base = stubModelCandidate();
- const candidate = stubModelCandidate({
- provider: {
- ...base.provider,
- kind: 'custom',
- instance: stubProvider({
- callOpenAIEmbeddings: async (_model, _body, _signal, opts) => {
- observed = opts.headers;
- return { response: new Response('{}'), modelKey: 'test-model' };
- },
- }),
- },
- });
- const app = new Hono<{ Variables: AuthVars }>();
- app.post('/test', async c => {
- await passthroughAttempt({
- c,
- ctx: mockGatewayCtx(),
- candidate,
- operation: 'embeddings',
- call: (provider, model, opts) => provider.instance.callOpenAIEmbeddings(model, { input: 'hi' }, undefined, opts),
- });
- return c.text('ok');
- });
- await app.request('/test', {
- method: 'POST',
- headers: {
- authorization: 'Bearer secret',
- 'x-client-request-id': 'request-1',
- 'x-debug': 'discard',
- },
- });
-
- assertExists(observed);
- assertEquals([...observed], []);
-});
-
-// Drives one attempt against a canned upstream response, through the same Hono
-// context the serve layer builds.
-const runPassthroughAttempt = async (upstream: Response): Promise => {
- const base = stubModelCandidate();
- const candidate = stubModelCandidate({
- provider: {
- ...base.provider,
- kind: 'custom',
- instance: stubProvider({
- callOpenAIEmbeddings: async () => ({ response: upstream, modelKey: 'test-model' }),
- }),
- },
- });
- let result: PassthroughAttemptResult | undefined;
- const app = new Hono<{ Variables: AuthVars }>();
- app.post('/test', async c => {
- result = await passthroughAttempt({
- c,
- ctx: mockGatewayCtx(),
- candidate,
- operation: 'embeddings',
- call: (provider, model, opts) => provider.instance.callOpenAIEmbeddings(model, { input: 'hi' }, undefined, opts),
- });
- return c.text('ok');
- });
- await app.request('/test', { method: 'POST' });
- assertExists(result);
- return result;
-};
-
-// The fallback loop keeps only the most recent failure, so every superseded
-// attempt is dropped. On the direct-connect egress a dropped response strands
-// its socket, which is why a failed attempt must not carry a live body.
-test('passthroughAttempt materializes a failed upstream response so a discarded attempt holds no transport', async () => {
- let sourceCancelled = false;
- let sourceRead = false;
- const upstream = new Response(
- new ReadableStream({
- start(controller) {
- sourceRead = true;
- controller.enqueue(new TextEncoder().encode('{"error":"upstream busy"}'));
- controller.close();
- },
- cancel() { sourceCancelled = true; },
- }),
- { status: 503, statusText: 'Service Unavailable', headers: { 'content-type': 'application/json', 'retry-after': '12' } },
- );
-
- const result = await runPassthroughAttempt(upstream);
-
- assertEquals(result.status, 503);
- // The upstream body was consumed rather than left for someone else to close.
- assertEquals(sourceRead, true);
- assertEquals(sourceCancelled, false);
- assertEquals(upstream.bodyUsed, true);
- // Status, headers and bytes still forward verbatim.
- assertEquals(result.response.status, 503);
- assertEquals(result.response.statusText, 'Service Unavailable');
- assertEquals(result.response.headers.get('retry-after'), '12');
- assertEquals(await result.response.text(), '{"error":"upstream busy"}');
-});
-
-test('passthroughAttempt forwards a successful upstream response without reading it', async () => {
- const upstream = new Response('ok', { status: 200 });
- const result = await runPassthroughAttempt(upstream);
-
- // A 2xx is returned to the caller immediately and is never discarded, so it
- // must keep streaming rather than be buffered here.
- assertEquals(result.response, upstream);
- assertEquals(upstream.bodyUsed, false);
-});
diff --git a/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts b/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts
deleted file mode 100644
index 311a94ade8..0000000000
--- a/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts
+++ /dev/null
@@ -1,435 +0,0 @@
-// Behavioral coverage for the shared passthrough serve scaffold used by
-// /v1/embeddings, /v1/images/{generations,edits}, and audio transcription. Each test exercises a
-// full client request through the in-memory app rather than constructing a
-// synthetic hono Context so the integration with model resolution,
-// upstream HTTP, and background scheduling stays honest.
-//
-// We pick /v1/embeddings as the source under test because:
-// - its acceptBinding gate is `kind === 'embedding'`, satisfied by any
-// embedding-only custom upstream, with no per-endpoint Copilot setup
-// required;
-// - its extractBilling reads the OpenAI-style `usage.prompt_tokens` off
-// a 2xx JSON body, so a body with that shape triggers a real usage
-// write;
-// - it shares the exact same forwardUpstreamResponse + settle
-// path as the images endpoints — the behaviors under test are owned by
-// passthroughServe, not the endpoint shape.
-
-import { test, vi } from 'vitest';
-
-import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts';
-import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot';
-import { jsonResponse, withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils';
-
-const registerOpenAIEmbeddingsUpstream = async (
- repo: Awaited>['repo'],
- ingressHeadersRules: { key: string; value: string | null }[] = [],
-): Promise => {
- await repo.upstreams.deleteAll();
- clearInProcessCopilotTokenCache();
- await repo.upstreams.save(buildCustomUpstreamRecord({
- id: 'up_passthrough',
- name: 'Passthrough Embedding Provider',
- sortOrder: 100,
- config: {
- baseUrl: 'https://passthrough.example.com',
- authStyle: 'bearer',
- ingressHeadersRules,
- apiKey: 'sk-passthrough',
- endpoints: {},
- },
- }));
-};
-
-test('passthrough-serve: usage-record failure does not turn upstream 2xx into 502', async () => {
- const { apiKey, repo } = await setupAppTest();
- await registerOpenAIEmbeddingsUpstream(repo);
-
- repo.usage.record = () => Promise.reject(new Error('simulated SQL write failure'));
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
-
- try {
- await withMockedFetch(
- request => {
- const url = new URL(request.url);
- if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/models') {
- return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] });
- }
- if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/embeddings') {
- assertEquals(request.headers.get('anthropic-beta'), null);
- assertEquals(request.headers.get('x-client-request-id'), null);
- assertEquals(request.headers.get('x-debug'), null);
- return jsonResponse({
- object: 'list',
- model: 'custom-embed-model',
- data: [{ object: 'embedding', index: 0, embedding: [0.5] }],
- usage: { prompt_tokens: 3, total_tokens: 3 },
- });
- }
- throw new Error(`Unhandled fetch ${request.url}`);
- },
- async () => {
- const response = await requestApp('/v1/embeddings', {
- method: 'POST',
- headers: {
- 'content-type': 'application/json',
- 'x-api-key': apiKey.key,
- 'anthropic-beta': 'context-1m-2025-08-07',
- 'x-client-request-id': 'request-1',
- 'x-debug': 'discard',
- },
- body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }),
- });
-
- assertEquals(response.status, 200);
- const body = await response.json() as { data: Array<{ embedding: number[] }> };
- assertEquals(body.data[0].embedding, [0.5]);
- await flushAsyncWork();
- },
- );
-
- assertEquals(errorSpy.mock.calls.some(call => call[0] === 'Failed to record usage:'), true);
- } finally {
- errorSpy.mockRestore();
- }
-});
-
-test('passthrough-serve: Custom resolves configured ingress header rules before provider dispatch', async () => {
- const { apiKey, repo } = await setupAppTest();
- await registerOpenAIEmbeddingsUpstream(repo, [
- { key: 'x-passthrough', value: null },
- { key: 'x-overwrite', value: 'configured' },
- { key: 'x-empty', value: '' },
- ]);
-
- await withMockedFetch(
- request => {
- const url = new URL(request.url);
- if (url.pathname === '/v1/models') {
- return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] });
- }
- if (url.pathname === '/v1/embeddings') {
- assertEquals(request.headers.get('x-passthrough'), 'client');
- assertEquals(request.headers.get('x-overwrite'), 'configured');
- assertEquals(request.headers.get('x-empty'), '');
- assertEquals(request.headers.get('authorization'), 'Bearer sk-passthrough');
- assertEquals(request.headers.get('x-unlisted'), null);
- return jsonResponse({ object: 'list', data: [], model: 'custom-embed-model' });
- }
- throw new Error(`Unhandled fetch ${request.url}`);
- },
- async () => {
- const response = await requestApp('/v1/embeddings', {
- method: 'POST',
- headers: {
- 'content-type': 'application/json',
- 'x-api-key': apiKey.key,
- 'x-empty': 'client',
- 'x-overwrite': 'client',
- 'x-passthrough': 'client',
- 'x-unlisted': 'discard',
- },
- body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }),
- });
- assertEquals(response.status, 200);
- },
- );
-});
-
-test('passthrough-serve: non-JSON 2xx upstream body is forwarded verbatim with a request-only usage record', async () => {
- const { apiKey, repo } = await setupAppTest();
- await registerOpenAIEmbeddingsUpstream(repo);
-
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
-
- try {
- const binary = new Uint8Array([0xde, 0xad, 0xbe, 0xef]);
- await withMockedFetch(
- request => {
- const url = new URL(request.url);
- if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/models') {
- return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] });
- }
- if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/embeddings') {
- return new Response(binary, {
- status: 200,
- headers: { 'content-type': 'application/octet-stream' },
- });
- }
- throw new Error(`Unhandled fetch ${request.url}`);
- },
- async () => {
- const response = await requestApp('/v1/embeddings', {
- method: 'POST',
- headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
- body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }),
- });
-
- assertEquals(response.status, 200);
- assertEquals(response.headers.get('content-type'), 'application/octet-stream');
- const bytes = new Uint8Array(await response.arrayBuffer());
- assertEquals(Array.from(bytes), Array.from(binary));
- await flushAsyncWork();
- },
- );
-
- const usage = await repo.usage.listAll();
- assertEquals(usage.length, 1);
- assertEquals(usage[0].requests, 1);
- assertEquals(usage[0].metrics, []);
- // The parse failure is observable through console.warn so operators can
- // correlate missing usage rows against upstream body shape regressions.
- assertEquals(warnSpy.mock.calls.some(call => typeof call[0] === 'string' && call[0].includes('passthrough-serve: failed to parse 2xx upstream body for /embeddings')), true);
- } finally {
- warnSpy.mockRestore();
- }
-});
-
-test('passthrough-serve: response header blocklist preserves vendor metadata and drops unsafe headers', async () => {
- const { apiKey, repo } = await setupAppTest();
- await registerOpenAIEmbeddingsUpstream(repo);
-
- await withMockedFetch(
- request => {
- const url = new URL(request.url);
- if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/models') {
- return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] });
- }
- if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/embeddings') {
- return new Response(JSON.stringify({
- object: 'list',
- model: 'custom-embed-model',
- data: [{ object: 'embedding', index: 0, embedding: [0.1] }],
- usage: { prompt_tokens: 1, total_tokens: 1 },
- }), {
- status: 200,
- headers: {
- 'content-type': 'application/json',
- 'x-request-id': 'req-123',
- 'openai-organization': 'org-1',
- 'x-ratelimit-remaining': '100',
- 'retry-after': '30',
- 'cf-ray': 'abc',
- 'x-internal-secret': 'leak',
- 'set-cookie': 'nope=1',
- },
- });
- }
- throw new Error(`Unhandled fetch ${request.url}`);
- },
- async () => {
- const response = await requestApp('/v1/embeddings', {
- method: 'POST',
- headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
- body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }),
- });
-
- assertEquals(response.status, 200);
- assertExists(response.headers.get('x-request-id'));
- assertEquals(response.headers.get('x-request-id'), 'req-123');
- assertEquals(response.headers.get('openai-organization'), 'org-1');
- assertEquals(response.headers.get('x-ratelimit-remaining'), '100');
- assertEquals(response.headers.get('retry-after'), '30');
- assertEquals(response.headers.get('cf-ray'), 'abc');
- assertEquals(response.headers.get('x-internal-secret'), 'leak');
- assertEquals(response.headers.get('set-cookie'), null);
- await response.json();
- },
- );
-});
-
-test('passthrough-serve: alias whose targets have no kind-matching binding surfaces as the regular model-missing 404', async () => {
- // The inlined alias resolver walks alias targets in `selection` order and
- // stops at the first target with kind-matching candidates. When every
- // target is unroutable (as here, where the single target id doesn't
- // exist in any upstream catalog), the resolver returns empty candidates
- // + sawModel=false, and the passthrough seam surfaces the regular
- // model-missing 404. No upstream call should fire.
- const { apiKey, repo } = await setupAppTest();
- await registerOpenAIEmbeddingsUpstream(repo);
- await repo.modelAliases.insert({
- id: 'alias_embed-fast',
- name: 'embed-fast',
- kind: 'embedding',
- selection: 'first-available',
- displayName: null,
- visibleInModelsList: true,
- targets: [{ target_model_id: 'unknown-embed', rules: {} }],
- announcedMetadata: null,
- sortOrder: 0,
- createdAt: '2026-01-01T00:00:00.000Z',
- updatedAt: '2026-01-01T00:00:00.000Z',
- });
-
- await withMockedFetch(
- request => {
- const url = new URL(request.url);
- if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/models') {
- return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] });
- }
- if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/embeddings') {
- throw new Error('passthrough-serve: upstream must not be called when alias has no routable target');
- }
- throw new Error(`Unhandled fetch ${request.url}`);
- },
- async () => {
- const response = await requestApp('/v1/embeddings', {
- method: 'POST',
- headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
- body: JSON.stringify({ model: 'embed-fast', input: 'hi' }),
- });
-
- assertEquals(response.status, 404);
- const body = await response.json() as { error: { message: string; type: string } };
- assertEquals(body.error.type, 'api_error');
- // The alias name (still on `payload.model` because no candidate was
- // rewritten in) reaches the wording verbatim.
- assertEquals(body.error.message, 'Model embed-fast is not available on any configured upstream.');
- },
- );
-});
-
-// Register two custom upstreams both exposing the same embedding model, so
-// the shared narrow phase produces a two-element candidate list ordered by
-// `sortOrder`. The passthrough loop must try `up_a` first (sortOrder 100)
-// and `up_b` second (sortOrder 200).
-const registerTwoOpenAIEmbeddingsUpstreams = async (repo: Awaited>['repo']): Promise => {
- await repo.upstreams.deleteAll();
- clearInProcessCopilotTokenCache();
- await repo.upstreams.save(buildCustomUpstreamRecord({
- id: 'up_a', name: 'Upstream A', sortOrder: 100,
- config: { baseUrl: 'https://up-a.example.com', authStyle: 'bearer', apiKey: 'sk-a', endpoints: {}, ingressHeadersRules: [] },
- }));
- await repo.upstreams.save(buildCustomUpstreamRecord({
- id: 'up_b', name: 'Upstream B', sortOrder: 200,
- config: { baseUrl: 'https://up-b.example.com', authStyle: 'bearer', apiKey: 'sk-b', endpoints: {}, ingressHeadersRules: [] },
- }));
-};
-
-test('passthrough-serve: 5xx from the first candidate falls through to the next successful upstream', async () => {
- const { apiKey, repo } = await setupAppTest();
- await registerTwoOpenAIEmbeddingsUpstreams(repo);
-
- let firstOpenAIEmbeddingsCalls = 0;
- let secondOpenAIEmbeddingsCalls = 0;
- await withMockedFetch(
- request => {
- const url = new URL(request.url);
- if (url.pathname === '/v1/models') {
- return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] });
- }
- if (url.hostname === 'up-a.example.com' && url.pathname === '/v1/embeddings') {
- firstOpenAIEmbeddingsCalls += 1;
- return new Response('upstream boom', { status: 503 });
- }
- if (url.hostname === 'up-b.example.com' && url.pathname === '/v1/embeddings') {
- secondOpenAIEmbeddingsCalls += 1;
- return jsonResponse({
- object: 'list',
- model: 'custom-embed-model',
- data: [{ object: 'embedding', index: 0, embedding: [0.25] }],
- usage: { prompt_tokens: 2, total_tokens: 2 },
- });
- }
- throw new Error(`Unhandled fetch ${request.url}`);
- },
- async () => {
- const response = await requestApp('/v1/embeddings', {
- method: 'POST',
- headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
- body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }),
- });
-
- assertEquals(response.status, 200);
- const body = await response.json() as { data: Array<{ embedding: number[] }> };
- assertEquals(body.data[0].embedding, [0.25]);
- await flushAsyncWork();
- },
- );
-
- assertEquals(firstOpenAIEmbeddingsCalls, 1);
- assertEquals(secondOpenAIEmbeddingsCalls, 1);
-});
-
-test('passthrough-serve: when every candidate returns non-2xx the most recent upstream response is forwarded verbatim', async () => {
- const { apiKey, repo } = await setupAppTest();
- await registerTwoOpenAIEmbeddingsUpstreams(repo);
-
- await withMockedFetch(
- request => {
- const url = new URL(request.url);
- if (url.pathname === '/v1/models') {
- return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] });
- }
- if (url.hostname === 'up-a.example.com' && url.pathname === '/v1/embeddings') {
- return new Response('first upstream unavailable', { status: 503 });
- }
- if (url.hostname === 'up-b.example.com' && url.pathname === '/v1/embeddings') {
- return new Response(JSON.stringify({ error: { message: 'rate limited' } }), {
- status: 429, headers: { 'content-type': 'application/json', 'retry-after': '17' },
- });
- }
- throw new Error(`Unhandled fetch ${request.url}`);
- },
- async () => {
- const response = await requestApp('/v1/embeddings', {
- method: 'POST',
- headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
- body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }),
- });
-
- // The last-attempted upstream's status, body, and allow-listed
- // headers pass through unchanged so clients see real upstream
- // telemetry — no synthetic gateway envelope.
- assertEquals(response.status, 429);
- assertEquals(response.headers.get('retry-after'), '17');
- const body = await response.json() as { error: { message: string } };
- assertEquals(body.error.message, 'rate limited');
- await flushAsyncWork();
- },
- );
-});
-
-// A throw during candidate rollover attributes the error row to the
-// throwing candidate, not the previously-succeeded one.
-test('passthrough-serve: throw during rollover attributes the error perf row to the throwing candidate, not the previous one', async () => {
- const { apiKey, repo } = await setupAppTest();
- await registerTwoOpenAIEmbeddingsUpstreams(repo);
-
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
-
- try {
- await withMockedFetch(
- request => {
- const url = new URL(request.url);
- if (url.pathname === '/v1/models') {
- return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] });
- }
- if (url.hostname === 'up-a.example.com' && url.pathname === '/v1/embeddings') {
- return new Response('first upstream unavailable', { status: 503 });
- }
- if (url.hostname === 'up-b.example.com' && url.pathname === '/v1/embeddings') {
- throw new Error('simulated network error to up_b');
- }
- throw new Error(`Unhandled fetch ${request.url}`);
- },
- async () => {
- const response = await requestApp('/v1/embeddings', {
- method: 'POST',
- headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key },
- body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }),
- });
-
- assertEquals(response.status, 502);
- await flushAsyncWork();
- },
- );
-
- const perfRows = await repo.performance.listAll();
- const errorRows = perfRows.filter(row => row.errorsNoOutput + row.errorsWithOutput > 0);
- assertEquals(errorRows.length, 1);
- assertEquals(errorRows[0].upstream, 'up_b');
- } finally {
- errorSpy.mockRestore();
- }
-});
diff --git a/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/relay-response_test.ts b/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/relay-response_test.ts
deleted file mode 100644
index 5e601b11e6..0000000000
--- a/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/relay-response_test.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-import { describe, expect, it } from 'vitest';
-
-import { relayFetchedResponse } from '../../../../../src/data-plane/tools/web-search/alpha-search/relay-response.ts';
-
-describe('relayFetchedResponse', () => {
- it('keeps decoded bytes and representation-safe upstream headers', async () => {
- const body = JSON.stringify({ output: 'decoded' });
- const upstream = new Response(body, {
- status: 202,
- statusText: 'Accepted upstream',
- headers: {
- connection: 'close',
- 'content-encoding': 'gzip',
- 'content-length': '999',
- 'content-type': 'application/json',
- 'set-cookie': 'session=upstream-secret',
- 'transfer-encoding': 'chunked',
- 'x-oai-request-id': 'req_search_1',
- },
- });
-
- const relayed = relayFetchedResponse(upstream);
-
- expect(relayed.status).toBe(202);
- expect(relayed.statusText).toBe('Accepted upstream');
- expect(await relayed.text()).toBe(body);
- expect(relayed.headers.get('content-type')).toBe('application/json');
- expect(relayed.headers.get('x-oai-request-id')).toBe('req_search_1');
- expect(relayed.headers.get('connection')).toBeNull();
- expect(relayed.headers.get('content-encoding')).toBeNull();
- expect(relayed.headers.get('content-length')).toBeNull();
- expect(relayed.headers.get('set-cookie')).toBeNull();
- expect(relayed.headers.get('transfer-encoding')).toBeNull();
- });
-});
diff --git a/packages/gateway/__tests__/dump/run-sink_test.ts b/packages/gateway/__tests__/dump/run-sink_test.ts
new file mode 100644
index 0000000000..25bc74bf4d
--- /dev/null
+++ b/packages/gateway/__tests__/dump/run-sink_test.ts
@@ -0,0 +1,168 @@
+import { test } from 'vitest';
+
+import { installDumpStubs } from './test-fixtures.ts';
+import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts';
+import { openRunDump } from '../../src/dump/run-sink.ts';
+import type { StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts';
+import type { ApiKey } from '../../src/repo/types.ts';
+import { flushBackground, trackBackground } from '../test-utils/background-tracker.ts';
+import { compose, defineStage, move, run, type DumpEvent, type Event } from '@floway-dev/pipeline';
+import { assertEquals } from '@floway-dev/test-utils';
+
+const apiKey = (dumpRetentionSeconds: number | null): ApiKey => ({
+ id: 'key_run',
+ userId: 1,
+ name: 'Run key',
+ key: 'raw-run-key',
+ serverSecret: '11'.repeat(32),
+ createdAt: '2026-01-01T00:00:00.000Z',
+ upstreamIds: null,
+ deletedAt: null,
+ dumpRetentionSeconds,
+ openaiResponsesRetentionSeconds: 0,
+});
+
+const requestBody = { bytes: new TextEncoder().encode('{"input":"hi"}'), streamError: null };
+
+const turn = { method: 'POST', path: '/v1/embeddings', body: requestBody };
+
+// Two stages, so the record has a shape to hold: one that hands down and one
+// that answers.
+interface Facts {
+ 'in.text': string;
+ 'out.result': string;
+}
+
+const answer = defineStage, Pick>({
+ name: 'answer',
+ return: { provides: ['out.result'] },
+ execute: async facts => move({ ...facts, 'out.result': facts['in.text'].toUpperCase() }),
+});
+
+const shout = defineStage, Pick, Pick, Pick>({
+ name: 'shout',
+ through: {
+ request: { needs: ['in.text'], consumes: [], provides: [] },
+ response: { needs: ['out.result'], consumes: [], provides: [] },
+ },
+ execute: async (facts, next, use) => {
+ use.log.info('shouting', { length: facts['in.text'].length });
+ return await next(move({ ...facts, 'in.text': `${facts['in.text']}!` }));
+ },
+});
+
+const pipeline = compose, Pick>('shout-it', [shout, answer]);
+
+const runRecordOf = (stored: { record: StoredDumpRecord } | undefined): StoredDumpRunRecord => {
+ if (!stored) throw new Error('expected a stored dump record');
+ if (stored.record.shape !== 'run') throw new Error(`expected the run shape, got ${stored.record.shape}`);
+ return stored.record;
+};
+
+const ndjson = (record: StoredDumpRunRecord): string => new TextDecoder().decode(record.events);
+
+const lines = (record: StoredDumpRunRecord): DumpEvent[] =>
+ ndjson(record).split('\n').filter(Boolean).map(line => JSON.parse(line) as DumpEvent);
+
+test('a run under a key with retention stores its whole event stream as NDJSON', async () => {
+ const stubs = installDumpStubs(initDumpStore, initDumpBroker);
+ const dump = openRunDump(apiKey(3600), turn, trackBackground);
+ if (dump === null) throw new Error('a key with retention must open a run dump');
+
+ const { facts } = await run(pipeline, move({ 'in.text': 'hey' }), { dump: dump.sink });
+ assertEquals(facts['out.result'], 'HEY!');
+ dump.finalize(200, 12);
+ await flushBackground();
+
+ const record = runRecordOf(stubs.stored[0]);
+ const events = lines(record);
+ // Every stage is in the tree and the shape of the run is in the parent ids.
+ assertEquals(
+ events.filter(event => event.type === 'stage.entered').map(event => [event.name, event.parentStageId]),
+ [['shout', null], ['answer', 1]],
+ );
+ // A stage's own log line is content about that stage, like the other five kinds.
+ assertEquals(events.filter(event => event.type === 'stage.log').map(event => event.message), ['shouting']);
+ // A `stage.leaved` that hands up exactly what its last child handed up carries
+ // nothing and is not emitted, so `shout`'s exit goes and `answer`'s stays.
+ assertEquals(events.filter(event => event.type === 'stage.leaved').map(event => event.stageId), [2]);
+ // NDJSON: one event per line, appended in order, so the stored file and what a
+ // live observer would be handed are the same bytes.
+ assertEquals(ndjson(record).endsWith('\n'), true);
+ assertEquals(ndjson(record).trimEnd().split('\n').length, events.length);
+
+ assertEquals(record.meta.method, 'POST');
+ assertEquals(record.meta.path, '/v1/embeddings');
+ assertEquals(record.meta.status, 200);
+ assertEquals(record.meta.requestBytes, requestBody.bytes.byteLength);
+ assertEquals(record.meta.responseBytes, 12);
+ assertEquals(stubs.published.map(entry => entry.meta.id), [record.meta.id]);
+});
+
+test('a key without retention opens no sink, so a run records and stores nothing', async () => {
+ const stubs = installDumpStubs(initDumpStore, initDumpBroker);
+ const dump = openRunDump(apiKey(null), turn, trackBackground);
+ assertEquals(dump, null);
+
+ // The absence is the mechanism: with nothing to put in `services.dump` the
+ // runner does none of the recording, rather than feeding a sink that throws
+ // the result away.
+ const emitted: Event[] = [];
+ const services = dump === null ? {} : { dump: (event: Event) => { emitted.push(event); } };
+ const { facts } = await run(pipeline, move({ 'in.text': 'hey' }), services);
+ assertEquals(facts['out.result'], 'HEY!');
+ await flushBackground();
+
+ assertEquals(emitted, []);
+ assertEquals(stubs.stored, []);
+ assertEquals(stubs.published, []);
+});
+
+test('a run record carries the attribution the turn stamped on it', async () => {
+ const stubs = installDumpStubs(initDumpStore, initDumpBroker);
+ const dump = openRunDump(apiKey(3600), turn, trackBackground);
+ if (dump === null) throw new Error('a key with retention must open a run dump');
+
+ dump.requestedModel('text-embedding-3-small');
+ dump.failed(new Error('upstream went\naway'));
+ dump.finalize(null, 0);
+ await flushBackground();
+
+ const record = runRecordOf(stubs.stored[0]);
+ assertEquals(record.meta.model, 'text-embedding-3-small');
+ assertEquals(record.meta.status, null);
+ assertEquals(record.meta.error, { kind: 'failed', reason: 'upstream went away' });
+});
+
+test('a run whose request never arrived intact records that as the turn\'s failure', async () => {
+ const stubs = installDumpStubs(initDumpStore, initDumpBroker);
+ const dump = openRunDump(
+ apiKey(3600),
+ { ...turn, body: { bytes: new Uint8Array(), streamError: 'client aborted the upload' } },
+ trackBackground,
+ );
+ if (dump === null) throw new Error('a key with retention must open a run dump');
+
+ dump.finalize(400, 0);
+ await flushBackground();
+
+ assertEquals(runRecordOf(stubs.stored[0]).meta.error, { kind: 'failed', reason: 'client aborted the upload' });
+});
+
+test('finalizing on a response measures what the client reads and leaves it intact', async () => {
+ const stubs = installDumpStubs(initDumpStore, initDumpBroker);
+ const dump = openRunDump(apiKey(3600), turn, trackBackground);
+ if (dump === null) throw new Error('a key with retention must open a run dump');
+
+ await run(pipeline, move({ 'in.text': 'hey' }), { dump: dump.sink });
+ const answered = dump.finalize(new Response('data: one\n\ndata: two\n\n', {
+ status: 200,
+ headers: { 'content-type': 'text/event-stream' },
+ }));
+
+ assertEquals(await answered.text(), 'data: one\n\ndata: two\n\n');
+ await flushBackground();
+
+ const record = runRecordOf(stubs.stored[0]);
+ assertEquals(record.meta.responseBytes, 22);
+});
diff --git a/packages/gateway/__tests__/dump/test-fixtures.ts b/packages/gateway/__tests__/dump/test-fixtures.ts
index cf9d384140..30e5068dd2 100644
--- a/packages/gateway/__tests__/dump/test-fixtures.ts
+++ b/packages/gateway/__tests__/dump/test-fixtures.ts
@@ -1,6 +1,7 @@
import type { DumpBroker } from '../../src/dump/broker.ts';
import type { DumpStore } from '../../src/dump/store-contract.ts';
-import type { DumpMetadata, StoredDumpRecord } from '../../src/dump/types.ts';
+import type { DumpMetadata, StoredDumpEdgeRecord, StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts';
+import { encodeRun, toNdjson, type Event } from '@floway-dev/pipeline';
export const fakeMeta = (overrides: Partial = {}): DumpMetadata => ({
id: 'test-id',
@@ -20,12 +21,41 @@ export const fakeMeta = (overrides: Partial = {}): DumpMetadata =>
...overrides,
});
-export const fakeRecord = (overrides: Partial = {}): StoredDumpRecord => ({
+export const fakeRecord = (overrides: Partial = {}): StoredDumpEdgeRecord => ({
+ shape: 'edge',
meta: fakeMeta(overrides),
request: { method: 'POST', path: '/v1/x', headers: [], body: new Uint8Array() },
response: { status: 200, headers: [], body: { type: 'none' } },
});
+// A run record holds one NDJSON stream and no edge halves, so a fixture takes
+// the events a run emitted and encodes them the way the sink does — object
+// space, folding and all.
+export const fakeRunRecord = (events: readonly Event[], overrides: Partial = {}): StoredDumpRunRecord => ({
+ shape: 'run',
+ meta: fakeMeta(overrides),
+ events: new TextEncoder().encode(toNdjson(encodeRun(events))),
+});
+
+// A reader hands back whichever shape was written, so a test asserting on the
+// request or response halves says which one it wrote.
+export const edgeRecordOf = (record: StoredDumpRecord | null | undefined): StoredDumpEdgeRecord => {
+ if (!record) throw new Error('expected a stored dump record');
+ if (record.shape !== 'edge') throw new Error(`expected the edge shape, got ${record.shape}`);
+ return record;
+};
+
+export const runRecordOf = (record: StoredDumpRecord | null | undefined): StoredDumpRunRecord => {
+ if (!record) throw new Error('expected a stored dump record');
+ if (record.shape !== 'run') throw new Error(`expected the run shape, got ${record.shape}`);
+ return record;
+};
+
+/** The events a run recorded, decoded from the NDJSON one line at a time. */
+export const eventsOf = (record: StoredDumpRunRecord): readonly Record[] =>
+ new TextDecoder().decode(record.events).split('\n').filter(line => line.length > 0)
+ .map(line => JSON.parse(line) as Record);
+
type DumpStubFailMethod =
| 'put'
| 'list'
@@ -60,11 +90,14 @@ export const installDumpStubs = (
},
async put(keyId, record) {
if (throws.put) throw throws.put;
- if (record.request.body.encoding !== 'identity') throw new Error('dump test stub expected identity request body');
- const storedRecord: StoredDumpRecord = {
- ...record,
- request: { ...record.request, body: record.request.body.bytes },
- };
+ // The run shape is already stored-shaped; only the edge shape carries a
+ // separately prepared request body to rehydrate.
+ if (record.shape === 'edge' && record.request.body.encoding !== 'identity') {
+ throw new Error('dump test stub expected identity request body');
+ }
+ const storedRecord: StoredDumpRecord = record.shape === 'run'
+ ? record
+ : { ...record, request: { ...record.request, body: record.request.body.bytes } };
stored.push({ keyId, record: storedRecord });
const list = records.get(keyId) ?? [];
list.unshift(storedRecord);
diff --git a/packages/gateway/__tests__/dump/wire_test.ts b/packages/gateway/__tests__/dump/wire_test.ts
index 98bfabf688..e4abae2283 100644
--- a/packages/gateway/__tests__/dump/wire_test.ts
+++ b/packages/gateway/__tests__/dump/wire_test.ts
@@ -1,10 +1,11 @@
-import { test } from 'vitest';
+import { expect, test } from 'vitest';
-import type { StoredDumpRecord } from '../../src/dump/types.ts';
+import type { DumpEdgeRecord, StoredDumpEdgeRecord } from '../../src/dump/types.ts';
import { dumpRecordToWire } from '../../src/dump/wire.ts';
import { assertEquals } from '@floway-dev/test-utils';
-const baseStored = (overrides: Partial = {}): StoredDumpRecord => ({
+const baseStored = (overrides: Partial = {}): StoredDumpEdgeRecord => ({
+ shape: 'edge',
meta: {
id: 'rec',
startedAt: 0,
@@ -26,10 +27,16 @@ const baseStored = (overrides: Partial = {}): StoredDumpRecord
...overrides,
});
+const edgeWire = (record: StoredDumpEdgeRecord): DumpEdgeRecord => {
+ const wire = dumpRecordToWire(record);
+ if (wire.shape !== 'edge') throw new Error('expected the edge shape');
+ return wire;
+};
+
// A textual request content-type with valid UTF-8 bytes serializes as utf8 on
// the wire — the dashboard reads `data` directly as a string.
test('dumpRecordToWire encodes a textual request body as utf8', () => {
- const wire = dumpRecordToWire(baseStored({
+ const wire = edgeWire(baseStored({
request: {
method: 'POST',
path: '/v1/messages',
@@ -42,7 +49,7 @@ test('dumpRecordToWire encodes a textual request body as utf8', () => {
});
test('dumpRecordToWire recognizes structured textual suffixes without accepting near matches', () => {
- const structured = dumpRecordToWire(baseStored({
+ const structured = edgeWire(baseStored({
request: {
method: 'POST',
path: '/v1/x',
@@ -52,7 +59,7 @@ test('dumpRecordToWire recognizes structured textual suffixes without accepting
}));
assertEquals(structured.request.body.encoding, 'utf8');
- const nearMatch = dumpRecordToWire(baseStored({
+ const nearMatch = edgeWire(baseStored({
request: {
method: 'POST',
path: '/v1/x',
@@ -67,7 +74,7 @@ test('dumpRecordToWire recognizes structured textual suffixes without accepting
// preserves every byte; the dashboard decodes base64 client-side.
test('dumpRecordToWire encodes a binary response body as base64', () => {
const png = new Uint8Array([0x89, 0x50, 0x4E, 0x47]); // PNG magic
- const wire = dumpRecordToWire(baseStored({
+ const wire = edgeWire(baseStored({
response: {
status: 200,
headers: [['content-type', 'image/png']],
@@ -85,7 +92,7 @@ test('dumpRecordToWire encodes a binary response body as base64', () => {
// as UTF-8 falls through to base64 so the wire never silently corrupts.
test('dumpRecordToWire falls back to base64 when textual content-type carries non-UTF-8 bytes', () => {
const bytes = new Uint8Array([0xFF, 0xFE, 0xFD]);
- const wire = dumpRecordToWire(baseStored({
+ const wire = edgeWire(baseStored({
request: {
method: 'POST',
path: '/v1/x',
@@ -101,13 +108,39 @@ test('dumpRecordToWire falls back to base64 when textual content-type carries no
// `stream` and `none` response bodies pass through wire serialization
// unchanged because they carry no raw bytes.
test('dumpRecordToWire passes stream + none response bodies through', () => {
- const streamWire = dumpRecordToWire(baseStored({
+ const streamWire = edgeWire(baseStored({
response: { status: 200, headers: [], body: { type: 'stream', events: [] } },
}));
assertEquals(streamWire.response.body.type, 'stream');
- const noneWire = dumpRecordToWire(baseStored({
+ const noneWire = edgeWire(baseStored({
response: { status: null, headers: [], body: { type: 'none' } },
}));
assertEquals(noneWire.response.body.type, 'none');
});
+
+// A run record crosses to the wire as the stored bytes decoded, because a line
+// of NDJSON is one event and one SSE `data:` payload — what the dashboard reads
+// here is what a live observer will read frame by frame.
+test('dumpRecordToWire hands a run record its NDJSON verbatim', () => {
+ const ndjson = '{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null}\n'
+ + '{"type":"stage.leaved","stageId":1,"facts":{"response.http.status":200}}\n';
+ const wire = dumpRecordToWire({
+ shape: 'run',
+ meta: baseStored().meta,
+ events: new TextEncoder().encode(ndjson),
+ });
+ if (wire.shape !== 'run') throw new Error('expected the run shape');
+ assertEquals(wire.events, ndjson);
+ assertEquals(wire.meta.id, 'rec');
+});
+
+// The gateway wrote those bytes itself, so bytes that are not UTF-8 mean a
+// corrupted record and the reader says so rather than serving mojibake.
+test('dumpRecordToWire refuses a run stream that is not UTF-8', () => {
+ expect(() => dumpRecordToWire({
+ shape: 'run',
+ meta: baseStored().meta,
+ events: new Uint8Array([0xFF, 0xFE, 0xFD]),
+ })).toThrow();
+});
diff --git a/packages/gateway/__tests__/repo/dump-store_test.ts b/packages/gateway/__tests__/repo/dump-store_test.ts
index c4a685eeb3..d0ca1d14f0 100644
--- a/packages/gateway/__tests__/repo/dump-store_test.ts
+++ b/packages/gateway/__tests__/repo/dump-store_test.ts
@@ -6,11 +6,12 @@ import { expect, test } from 'vitest';
import { createSqliteTestDb, mapRunChangeCount } from './test-sqlite.ts';
import { decodeDumpBodyDescriptor } from '../../src/dump/storage-codec.ts';
-import type { DumpWriteRecord } from '../../src/dump/types.ts';
+import type { DumpWriteEdgeRecord, StoredDumpEdgeRecord, StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts';
import { FileDumpStore } from '../../src/repo/dump-store.ts';
import { initRepo } from '../../src/repo/index.ts';
import { SqlRepo } from '../../src/repo/sql.ts';
import { collectSpilledFiles } from '../../src/scheduled/spilled-files.ts';
+import { encodeRun, toNdjson, type Facts } from '@floway-dev/pipeline';
import { initFileStore, MemoryFileStore } from '@floway-dev/platform';
import type { FileStore, SqlDatabase } from '@floway-dev/platform';
import { assertEquals, assertExists } from '@floway-dev/test-utils';
@@ -36,7 +37,8 @@ const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s);
const requestBody = utf8('{"hello":"world"}');
-const baseRecord = (id: string, completedAt: number): DumpWriteRecord => ({
+const baseRecord = (id: string, completedAt: number): DumpWriteEdgeRecord => ({
+ shape: 'edge',
meta: {
id, startedAt: completedAt - 1, completedAt, method: 'POST', path: '/v1/x', status: 200,
upstream: null, model: 'm', inputTokens: 1, outputTokens: 2,
@@ -54,6 +56,14 @@ const baseRecord = (id: string, completedAt: number): DumpWriteRecord => ({
},
});
+// Every fixture above writes the edge shape, and a reader hands back what was
+// written — so a test that asserts on the request or response halves says so.
+const edgeOf = (record: StoredDumpRecord | null): StoredDumpEdgeRecord => {
+ assertExists(record);
+ if (record.shape !== 'edge') throw new Error('expected the edge shape');
+ return record;
+};
+
test('FileDumpStore prepares request gzip before terminal persistence', async () => {
const db = await openDb();
const files = new MemoryFileStore();
@@ -61,7 +71,7 @@ test('FileDumpStore prepares request gzip before terminal persistence', async ()
const raw = utf8(`{"content":"${'repeatable '.repeat(4096)}"}`);
const prepared = await store.prepareRequestBody(raw);
const base = baseRecord('01HZZ0000000000000000000P1', Date.UTC(2026, 5, 1, 12, 0, 0));
- const record: DumpWriteRecord = {
+ const record: DumpWriteEdgeRecord = {
...base,
meta: { ...base.meta, requestBytes: raw.byteLength },
request: {
@@ -76,8 +86,7 @@ test('FileDumpStore prepares request gzip before terminal persistence', async ()
assertEquals(prepared.decodedByteLength, raw.byteLength);
assertEquals(prepared.bytes.byteLength < raw.byteLength, true);
await store.put('key_x', record);
- const fetched = await store.get('key_x', record.meta.id);
- assertExists(fetched);
+ const fetched = edgeOf(await store.get('key_x', record.meta.id));
assertEquals(Array.from(fetched.request.body), Array.from(raw));
});
@@ -88,8 +97,7 @@ test('FileDumpStore round-trips a JSON record through gzip', async () => {
const record = baseRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 12, 0, 0));
await store.put('key_x', record);
- const fetched = await store.get('key_x', '01HZZ0000000000000000000A1');
- assertExists(fetched);
+ const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000A1'));
assertEquals(fetched.meta.id, record.meta.id);
assertEquals(new TextDecoder().decode(fetched.request.body), '{"hello":"world"}');
if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes');
@@ -101,7 +109,7 @@ test('FileDumpStore preserves the original content-type header on binary bodies'
const files = new MemoryFileStore();
const store = new FileDumpStore(db, files);
const pngMagic = new Uint8Array([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]);
- const record: DumpWriteRecord = {
+ const record: DumpWriteEdgeRecord = {
...baseRecord('01HZZ0000000000000000000PNG', Date.UTC(2026, 5, 1, 12, 0, 0)),
response: {
status: 200,
@@ -111,8 +119,7 @@ test('FileDumpStore preserves the original content-type header on binary bodies'
};
await store.put('key_x', record);
- const fetched = await store.get('key_x', '01HZZ0000000000000000000PNG');
- assertExists(fetched);
+ const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000PNG'));
// The header pair must survive verbatim — no `;base64` suffix tacked on.
assertEquals(fetched.response.headers.find(([k]) => k === 'content-type')?.[1], 'image/png');
if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes');
@@ -126,7 +133,7 @@ test('FileDumpStore preserves the bytes discriminator on an empty-body response'
// 204-style: real upstream response with status + headers but a zero-length
// body. Persistence drops the body file (nothing to gzip), but headers are
// still written — the read path must surface this as `bytes`, not `none`.
- const record: DumpWriteRecord = {
+ const record: DumpWriteEdgeRecord = {
...baseRecord('01HZZ0000000000000000000E1', Date.UTC(2026, 5, 1, 12, 0, 0)),
response: {
status: 204,
@@ -136,8 +143,7 @@ test('FileDumpStore preserves the bytes discriminator on an empty-body response'
};
await store.put('key_x', record);
- const fetched = await store.get('key_x', '01HZZ0000000000000000000E1');
- assertExists(fetched);
+ const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000E1'));
if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes');
assertEquals(fetched.response.body.body.byteLength, 0);
assertEquals(fetched.response.headers.find(([k]) => k === 'content-type')?.[1], 'application/json');
@@ -147,7 +153,7 @@ test('FileDumpStore round-trips an SSE record as a stream events array', async (
const db = await openDb();
const files = new MemoryFileStore();
const store = new FileDumpStore(db, files);
- const record: DumpWriteRecord = {
+ const record: DumpWriteEdgeRecord = {
...baseRecord('01HZZ0000000000000000000A2', Date.UTC(2026, 5, 1, 12, 0, 0)),
response: {
status: 200,
@@ -162,13 +168,98 @@ test('FileDumpStore round-trips an SSE record as a stream events array', async (
},
};
await store.put('key_x', record);
- const fetched = await store.get('key_x', '01HZZ0000000000000000000A2');
- assertExists(fetched);
+ const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000A2'));
if (fetched.response.body.type !== 'stream') throw new Error('expected stream');
assertEquals(fetched.response.body.events.length, 2);
assertEquals(fetched.response.body.events[0]!.frame.type, 'event');
});
+// A pipelined turn stores the run rather than its edges: one NDJSON body file
+// under the same contract, and a row that carries the same metadata.
+const runRecord = (id: string, completedAt: number): StoredDumpRunRecord => {
+ const entered: Facts = { 'request.http.path': '/v1/embeddings' };
+ const left: Facts = { ...entered, 'response.http.status': 200 };
+ return {
+ shape: 'run',
+ meta: baseRecord(id, completedAt).meta,
+ events: new TextEncoder().encode(toNdjson(encodeRun([
+ { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: entered },
+ { type: 'stage.entered', stageId: 2, name: 'dial', parentStageId: 1, facts: entered },
+ { type: 'stage.leaved', stageId: 2, facts: left },
+ { type: 'stage.leaved', stageId: 1, facts: left },
+ ]))),
+ };
+};
+
+test('FileDumpStore round-trips a run record as its NDJSON event stream', async () => {
+ const db = await openDb();
+ const files = new MemoryFileStore();
+ const store = new FileDumpStore(db, files);
+ const record = runRecord('01HZZ00000000000000000RUN1', Date.UTC(2026, 5, 1, 12, 0, 0));
+
+ await store.put('key_x', record);
+ const fetched = await store.get('key_x', record.meta.id);
+ assertExists(fetched);
+ if (fetched.shape !== 'run') throw new Error(`expected the run shape, got ${fetched.shape}`);
+ assertEquals(new TextDecoder().decode(fetched.events), new TextDecoder().decode(record.events));
+
+ // The stream is a body file like the other two: gzipped, named for what it
+ // holds, and pointed at by the row's descriptor.
+ const row = await db.prepare('SELECT request_body_descriptor, response_body_descriptor FROM dump_records WHERE key_id = ? AND id = ?')
+ .bind('key_x', record.meta.id)
+ .first<{ request_body_descriptor: string | null; response_body_descriptor: string }>();
+ assertExists(row);
+ assertEquals(row.request_body_descriptor, null);
+ const descriptor = decodeDumpBodyDescriptor(row.response_body_descriptor, 'test run descriptor');
+ assertEquals(descriptor.type, 'run');
+ assertEquals(descriptor.key.endsWith('.run.gz'), true);
+});
+
+test('FileDumpStore lists a run record alongside an edge-shaped one', async () => {
+ const db = await openDb();
+ const store = new FileDumpStore(db, new MemoryFileStore());
+ const base = Date.UTC(2026, 5, 1, 12, 0, 0);
+ await store.put('key_x', baseRecord('01HZZ00000000000000000EDG1', base));
+ await store.put('key_x', runRecord('01HZZ00000000000000000RUN2', base + 1));
+
+ const listed = await store.list('key_x', { limit: 10 });
+ assertEquals(listed.map(meta => meta.id), ['01HZZ00000000000000000RUN2', '01HZZ00000000000000000EDG1']);
+ // One metadata shape for both: the list says nothing about how a turn was
+ // served, and every column a row renders is filled either way.
+ assertEquals(listed.map(meta => meta.model), ['m', 'm']);
+ assertEquals(listed.map(meta => meta.status), [200, 200]);
+});
+
+test('FileDumpStore retires an expired run record and collects its stream file', async () => {
+ const db = await openDb();
+ const repo = new SqlRepo(db);
+ initRepo(repo);
+ const files = new MemoryFileStore();
+ initFileStore(files);
+ const store = new FileDumpStore(db, files);
+ const now = Date.UTC(2026, 5, 1, 12, 0, 0);
+ await store.put('key_x', runRecord('01HZZ00000000000000000RUN3', Date.UTC(2026, 5, 1, 9, 0, 0)));
+ await store.put('key_x', runRecord('01HZZ00000000000000000RUN4', now));
+ await repo.apiKeys.update('key_x', { dumpRetentionSeconds: 2 * 3600 });
+
+ const originalNow = Date.now;
+ Date.now = () => now + 1;
+ try {
+ assertEquals((await store.list('key_x', { limit: 10 })).map(meta => meta.id), ['01HZZ00000000000000000RUN4']);
+ assertEquals(await store.get('key_x', '01HZZ00000000000000000RUN3'), null);
+ assertEquals(await store.deleteExpiredBatch('key_x', now + 1, 100), 1);
+ await collectSpilledFiles(now + 1);
+ } finally {
+ Date.now = originalNow;
+ }
+
+ const { results: remaining } = await db
+ .prepare("SELECT file_key FROM spilled_files WHERE state = 'owned' ORDER BY file_key")
+ .all<{ file_key: string }>();
+ assertEquals(remaining.map(row => row.file_key.endsWith('.run.gz')), [true]);
+ assertEquals(remaining.every(row => !row.file_key.includes('2026060109')), true);
+});
+
test('FileDumpStore rejects malformed metadata with its row identity', async () => {
const db = await openDb();
const store = new FileDumpStore(db, new MemoryFileStore());
@@ -212,7 +303,7 @@ test('FileDumpStore rejects malformed stream events with record and file context
const db = await openDb();
const files = new MemoryFileStore();
const store = new FileDumpStore(db, files);
- const record: DumpWriteRecord = {
+ const record: DumpWriteEdgeRecord = {
...baseRecord('01HZZ000000000000000000BADE', Date.UTC(2026, 5, 1, 12)),
response: {
status: 200,
@@ -415,8 +506,7 @@ test('FileDumpStore: put + get round-trips through real-filesystem IO', async ()
const record = baseRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 12, 0, 0));
await store.put('key_x', record);
- const fetched = await store.get('key_x', '01HZZ0000000000000000000A1');
- assertExists(fetched);
+ const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000A1'));
assertEquals(new TextDecoder().decode(fetched.request.body), '{"hello":"world"}');
if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes');
assertEquals(new TextDecoder().decode(fetched.response.body.body), '{"id":"abc"}');
diff --git a/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts b/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts
index b0f8058e17..3719bd4c72 100644
--- a/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts
+++ b/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts
@@ -38,6 +38,7 @@ const responseItem = (id: string, refreshedAt: number, apiKeyId = 'key-a'): Stor
});
const dumpRecord = (id: string, completedAt: number): DumpWriteRecord => ({
+ shape: 'edge',
meta: {
id,
startedAt: completedAt - 1,
diff --git a/packages/gateway/__tests__/scheduled_test.ts b/packages/gateway/__tests__/scheduled_test.ts
index 02c51f76e9..5bb13c1a15 100644
--- a/packages/gateway/__tests__/scheduled_test.ts
+++ b/packages/gateway/__tests__/scheduled_test.ts
@@ -30,6 +30,7 @@ const apiKey = (id: string, now: number, secretDigit: number): ApiKey => ({
});
const fileBackedDumpRecord = (id: string, completedAt: number): DumpWriteRecord => ({
+ shape: 'edge',
meta: {
id,
startedAt: completedAt - 1,
diff --git a/packages/gateway/package.json b/packages/gateway/package.json
index 69fd159774..3baff3e40a 100644
--- a/packages/gateway/package.json
+++ b/packages/gateway/package.json
@@ -42,6 +42,7 @@
"@floway-dev/agent-setup": "workspace:*",
"@floway-dev/http": "workspace:*",
"@floway-dev/interceptor": "workspace:*",
+ "@floway-dev/pipeline": "workspace:*",
"@floway-dev/platform": "workspace:*",
"@floway-dev/protocols": "workspace:*",
"@floway-dev/provider": "workspace:*",
diff --git a/packages/gateway/src/data-plane/alpha-search/http.ts b/packages/gateway/src/data-plane/alpha-search/http.ts
new file mode 100644
index 0000000000..ad130eb635
--- /dev/null
+++ b/packages/gateway/src/data-plane/alpha-search/http.ts
@@ -0,0 +1,95 @@
+// POST /alpha/search, served through the pipeline.
+//
+// What the handler decides before the run is which ending the chain gets: the operator's
+// `passthroughOpenAiSearch` setting pins an upstream, and everything else runs the commands
+// here. That is a property of configuration rather than of the request, so it is read once and
+// the chain is assembled around it — which is why the family has one pipeline builder rather
+// than a stage that branches.
+//
+// The configured search backend reaches the chain as a service and never as a fact: it holds
+// live handles built from the operator's provider credential, so none of it is dumpable and
+// none of it belongs in the record.
+
+import type { Context } from 'hono';
+
+import { searchServePipeline, type SearchExecution, type SearchServices } from './pipeline.ts';
+import { alphaSearchRequestSchema, type AlphaSearchRequest } from './protocol.ts';
+import type { AuthedContext } from '../../middleware/auth.ts';
+import { openPrologue, readIngress, serveThrough, type Prologue } from '../pipeline/serve.ts';
+import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts';
+import { loadWebSearchConfig } from '../tools/web-search/config.ts';
+import { resolveConfiguredWebSearchProvider } from '../tools/web-search/provider.ts';
+import type { ConfiguredWebSearchProvider, WebSearchConfig } from '../tools/web-search/types.ts';
+import { move } from '@floway-dev/pipeline';
+
+/** Which ending this gateway's configuration asks for. The word "passthrough" is about whose
+ * search results the client is given, not about carrying a protocol the gateway has not
+ * parsed — what comes back is read and written again either way. */
+const executionFor = (config: WebSearchConfig): SearchExecution =>
+ config.passthroughOpenAiSearch.enabled
+ ? { kind: 'upstream', upstreamId: config.passthroughOpenAiSearch.upstreamId, model: config.passthroughOpenAiSearch.model }
+ : { kind: 'local' };
+
+/** Resolved once per turn and shared by every operation in it: one turn's commands run against
+ * one backend, and resolving per operation would build the same handles again. */
+const searchProviderFor = (config: WebSearchConfig): SearchServices['searchProvider'] => {
+ let resolved: Promise | undefined;
+ return () => {
+ resolved ??= Promise.resolve(resolveConfiguredWebSearchProvider(config));
+ return resolved;
+ };
+};
+
+/** The request as this protocol, or the sentence saying why it is not. The family reads its own
+ * body because the run is given the bytes the client sent, and a validator middleware would
+ * have consumed them first. */
+type Read = { readonly ok: true; readonly request: AlphaSearchRequest } | { readonly ok: false; readonly message: string };
+
+const readAlphaSearchRequest = (bytes: Uint8Array): Read => {
+ let body: unknown;
+ try {
+ body = JSON.parse(new TextDecoder().decode(bytes)) as unknown;
+ } catch (error) {
+ return { ok: false, message: error instanceof Error ? error.message : String(error) };
+ }
+ const parsed = alphaSearchRequestSchema.safeParse(body);
+ // The first issue's message, verbatim: a schema attaches a field-aware message where it wants
+ // one, and prepending the path would name the field twice.
+ return parsed.success
+ ? { ok: true, request: parsed.data }
+ : { ok: false, message: parsed.error.issues[0]?.message ?? 'Invalid input' };
+};
+
+export const alphaSearch = async (c: Context): Promise => {
+ const ingress = await readIngress(c);
+ const read = readAlphaSearchRequest(ingress.body.bytes);
+ if (!read.ok) {
+ // A request the gateway could not read never reaches a pipeline: there is no backend to
+ // reach and no attempt to make, so there is nothing for a run to record.
+ const refused = openPrologue(c as AuthedContext, ingress, { wantsStream: false });
+ refused.gateway.dump?.error('gateway');
+ return finalizeGatewayResponse(refused.gateway, Response.json({ error: read.message }, { status: 400 }));
+ }
+
+ const config = await loadWebSearchConfig();
+ const execution = executionFor(config);
+ // A pinned turn is attributed to the model the operator pinned, because that is the one that
+ // will be called: the id the caller sent names a model of its own that never travels.
+ const prologue = openPrologue(c as AuthedContext, ingress, {
+ wantsStream: false,
+ ...(execution.kind === 'upstream' ? { model: execution.model } : {}),
+ });
+ const services: SearchServices = { ...prologue.services, searchProvider: searchProviderFor(config) };
+ const withBackend: Prologue = { ...prologue, services };
+
+ return await serveThrough(
+ c,
+ withBackend,
+ searchServePipeline(execution),
+ move({
+ 'ingress.http.headers': prologue.headers,
+ 'request.search.alphaSearch': read.request,
+ }) as never,
+ facts => ({ body: JSON.stringify(facts['response.search.rendered']), contentType: 'application/json' }),
+ );
+};
diff --git a/packages/gateway/src/data-plane/alpha-search/pipeline.ts b/packages/gateway/src/data-plane/alpha-search/pipeline.ts
new file mode 100644
index 0000000000..0f6c116d11
--- /dev/null
+++ b/packages/gateway/src/data-plane/alpha-search/pipeline.ts
@@ -0,0 +1,381 @@
+// Search as a pipeline. The family with no upstream model and no candidate list: by default
+// it runs the request's commands here, through the configured search backend, and when the
+// operator has pinned a Codex or Custom upstream it asks that upstream's own search endpoint
+// instead.
+//
+// That is an ending, not a missing family. What this shares with the other families is the
+// edge; what varies is where the chain stops:
+//
+// emitAlphaSearch the edge: serializes the answer into Codex's protocol
+// parseSearchOperations local only: Codex's commands become the gateway's own
+// executeSearchOperations the local ending: runs them, and provides the answer
+// callSearchUpstream the pinned ending: dials, and provides what came back
+//
+// `resolveCandidates` and `failover` have nothing to range over here. Local execution reaches
+// no upstream at all, and the pinned mode names exactly one upstream and one model in
+// operator configuration, so a run is its own only attempt.
+
+import {
+ parseAlphaSearchResponse,
+ renderAlphaSearchResponse,
+ webSearchFiltersFromSettings,
+ type AlphaSearchRequest,
+ type AlphaSearchResponse,
+} from './protocol.ts';
+import { isJsonObject } from '../../shared/json-helpers.ts';
+import type { Failure, GatewayFacts } from '../pipeline/facts.ts';
+import { isFailure } from '../pipeline/facts.ts';
+import type { GatewayServices } from '../pipeline/services.ts';
+import { writeSettlement } from '../pipeline/settlement.ts';
+import { enumerateModelCandidates } from '../providers/resolution.ts';
+import type { GatewayCtx } from '../shared/gateway-ctx.ts';
+import { filterInboundHeadersForProvider } from '../shared/inbound-headers.ts';
+import { telemetryModelIdentity } from '../shared/telemetry/attribution.ts';
+import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts';
+import {
+ assertLocalWebSearchSupport,
+ executeOperationToText,
+ parseWebSearchOperations,
+ startBatchFetch,
+ UnsupportedLocalWebSearchFeatureError,
+ type WebSearchExecutionSession,
+ type WebSearchFilters,
+ type WebSearchOperation,
+} from '../tools/web-search/operations.ts';
+import type { ConfiguredWebSearchProvider } from '../tools/web-search/types.ts';
+import type { Pipeline } from '@floway-dev/pipeline';
+import { compose, defineStage, move } from '@floway-dev/pipeline';
+import { identityWrapUpstreamCall, providerModelOf, type ModelCandidate } from '@floway-dev/provider';
+
+/** Search's own keys. They extend the shared space and never merge into it, so a stage
+ * written against the gateway alone cannot name one. */
+export interface SearchFacts extends GatewayFacts {
+ 'request.search.alphaSearch': AlphaSearchRequest;
+ /** What Codex's commands mean to this gateway. Local execution runs on these and never
+ * reads Codex's request again, which is why the stage that provides them consumes it. */
+ 'request.search.operations': readonly WebSearchOperation[];
+ 'request.search.filters': WebSearchFilters;
+ 'response.search.alphaSearch': AlphaSearchResponse | Failure;
+ /** What the client is actually sent, in Codex's protocol. The edge provides it, so a dump
+ * shows the bytes the client received rather than the gateway's own form. */
+ 'response.search.rendered': Record;
+}
+
+type S = { [P in K]: SearchFacts[P] };
+
+/** The configured search backend is reached through a resolver rather than a fact: it holds
+ * live handles, and the operator's provider credential is what builds one — so nothing about
+ * it is dumpable and none of it belongs in the record. */
+export interface SearchServices extends GatewayServices {
+ readonly searchProvider: () => Promise;
+}
+
+/**
+ * The outermost edge. Renders the answer into Codex's protocol and says what status the
+ * client is owed — the one thing a failure value carries that a rendered body cannot.
+ *
+ * It declares needing `response.usage.billable` without reading it, which is this family's
+ * statement that every path accounts for usage: assembly then rejects an ending, or a
+ * short-circuit, that does not provide it.
+ */
+const emitAlphaSearch = defineStage<
+ S,
+ S,
+ S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>,
+ S<'response.search.rendered' | 'response.http.status' | 'response.http.headers'>
+>({
+ name: 'emitAlphaSearch',
+ through: {
+ request: { needs: [], consumes: [], provides: [] },
+ response: {
+ needs: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'],
+ consumes: ['response.search.alphaSearch', 'response.http.headers'],
+ provides: ['response.search.rendered', 'response.http.status', 'response.http.headers'],
+ },
+ },
+ execute: async (facts, next) => {
+ const back = await next(facts);
+ const { 'response.search.alphaSearch': answer, 'response.http.headers': headers, ...rest } = back;
+ // Vendor traces and quota state stay visible; what an intermediary must strip, and what
+ // would misdescribe a body this gateway serialized itself, does not. A filter that removed
+ // nothing hands the same array on, so the record shows no change where none happened.
+ const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name));
+ const forClient = forwardable.length === headers.length ? headers : move(forwardable);
+ if (isFailure(answer)) {
+ // An upstream error body is JSON like any other: it was parsed below and is serialized
+ // again here. A body that was not an object is one this protocol cannot carry, so what
+ // goes out instead is the gateway's own envelope.
+ return {
+ ...rest,
+ 'response.http.headers': forClient,
+ 'response.search.rendered': move(isJsonObject(answer.body)
+ ? answer.body
+ : { error: { message: answer.message, type: 'api_error' } }),
+ 'response.http.status': answer.status,
+ };
+ }
+ return {
+ ...rest,
+ 'response.http.headers': forClient,
+ 'response.search.rendered': move(renderAlphaSearchResponse(answer)),
+ 'response.http.status': 200,
+ };
+ },
+});
+
+/** Both in-band answers a local run can give: Codex reads `output`, so a command it asked for
+ * and this gateway cannot run is text the model sees rather than an HTTP failure. */
+const inBandOutput = (facts: S<'request.search.alphaSearch'>, output: string) => move({
+ ...facts,
+ 'response.search.alphaSearch': { encryptedOutput: null, output },
+ 'response.usage.billable': [],
+ // Nothing was called, so there are no upstream headers to carry.
+ 'response.http.headers': [],
+});
+
+/**
+ * Codex's commands become the gateway's own operations, and its settings become the filters
+ * they run under. Nothing below reads Codex's request afterwards, so it is consumed here and
+ * assembly is what holds that: a stage placed under this one cannot need it back.
+ */
+const parseSearchOperations = defineStage<
+ S<'request.search.alphaSearch'>,
+ S<'request.search.operations' | 'request.search.filters'>,
+ S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>,
+ S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>,
+ S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>
+>({
+ name: 'parseSearchOperations',
+ through: {
+ request: {
+ needs: ['request.search.alphaSearch'],
+ consumes: ['request.search.alphaSearch'],
+ provides: ['request.search.operations', 'request.search.filters'],
+ },
+ response: { needs: [], consumes: [], provides: [] },
+ },
+ return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] },
+ execute: async (facts, next) => {
+ const { 'request.search.alphaSearch': request, ...rest } = facts;
+ const commands = request.commands ?? {};
+
+ try {
+ assertLocalWebSearchSupport(commands);
+ } catch (error) {
+ if (!(error instanceof UnsupportedLocalWebSearchFeatureError)) throw error;
+ return inBandOutput(facts, error.message);
+ }
+
+ const parsed = parseWebSearchOperations(commands);
+ if (parsed.kind !== 'ops' || parsed.ops.length === 0) {
+ return inBandOutput(facts, 'No web search commands were provided. Populate at least one of `search_query`, `open`, or `find`.');
+ }
+
+ return await next({
+ ...rest,
+ 'request.search.operations': move(parsed.ops),
+ 'request.search.filters': move(webSearchFiltersFromSettings(request.settings)),
+ });
+ },
+});
+
+/**
+ * The local ending. Runs every operation against the configured backend and provides the
+ * answer plus what the run is billable for — which is nothing, because no upstream model was
+ * called. What the search backend itself charges is accounted per api key by the operations
+ * as they run, in units no model prices.
+ */
+const executeSearchOperations = defineStage<
+ S<'request.search.operations' | 'request.search.filters'>,
+ S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>,
+ SearchServices
+>({
+ name: 'executeSearchOperations',
+ return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] },
+ execute: async (facts, use) => {
+ const session: WebSearchExecutionSession = {
+ getProvider: use.searchProvider,
+ filters: facts['request.search.filters'],
+ apiKeyId: use.gateway.apiKeyId,
+ pageCache: new Map(),
+ // Codex renders `output` as plain text; the search-action sources list is a Responses
+ // protocol concern with no place here.
+ includeSearchActionSources: false,
+ ...(use.gateway.abortSignal === undefined ? {} : { signal: use.gateway.abortSignal }),
+ };
+
+ // One batched fetchPage covers every open and find URL; each operation then renders its
+ // own text block, in the parser's canonical order — search_query, open, find, preserving
+ // array order within each command kind.
+ const ops = [...facts['request.search.operations']];
+ const batch = await startBatchFetch({ kind: 'ops', ops }, session);
+ const blocks = await Promise.all(ops.map(op => executeOperationToText(op, session, batch)));
+ use.log.debug('ran the search operations', { operations: ops.length });
+
+ return move({
+ ...facts,
+ 'response.search.alphaSearch': { encryptedOutput: null, output: blocks.join('\n\n') },
+ 'response.usage.billable': [],
+ // Nothing was called, so there are no upstream headers to carry.
+ 'response.http.headers': [],
+ });
+ },
+});
+
+/** The operator's pinned search upstream. One upstream and one model, both from
+ * configuration, which is why nothing here narrows a candidate list. */
+export interface PinnedSearchUpstream {
+ readonly kind: 'upstream';
+ readonly upstreamId: string;
+ readonly model: string;
+}
+
+/**
+ * A misconfigured pin is the operator's to see with its stack, the way the gateway surfaces
+ * any internal failure. It is not a failure value: a failure value exists so an earlier stage
+ * can fail over it, and there is no earlier stage here with anywhere to go.
+ */
+const resolvePinnedUpstream = async (pinned: PinnedSearchUpstream, gateway: GatewayCtx): Promise => {
+ if (gateway.upstreamIds !== null && !gateway.upstreamIds.includes(pinned.upstreamId)) {
+ throw new Error('Selected OpenAI search upstream is outside this API key scope');
+ }
+ const { candidates } = await enumerateModelCandidates({
+ upstreamIds: [pinned.upstreamId],
+ model: pinned.model,
+ kind: 'chat',
+ scheduler: gateway.backgroundScheduler,
+ runtimeLocation: gateway.runtimeLocation,
+ });
+ const candidate = candidates.find(value => value.provider.upstreamId === pinned.upstreamId);
+ if (candidate === undefined) {
+ throw new Error(`Selected OpenAI search model ${pinned.model} is unavailable`);
+ }
+ if (candidate.provider.kind !== 'codex' && candidate.provider.kind !== 'custom') {
+ throw new Error('Selected upstream does not support OpenAI search passthrough');
+ }
+ return candidate;
+};
+
+/** Codex projects one per-turn metadata snapshot onto several surfaces, and `SearchRequest`
+ * carries no field for it — so on this endpoint the header is the whole of that surface, and
+ * it is the only inbound header this call reads.
+ * https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/search.rs#L8-L29 */
+const turnMetadataHeaders = (ingress: readonly (readonly [string, string])[]): Headers => {
+ const headers = new Headers();
+ const metadata = ingress.find(([name]) => name.toLowerCase() === 'x-codex-turn-metadata');
+ if (metadata !== undefined) headers.set('x-codex-turn-metadata', metadata[1]);
+ return headers;
+};
+
+/** The body as JSON, or nothing when it was not JSON at all. Which of the two happened is
+ * what the caller reports; here it is only the question being asked. */
+const asJson = (raw: string): unknown => {
+ try {
+ return JSON.parse(raw) as unknown;
+ } catch {
+ return undefined;
+ }
+};
+
+/**
+ * The pinned ending. Dials the operator's search upstream and provides what came back plus
+ * what the call is billable for. A failure is a value here as everywhere, even though this
+ * family has nothing that fails over it: the edge is what turns one into a status and a body.
+ */
+const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage<
+ S<'request.search.alphaSearch' | 'ingress.http.headers'>,
+ S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>,
+ GatewayServices
+>({
+ name: 'callSearchUpstream',
+ return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] },
+ execute: async (facts, use) => {
+ const candidate = await resolvePinnedUpstream(pinned, use.gateway);
+ // The caller's model is dropped: this endpoint is pinned to the operator's model, and the
+ // provider stamps that one on the way out.
+ // TODO: pin SearchRequest.id to one provider account when Codex upstreams support account
+ // pools. The current Codex provider has one active account.
+ const { model: _named, ...request } = facts['request.search.alphaSearch'];
+ const result = await candidate.provider.instance.callAlphaSearch(
+ providerModelOf(candidate),
+ request,
+ use.gateway.abortSignal,
+ {
+ fetcher: candidate.fetcher,
+ waitUntil: use.gateway.backgroundScheduler,
+ // The client's own headers reach the upstream from the record, not from a live
+ // request object: what a provider is allowed to forward is filtered per provider,
+ // and the dump shows what was there to filter.
+ headers: filterInboundHeadersForProvider(turnMetadataHeaders(facts['ingress.http.headers']), candidate.provider),
+ // No `PerformanceOperation` names search, so there is no performance row for a
+ // stamping wrapper's interval to land on.
+ wrapUpstreamCall: identityWrapUpstreamCall,
+ },
+ );
+ // The alpha-search protocol reports no usage at all, so the entity is present with no
+ // quantities — the upstream was called and reported nothing, which is a different
+ // situation from reporting zero.
+ const billable = [{ identity: telemetryModelIdentity(candidate, result.modelKey), quantities: {} }];
+
+ // Every protocol the gateway carries is one it fully understands: the body is read here
+ // and serialized again at the edge, an error body included.
+ const raw = await result.response.text();
+ if (!result.response.ok) {
+ use.log.warn('upstream refused', { status: result.response.status });
+ // The message is what came back as text and the body is the same thing parsed: a dump
+ // reader gets the upstream's own words either way, and only the parsed form is
+ // something the edge can serialize back out.
+ const body = asJson(raw);
+ return move({
+ ...facts,
+ 'response.search.alphaSearch': {
+ status: result.response.status,
+ message: raw,
+ ...(body === undefined ? {} : { body }),
+ },
+ 'response.usage.billable': billable,
+ 'response.http.headers': [...result.response.headers],
+ });
+ }
+
+ const verdict = parseAlphaSearchResponse(asJson(raw));
+ if (!verdict.ok) {
+ // A protocol that requires JSON and receives something else synthesizes its own error,
+ // which is also why the raw text rides along: a dump reader is owed what came back.
+ return move({
+ ...facts,
+ 'response.search.alphaSearch': {
+ status: 502,
+ message: `The search upstream answered ${result.response.status} but not in the search protocol: ${verdict.reason}.`,
+ body: raw,
+ },
+ 'response.usage.billable': billable,
+ 'response.http.headers': [...result.response.headers],
+ });
+ }
+ return move({
+ ...facts,
+ 'response.search.alphaSearch': verdict.response,
+ 'response.usage.billable': billable,
+ 'response.http.headers': [...result.response.headers],
+ });
+ },
+});
+
+/** Which ending the chain gets. `upstream` is the operator's `passthroughOpenAiSearch`
+ * setting: the word there is about whose search results the client is given, not about
+ * carrying a protocol the gateway has not parsed. */
+export type SearchExecution = { readonly kind: 'local' } | PinnedSearchUpstream;
+
+export const searchServePipeline = (execution: SearchExecution): Pipeline<
+ S<'request.search.alphaSearch'>,
+ S<'response.search.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>
+> => compose('searchServe', [
+ emitAlphaSearch,
+ // Unconditional, as everywhere: a run that searched locally reached no upstream and bills
+ // for none, and saying so is a row that names no billed entity rather than no row.
+ writeSettlement(handedUp => isFailure((handedUp as { 'response.search.alphaSearch'?: unknown })['response.search.alphaSearch'])),
+ ...(execution.kind === 'local'
+ ? [parseSearchOperations, executeSearchOperations]
+ : [callSearchUpstream(execution)]),
+]);
diff --git a/packages/gateway/src/data-plane/alpha-search/protocol.ts b/packages/gateway/src/data-plane/alpha-search/protocol.ts
new file mode 100644
index 0000000000..f2682bdf6f
--- /dev/null
+++ b/packages/gateway/src/data-plane/alpha-search/protocol.ts
@@ -0,0 +1,115 @@
+// Codex's `/alpha/search` protocol. The request carries model/session context plus a
+// command object; the response is `{ encrypted_output, output, results? }`.
+// https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/search.rs#L8-L29
+// https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/search.rs#L297-L305
+
+import { z } from 'zod';
+
+import { isJsonObject } from '../../shared/json-helpers.ts';
+import { maxResultsForContextSize, type WebSearchFilters } from '../tools/web-search/operations.ts';
+
+const domainListSchema = z.array(z.string());
+
+// This is OpenAI Codex's complete SearchSettings shape. The loose object keeps future fields
+// intact; local execution consumes the routing fields it implements while accepting Codex
+// metadata such as allowed_callers.
+// https://github.com/openai/codex/blob/2f19a57704fb7b1db032bc38cf995034254eaebb/codex-rs/codex-api/src/search.rs#L215-L295
+const searchSettingsSchema = z.looseObject({
+ filters: z.looseObject({
+ allowed_domains: domainListSchema.optional(),
+ blocked_domains: domainListSchema.optional(),
+ }).optional(),
+ user_location: z.looseObject({
+ type: z.literal('approximate').optional(),
+ city: z.string().optional(),
+ region: z.string().optional(),
+ country: z.string().optional(),
+ timezone: z.string().optional(),
+ }).optional(),
+ search_context_size: z.enum(['low', 'medium', 'high']).optional(),
+ image_settings: z.looseObject({
+ max_results: z.number().int().nonnegative().optional(),
+ caption: z.boolean().optional(),
+ }).optional(),
+ allowed_callers: z.array(z.enum(['direct', 'shell', 'code_interpreter'])).optional(),
+ external_web_access: z.union([
+ z.boolean(),
+ z.enum(['cached', 'indexed', 'live']),
+ ]).optional(),
+});
+
+// `commands` is validated only as "an object" — the per-kind arrays are parsed by the shared
+// command engine. `looseObject` preserves every OpenAI command and nested parameter so a
+// relayed request stays lossless and the local capability gate can reject unimplemented
+// fields explicitly.
+// https://github.com/openai/codex/blob/2f19a57704fb7b1db032bc38cf995034254eaebb/codex-rs/codex-api/src/search.rs#L31-L213
+export const alphaSearchRequestSchema = z.looseObject({
+ commands: z.looseObject({}).optional(),
+ settings: searchSettingsSchema.optional(),
+});
+
+export type AlphaSearchRequest = z.infer;
+
+export interface AlphaSearchResponse {
+ /** Search state Codex carries forward into a later turn, opaque to everyone between the
+ * two ends. Local execution has no state to carry, so it is null. */
+ encryptedOutput: string | null;
+ /** The model-facing text. Codex renders this and nothing else. */
+ output: string;
+ /** Result DTOs handed to clients out of band from `output`. Codex keeps them opaque so
+ * newer result variants stay forward-compatible, and so does this. */
+ results?: readonly unknown[];
+}
+
+/** Why a body could not be read as this protocol, so the ending stage can say so in the
+ * synthesized failure rather than reporting a bare status. */
+export type AlphaSearchResponseVerdict =
+ | { readonly ok: true; readonly response: AlphaSearchResponse }
+ | { readonly ok: false; readonly reason: string };
+
+export const parseAlphaSearchResponse = (body: unknown): AlphaSearchResponseVerdict => {
+ if (!isJsonObject(body)) return { ok: false, reason: 'the body is not a JSON object' };
+ const { encrypted_output: encryptedOutput, output, results } = body;
+ if (typeof output !== 'string') return { ok: false, reason: '`output` is missing or not a string' };
+ // `encrypted_output` is `Option` without a serde default, so Codex's own client
+ // requires the key and accepts null in it. Demanding the same here surfaces a truncated
+ // upstream at the gateway instead of as a deserialization failure inside the client.
+ if (encryptedOutput !== null && typeof encryptedOutput !== 'string') {
+ return { ok: false, reason: '`encrypted_output` is missing or is neither a string nor null' };
+ }
+ if (results !== undefined && results !== null && !Array.isArray(results)) {
+ return { ok: false, reason: '`results` is neither absent nor an array' };
+ }
+ return {
+ ok: true,
+ response: {
+ encryptedOutput,
+ output,
+ ...(Array.isArray(results) ? { results } : {}),
+ },
+ };
+};
+
+export const renderAlphaSearchResponse = (response: AlphaSearchResponse): Record => ({
+ encrypted_output: response.encryptedOutput,
+ output: response.output,
+ ...(response.results === undefined ? {} : { results: response.results }),
+});
+
+export const webSearchFiltersFromSettings = (settings: AlphaSearchRequest['settings']): WebSearchFilters => {
+ const filters: WebSearchFilters = {
+ maxResults: maxResultsForContextSize(settings?.search_context_size),
+ };
+ if (settings?.filters?.allowed_domains) filters.allowedDomains = settings.filters.allowed_domains;
+ if (settings?.filters?.blocked_domains) filters.blockedDomains = settings.filters.blocked_domains;
+ const loc = settings?.user_location;
+ if (loc && (loc.city !== undefined || loc.region !== undefined || loc.country !== undefined || loc.timezone !== undefined)) {
+ filters.userLocation = {
+ ...(loc.city !== undefined ? { city: loc.city } : {}),
+ ...(loc.region !== undefined ? { region: loc.region } : {}),
+ ...(loc.country !== undefined ? { country: loc.country } : {}),
+ ...(loc.timezone !== undefined ? { timezone: loc.timezone } : {}),
+ };
+ }
+ return filters;
+};
diff --git a/packages/gateway/src/data-plane/alpha-search/routes.ts b/packages/gateway/src/data-plane/alpha-search/routes.ts
index 141fbc735d..0f3cbd9029 100644
--- a/packages/gateway/src/data-plane/alpha-search/routes.ts
+++ b/packages/gateway/src/data-plane/alpha-search/routes.ts
@@ -1,158 +1,26 @@
-// Codex `/alpha/search` compatibility endpoint. The private request carries
-// model/session context plus a command object; the response is
-// `{ encrypted_output?, output, results? }`.
-// https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/search.rs#L8-L29
-// https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/search.rs#L297-L305
+// Codex `/alpha/search` compatibility endpoint. The protocol itself — the request
+// schema, the response shape, and the projection of Codex's settings onto the
+// gateway's search filters — lives in `protocol.ts`; the chain that serves it is
+// `pipeline.ts`, and `http.ts` is the seam between the two.
+//
// Clients append `alpha/search` to an OpenAI-compatible provider base. The
// aliases below cover Floway's general root and `/v1` base conventions.
// https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/endpoint/search.rs#L31-L47
//
-// In the default mode, Floway executes supported commands through the general
-// configured search provider and renders a local `{ encrypted_output: null,
-// output }` response. Passthrough mode instead returns the selected Codex or
-// Custom provider response verbatim, preserving its optional structured data.
-//
-// The shared data-plane auth middleware guards every alias; this handler reads
-// the resolved API key for per-key search-usage accounting.
+// The shared data-plane auth middleware guards every alias; the handler reads the
+// resolved API key through the run's own prologue.
import type { Hono } from 'hono';
-import { z } from 'zod';
-import { type AuthVars, apiKeyFromContext, effectiveUpstreamIdsFromContext } from '../../middleware/auth.ts';
-import { type CtxWithJson, zValidator } from '../../middleware/zod-validator.ts';
-import { backgroundSchedulerFromContext } from '../../runtime/background.ts';
-import { getRuntimeLocation } from '../../runtime/runtime-info.ts';
+import { alphaSearch } from './http.ts';
+import type { AuthVars } from '../../middleware/auth.ts';
import { mountPublicRoute } from '../public-route.ts';
-import { relayFetchedResponse } from '../tools/web-search/alpha-search/relay-response.ts';
-import { resolveAlphaSearchDispatcher } from '../tools/web-search/alpha-search/upstream.ts';
-import { loadWebSearchConfig } from '../tools/web-search/config.ts';
-import { assertLocalWebSearchSupport, executeOperationToText, maxResultsForContextSize, parseWebSearchOperations, startBatchFetch, UnsupportedLocalWebSearchFeatureError, type WebSearchExecutionSession, type WebSearchFilters } from '../tools/web-search/operations.ts';
-import { resolveConfiguredWebSearchProvider } from '../tools/web-search/provider.ts';
-import type { ConfiguredWebSearchProvider } from '../tools/web-search/types.ts';
import { PUBLIC_DATA_PLANE_ROUTES } from '@floway-dev/protocols/common';
-const domainListSchema = z.array(z.string());
-
-// This is OpenAI Codex's complete SearchSettings shape. The loose object keeps
-// future fields intact for passthrough; local providers consume the routing
-// fields they implement while accepting Codex metadata such as allowed_callers.
-// https://github.com/openai/codex/blob/2f19a57704fb7b1db032bc38cf995034254eaebb/codex-rs/codex-api/src/search.rs#L215-L295
-const searchSettingsSchema = z.looseObject({
- filters: z.looseObject({
- allowed_domains: domainListSchema.optional(),
- blocked_domains: domainListSchema.optional(),
- }).optional(),
- user_location: z.looseObject({
- type: z.literal('approximate').optional(),
- city: z.string().optional(),
- region: z.string().optional(),
- country: z.string().optional(),
- timezone: z.string().optional(),
- }).optional(),
- search_context_size: z.enum(['low', 'medium', 'high']).optional(),
- image_settings: z.looseObject({
- max_results: z.number().int().nonnegative().optional(),
- caption: z.boolean().optional(),
- }).optional(),
- allowed_callers: z.array(z.enum(['direct', 'shell', 'code_interpreter'])).optional(),
- external_web_access: z.union([
- z.boolean(),
- z.enum(['cached', 'indexed', 'live']),
- ]).optional(),
-});
-
-// `commands` is validated only as "an object" — the per-kind arrays are
-// parsed by the shared command engine. `looseObject` preserves every OpenAI
-// command and nested parameter so passthrough stays lossless and the local
-// capability gate can reject unimplemented fields explicitly.
-const alphaSearchRequestSchema = z.looseObject({
- commands: z.looseObject({}).optional(),
- settings: searchSettingsSchema.optional(),
-});
-
-type AlphaSearchRequest = z.infer;
-
-const filtersFromSettings = (settings: AlphaSearchRequest['settings']): WebSearchFilters => {
- const filters: WebSearchFilters = {
- maxResults: maxResultsForContextSize(settings?.search_context_size),
- };
- if (settings?.filters?.allowed_domains) filters.allowedDomains = settings.filters.allowed_domains;
- if (settings?.filters?.blocked_domains) filters.blockedDomains = settings.filters.blocked_domains;
- const loc = settings?.user_location;
- if (loc && (loc.city !== undefined || loc.region !== undefined || loc.country !== undefined || loc.timezone !== undefined)) {
- filters.userLocation = {
- ...(loc.city !== undefined ? { city: loc.city } : {}),
- ...(loc.region !== undefined ? { region: loc.region } : {}),
- ...(loc.country !== undefined ? { country: loc.country } : {}),
- ...(loc.timezone !== undefined ? { timezone: loc.timezone } : {}),
- };
- }
- return filters;
-};
-
-const alphaSearch = async (c: CtxWithJson): Promise => {
- const body = c.req.valid('json');
- const webSearchConfig = await loadWebSearchConfig();
- if (webSearchConfig.passthroughOpenAiSearch.enabled) {
- const dispatcher = await resolveAlphaSearchDispatcher({
- config: webSearchConfig.passthroughOpenAiSearch,
- upstreamIds: effectiveUpstreamIdsFromContext(c),
- scheduler: backgroundSchedulerFromContext(c),
- runtimeLocation: getRuntimeLocation(c.req.raw),
- });
- const headers = new Headers();
- const turnMetadata = c.req.header('x-codex-turn-metadata');
- if (turnMetadata !== undefined) headers.set('x-codex-turn-metadata', turnMetadata);
- const response = await dispatcher(body, c.req.raw.signal, headers);
- return relayFetchedResponse(response);
- }
-
- try {
- assertLocalWebSearchSupport(body.commands ?? {});
- } catch (error) {
- if (error instanceof UnsupportedLocalWebSearchFeatureError) {
- return c.json({ encrypted_output: null, output: error.message });
- }
- throw error;
- }
-
- let configuredProvider: Promise | undefined;
- const session: WebSearchExecutionSession = {
- getProvider: () => {
- configuredProvider ??= Promise.resolve(resolveConfiguredWebSearchProvider(webSearchConfig));
- return configuredProvider;
- },
- filters: filtersFromSettings(body.settings),
- apiKeyId: apiKeyFromContext(c).id,
- pageCache: new Map(),
- // Codex renders `output` as plain text; the search-action sources list
- // is an OpenAI Responses wire concern with no place here.
- includeSearchActionSources: false,
- signal: c.req.raw.signal,
- };
-
- const parsed = parseWebSearchOperations(body.commands ?? {});
- if (parsed.kind !== 'ops' || parsed.ops.length === 0) {
- return c.json({
- encrypted_output: null,
- output: 'No web search commands were provided. Populate at least one of `search_query`, `open`, or `find`.',
- });
- }
-
- // One batched provider.fetchPage covers every open/find URL; each op then
- // renders its own text block. The shared parser's canonical order is
- // search_query → open → find, preserving array order within each command
- // kind.
- const batch = await startBatchFetch(parsed, session);
- const blocks = await Promise.all(parsed.ops.map(op => executeOperationToText(op, session, batch)));
-
- return c.json({ encrypted_output: null, output: blocks.join('\n\n') });
-};
-
type AlphaSearchRoute = typeof PUBLIC_DATA_PLANE_ROUTES.alphaSearch | typeof PUBLIC_DATA_PLANE_ROUTES.codexAlphaSearch;
export const mountAlphaSearchRoute = (app: Hono<{ Variables: AuthVars }>, route: AlphaSearchRoute) => {
- mountPublicRoute(route, (method, path) => app.on(method, path, zValidator('json', alphaSearchRequestSchema), alphaSearch));
+ mountPublicRoute(route, (method, path) => app.on(method, path, alphaSearch));
};
export const mountAlphaSearchRoutes = (app: Hono<{ Variables: AuthVars }>) => {
diff --git a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts
index 924113d2f3..64edfa92f7 100644
--- a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts
+++ b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts
@@ -4,7 +4,7 @@ import { wrapOpenAIResponsesClientEgress } from './client-output.ts';
import { createOpenAIResponsesWsSession } from './items/store.ts';
import { PreviousResponseNotFoundError } from './serve-prep.ts';
import { openaiResponsesServe } from './serve.ts';
-import type { DumpAccumulator } from '../../../dump/accumulator.ts';
+import type { TurnDump } from '../../../dump/turn-dump.ts';
import { apiKeyFromContext, authenticateApiKey, type AuthedContext } from '../../../middleware/auth.ts';
import { backgroundSchedulerFromContext } from '../../../runtime/background.ts';
import { inboundHeaders } from '../../shared/inbound-headers.ts';
@@ -716,7 +716,7 @@ const sendError = (
status: number,
error: Record,
eventId?: string,
- dump?: DumpAccumulator | null,
+ dump?: TurnDump | null,
): void => {
sendJson(socket, { type: 'error', status, error }, eventId, dump);
};
@@ -733,14 +733,14 @@ const sendOpenAIResponsesEvent = (
socket: OpenAIResponsesWebSocketSocket,
event: ClientOpenAIResponsesStreamEvent,
eventId?: string,
- dump?: DumpAccumulator | null,
+ dump?: TurnDump | null,
): boolean => sendJson(socket, event, eventId, dump);
const sendJson = (
socket: OpenAIResponsesWebSocketSocket,
value: unknown,
eventId?: string,
- dump?: DumpAccumulator | null,
+ dump?: TurnDump | null,
): boolean => {
if (socket.readyState !== 1) return false;
const payload = eventId === undefined || !value || typeof value !== 'object'
diff --git a/packages/gateway/src/data-plane/openai-audio/http.ts b/packages/gateway/src/data-plane/openai-audio/http.ts
index 9870faf64f..ee5b286054 100644
--- a/packages/gateway/src/data-plane/openai-audio/http.ts
+++ b/packages/gateway/src/data-plane/openai-audio/http.ts
@@ -1,23 +1,26 @@
-// POST /v1/audio/transcriptions — buffered OpenAI-compatible multipart
-// transcription. The full body is parsed before routing because multipart
-// field order is unconstrained; providers receive ordered semantic entries
-// and rebuild a fresh body per candidate.
+// POST /v1/audio/transcriptions, served through the pipeline.
+//
+// The multipart body is read and parsed before routing because field order is unconstrained
+// and every candidate builds a fresh body from the entries. What the handler decides for
+// itself is written in that form: which of the six renderings the client asked for, and
+// whether the request streams — the run has to be opened knowing the second.
// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L714-L1040
import type { Context } from 'hono';
-import { respondOpenAIAudioTranscription } from './respond.ts';
-import { backgroundSchedulerFromContext } from '../../runtime/background.ts';
-import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../shared/gateway-ctx.ts';
-import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts';
-import { readRequestBody, takeRequestBody } from '../shared/request-body.ts';
+import { openaiAudioTranscriptionServePipeline } from './pipeline.ts';
+import { isFrames, openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts';
+import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts';
+import { move } from '@floway-dev/pipeline';
import { isMultipartFormDataMediaType } from '@floway-dev/protocols/common';
+import { parseOpenAIAudioTranscriptionResponseFormat, type OpenAIAudioTranscriptionResponseFormat } from '@floway-dev/protocols/openai-audio';
import type { OpenAIAudioTranscriptionFormEntry } from '@floway-dev/provider';
type PreparedOpenAIAudioTranscription =
| {
readonly type: 'ok';
readonly model: string;
+ readonly responseFormat: OpenAIAudioTranscriptionResponseFormat;
readonly wantsStream: boolean;
readonly entries: readonly OpenAIAudioTranscriptionFormEntry[];
}
@@ -43,6 +46,16 @@ const prepareOpenAIAudioTranscription = async (bytes: Uint8Array, contentType: s
if (files.length === 0 || files.some(file => !(file instanceof File))) {
return { type: 'invalid', message: 'OpenAI Audio Transcriptions request body must include a file upload.' };
}
+ const declaredFormat = form.get('response_format');
+ if (declaredFormat !== null && typeof declaredFormat !== 'string') {
+ return { type: 'invalid', message: 'OpenAI Audio Transcriptions response_format must be a text field.' };
+ }
+ let responseFormat: OpenAIAudioTranscriptionResponseFormat;
+ try {
+ responseFormat = parseOpenAIAudioTranscriptionResponseFormat(declaredFormat ?? undefined);
+ } catch (error) {
+ return { type: 'invalid', message: error instanceof Error ? error.message : String(error) };
+ }
const entries: OpenAIAudioTranscriptionFormEntry[] = [];
for (const [name, value] of form.entries()) {
@@ -51,35 +64,62 @@ const prepareOpenAIAudioTranscription = async (bytes: Uint8Array, contentType: s
return {
type: 'ok',
model,
+ responseFormat,
wantsStream: form.get('stream') === 'true',
entries,
};
};
+/** The client is sent bytes on every path, never a string, so nothing downstream of here
+ * guesses a media type: a `Response` built over a string is labelled `text/plain` by the
+ * platform, and an upstream that declared no media type has not asked for that. */
+const bodyOf = (rendered: Record | Uint8Array): Uint8Array =>
+ rendered instanceof Uint8Array ? rendered : new TextEncoder().encode(JSON.stringify(rendered));
+
+/**
+ * The epilogue: what the run answered with, as a response.
+ *
+ * It is `serveThrough` like every other family, and it took two things at the seam to make it
+ * one. A carried document goes out under the upstream's own media type, including the upstream
+ * that declared none — so `Rendered.contentType` carries a null and the seam answers without
+ * the header rather than inventing one. And a transcription's stream states its own outcome in
+ * its last event, which the family's meter reads because it is the one place that knows — so
+ * what `DeferredUsage` hands back is both what was billed and whether the stream finished,
+ * settled from one promise while the request is still live.
+ */
export const openaiAudioTranscriptions = async (c: Context): Promise => {
- const requestBody = await readRequestBody(c);
- const request = await prepareOpenAIAudioTranscription(requestBody.bytes, c.req.header('content-type'));
- const ctx = createGatewayCtxFromHono(c, {
- wantsStream: request.type === 'ok' ? request.wantsStream : false,
- requestBody: takeRequestBody(requestBody),
- backgroundScheduler: backgroundSchedulerFromContext(c),
- });
+ const ingress = await readIngress(c);
+ const request = await prepareOpenAIAudioTranscription(ingress.body.bytes, c.req.header('content-type'));
if (request.type === 'invalid') {
- ctx.dump?.error('gateway');
- return finalizeGatewayResponse(ctx, passthroughApiError(c, request.message, 400));
+ // A request the gateway could not read never reaches a pipeline: there is no model to
+ // resolve and no attempt to make, so there is nothing for a run to record.
+ const refused = openPrologue(c, ingress, { wantsStream: false });
+ refused.gateway.dump?.error('gateway');
+ return finalizeGatewayResponse(
+ refused.gateway,
+ Response.json({ error: { message: request.message, type: 'api_error' } }, { status: 400 }),
+ );
}
- ctx.dump?.requestedModel(request.model);
- const response = await passthroughServe({
+ const prologue = openPrologue(c, ingress, { wantsStream: request.wantsStream, model: request.model });
+
+ return await serveThrough(
c,
- ctx,
- sourceApi: '/audio/transcriptions',
- operation: 'audio_transcription',
- model: request.model,
- kind: 'transcription',
- modelServesEndpoint: model => model.endpoints.openaiAudioTranscriptions !== undefined,
- call: (provider, model, opts) => provider.instance.callOpenAIAudioTranscriptions(model, { entries: request.entries }, ctx.abortSignal, opts),
- response: { format: 'strategy', respond: respondOpenAIAudioTranscription },
- });
- return finalizeGatewayResponse(ctx, response);
+ prologue,
+ openaiAudioTranscriptionServePipeline,
+ move({
+ 'ingress.http.headers': prologue.headers,
+ 'ingress.openaiAudioTranscription.responseFormat': request.responseFormat,
+ 'request.openaiAudioTranscription.form': request.entries,
+ 'serve.model': request.model,
+ }) as never,
+ facts => {
+ const rendered = facts['response.openaiAudioTranscription.rendered'];
+ if (isFrames(rendered)) return { frames: rendered };
+ // The upstream's own media type, or none where it declared none: a document this
+ // gateway carried rather than wrote is not one it can describe.
+ return { body: bodyOf(rendered) as BodyInit, contentType: facts['response.openaiAudioTranscription.mediaType'] };
+ },
+ facts => facts['response.openaiAudioTranscription.streamedOutcome'],
+ );
};
diff --git a/packages/gateway/src/data-plane/openai-audio/pipeline.ts b/packages/gateway/src/data-plane/openai-audio/pipeline.ts
new file mode 100644
index 0000000000..de1d0c12cf
--- /dev/null
+++ b/packages/gateway/src/data-plane/openai-audio/pipeline.ts
@@ -0,0 +1,470 @@
+// OpenAI Audio Transcriptions as a pipeline. The family whose answer is not one document but
+// six: `response_format` picks between three JSON objects and three text documents, `stream`
+// picks a sequence of events instead of any of them, and each is its own media type. That
+// is the whole reason this endpoint needed a response strategy of its own, and it is what
+// one canonical fact plus one media-type fact dissolve.
+//
+// The shape:
+//
+// emitOpenAIAudioTranscription the edge: writes the answer back in the rendering
+// the client asked for, SSE framing included
+// resolveCandidates narrows to the upstreams that expose the endpoint
+// failover runs what follows once per candidate
+// callOpenAIAudioTranscriptionUpstream the ending: dials, reads what came back, and
+// provides the answer plus what is billable
+
+import { recordStream, streamReferenceOf } from '../../dump/turn-dump.ts';
+import type { UsageQuantities } from '../../repo/types.ts';
+import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts';
+import { isFailure } from '../pipeline/facts.ts';
+import type { GatewayServices } from '../pipeline/services.ts';
+import { writeSettlement } from '../pipeline/settlement.ts';
+import { failover, resolveCandidates } from '../pipeline/stages.ts';
+import { dialFailure } from '../pipeline/upstream-body.ts';
+import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts';
+import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts';
+import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts';
+import { compose, defer, defineStage, move, own, type Deferred, type Logger, type Owned, type Pipeline } from '@floway-dev/pipeline';
+import { eventFrame, isEventStreamMediaType, parseDecimalString, parseSSEStream, renderErrorEnvelope, sseFrame, type SseFrame } from '@floway-dev/protocols/common';
+import {
+ isOpenAIAudioTranscriptionDoneEvent,
+ parseOpenAIAudioTranscription,
+ parseOpenAIAudioTranscriptionStreamEvent,
+ parseOpenAIAudioTranscriptionStreamUsage,
+ parseOpenAIAudioTranscriptionUsage,
+ renderOpenAIAudioTranscription,
+ type OpenAIAudioTranscriptionResponseFormat,
+ type OpenAIAudioTranscriptionStreamEvent,
+ type OpenAIAudioTranscriptionUsage,
+ type CanonicalOpenAIAudioTranscription,
+} from '@floway-dev/protocols/openai-audio';
+import { providerModelOf, type OpenAIAudioTranscriptionFormEntry, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider';
+
+/** The answer while it is still the upstream's, one event at a time. It is a view and not a
+ * resource: what owns the connection is `response.http.body`, which is where release and
+ * failover's ownership both read.
+ *
+ * A view is a wrapper around the generator rather than the generator itself, which is what
+ * says where the resource is: the upstream's body at `response.http.body`, claimed with
+ * `own()`, and nothing else here. */
+export type OpenAIAudioTranscriptionEvents = AsyncIterable;
+
+const viewOf = (events: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => events });
+
+/** What settling this run will be told once the events run out: what the upstream metered,
+ * and whether the transcript ever finished. */
+export interface OpenAIAudioTranscriptionStreamOutcome {
+ readonly billable: readonly BillableEntity[];
+ /** An upstream that stopped before `transcript.text.done` answered 200 and then did not
+ * finish what it started, which is a failed request however much of the transcript
+ * reached the client. */
+ readonly failed: boolean;
+}
+
+/** OpenAI Audio Transcriptions' own keys, extending the shared space by intersection. */
+export interface OpenAIAudioTranscriptionFacts extends GatewayFacts {
+ /** Which of the six renderings the client asked for. It belongs to the ingress and stays
+ * put: the same value travels to the upstream inside the form, so the rendering the
+ * answer arrives in is the rendering the answer is written back in — and a text document
+ * does not say which of the three it is, so nothing else could decide. */
+ 'ingress.openaiAudioTranscription.responseFormat': OpenAIAudioTranscriptionResponseFormat;
+ /** The multipart form as ordered semantic entries. The body is parsed before routing
+ * because field order is unconstrained, and every candidate builds a fresh body from
+ * these, so a retry never reuses a consumed one. The bytes the client sent are recorded
+ * at `ingress.http.body`, which is where a dump reads the upload itself. */
+ 'request.openaiAudioTranscription.form': readonly OpenAIAudioTranscriptionFormEntry[];
+ /** The one transcription, whichever rendering carried it — or the events it is arriving
+ * as, or the failure that came instead. A stream, a value and a failure sit at one key:
+ * telling them apart is reading a value, and each stage does that where it needs to. */
+ 'response.openaiAudioTranscription.canonical': CanonicalOpenAIAudioTranscription | OpenAIAudioTranscriptionEvents | Failure;
+ /** What the answer goes out under. A media type is upstream-owned — OpenAI answers
+ * `text`, `srt` and `vtt` under one media type and other upstreams label them apart, and
+ * the document beneath it is the upstream's own either way — so the upstream's travels,
+ * and the edge names one only where the gateway wrote the body out of nothing the
+ * upstream sent. `null` is an upstream that declared none, and it stays `null`: labelling
+ * a body nobody described is a statement this gateway has no grounds to make. */
+ 'response.openaiAudioTranscription.mediaType': string | null;
+ /** What the stream will have come to by the time the events run out, and `null` on every
+ * path that does not stream. A streamed transcription states its usage in the terminal
+ * event, which is after this run has answered — so the numbers cannot be in
+ * `response.usage.billable`, which says what had been reported when the ending stage
+ * handed up: the entity, and no quantities. Settling from this is the epilogue's job,
+ * after the drain. */
+ 'response.openaiAudioTranscription.streamedOutcome': Deferred | null;
+ /** What the client is actually sent — a JSON object, the upstream's own document, or the
+ * SSE frames of a stream. The edge provides it, so a dump shows what the client received
+ * rather than the gateway's own reading of it. */
+ 'response.openaiAudioTranscription.rendered': Record | Uint8Array | AsyncIterable;
+}
+
+type A = { [P in K]: OpenAIAudioTranscriptionFacts[P] };
+
+const isEvents = (answer: CanonicalOpenAIAudioTranscription | OpenAIAudioTranscriptionEvents): answer is OpenAIAudioTranscriptionEvents =>
+ Symbol.asyncIterator in answer;
+
+/**
+ * The outermost edge. Writes the answer back in the rendering the client asked for, and names
+ * a media type for the one body the gateway wrote out of nothing the upstream sent — its
+ * error envelope. Every other answer goes out under the media type it arrived with, because
+ * for a document that is carried rather than rewritten the upstream's label is the only true
+ * description there is.
+ *
+ * SSE framing is produced here and nowhere else — below this stage the answer is parsed
+ * events, so the same assembly would serve another transport by rendering differently at
+ * this one point. The upstream's own `event:` label is not carried: it is transport, and
+ * OpenAI's clients read this endpoint's frames by their payload's `type` rather than by the
+ * label. https://github.com/openai/openai-python/blob/10ee3f0da2ac6f93345c1204bd7bb1a2faa79ff2/src/openai/_streaming.py#L61-L107
+ */
+const emitOpenAIAudioTranscription = defineStage<
+ A<'ingress.openaiAudioTranscription.responseFormat'>,
+ A<'ingress.openaiAudioTranscription.responseFormat'>,
+ A<'ingress.openaiAudioTranscription.responseFormat' | 'response.openaiAudioTranscription.canonical' | 'response.openaiAudioTranscription.mediaType'>
+ & { 'response.http.headers': readonly (readonly [string, string])[] },
+ A<'response.openaiAudioTranscription.rendered' | 'response.openaiAudioTranscription.mediaType'>
+ & { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[] }
+>({
+ name: 'emitOpenAIAudioTranscription',
+ through: {
+ request: {
+ needs: ['ingress.openaiAudioTranscription.responseFormat'],
+ consumes: [],
+ provides: [],
+ },
+ response: {
+ needs: ['response.openaiAudioTranscription.canonical', 'response.openaiAudioTranscription.mediaType', 'response.http.headers'],
+ consumes: ['response.openaiAudioTranscription.canonical', 'response.http.headers'],
+ provides: ['response.openaiAudioTranscription.rendered', 'response.openaiAudioTranscription.mediaType', 'response.http.status', 'response.http.headers'],
+ },
+ },
+ execute: async (facts, next) => {
+ const back = await next(facts);
+ const { 'response.openaiAudioTranscription.canonical': answer, 'response.http.headers': headers, ...rest } = back;
+ // Vendor traces and quota state stay visible; what an intermediary must strip, and what
+ // would misdescribe a body this gateway serialized itself, does not. A filter that removed
+ // nothing hands the same array on, so the record shows no change where none happened.
+ const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name));
+ const forClient = forwardable.length === headers.length ? headers : move(forwardable);
+
+ if (isFailure(answer)) {
+ return {
+ ...rest,
+ 'response.http.headers': forClient,
+ 'response.http.status': answer.status,
+ 'response.openaiAudioTranscription.mediaType': 'application/json',
+ 'response.openaiAudioTranscription.rendered': move(renderErrorEnvelope(answer.message, answer.body)),
+ };
+ }
+ // Everything that reaches here answered. The same key carried the upstream's own status
+ // further down; re-providing it is what makes the top of the record the response the
+ // client gets rather than the one the upstream gave.
+ return {
+ ...rest,
+ 'response.http.headers': forClient,
+ 'response.http.status': 200,
+ 'response.openaiAudioTranscription.rendered': move(isEvents(answer)
+ ? renderSSE(answer)
+ : renderOpenAIAudioTranscription(back['ingress.openaiAudioTranscription.responseFormat'], answer)),
+ };
+ },
+});
+
+const renderSSE = (events: OpenAIAudioTranscriptionEvents): AsyncIterable => ({
+ // The frames the client reads are a reframing of the events the record holds, so this key
+ // points at that same stream rather than at nothing.
+ ...streamReferenceOf(events),
+ [Symbol.asyncIterator]: () => (async function* () {
+ for await (const event of events) yield sseFrame(JSON.stringify(event));
+ })(),
+});
+
+/**
+ * The ending. It dials, reads what came back in the rendering the request asked for, and
+ * provides the answer, the raw HTTP response beneath it, and what the call is billable for.
+ * A failure is a value: a 429 here is what an earlier stage fails over, and so is a dial that
+ * never reached anyone. A 200 is never one of them — this endpoint carries the document the
+ * upstream sent rather than serializing one from what it read, so a body no reading could
+ * open is still that upstream's answer and there is nothing to try the next candidate for.
+ */
+const callOpenAIAudioTranscriptionUpstream = defineStage<
+ A<'ingress.openaiAudioTranscription.responseFormat' | 'request.openaiAudioTranscription.form' | 'route.attempt' | 'ingress.http.headers'>,
+ A<'response.openaiAudioTranscription.canonical' | 'response.openaiAudioTranscription.mediaType' | 'response.openaiAudioTranscription.streamedOutcome'>
+ & { 'response.usage.billable': readonly BillableEntity[]; 'response.http.status': number;
+ 'response.http.headers': readonly (readonly [string, string])[];
+ 'response.http.body': ReadableStream & Owned; },
+ GatewayServices
+>({
+ name: 'callOpenAIAudioTranscriptionUpstream',
+ return: {
+ provides: [
+ 'response.openaiAudioTranscription.canonical',
+ 'response.openaiAudioTranscription.mediaType',
+ 'response.openaiAudioTranscription.streamedOutcome',
+ 'response.usage.billable',
+ 'response.http.status',
+ 'response.http.headers',
+ 'response.http.body',
+ ],
+ },
+ execute: async (facts, use) => {
+ const candidate = use.resolveAttempt(facts['route.attempt']);
+ // Attribution is set before the dial, so an attempt that never completes still names the
+ // candidate it was made against rather than the one tried before it.
+ use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'audio_transcription');
+
+ let result;
+ try {
+ result = await candidate.provider.instance.callOpenAIAudioTranscriptions(
+ providerModelOf(candidate),
+ { entries: facts['request.openaiAudioTranscription.form'] },
+ use.gateway.abortSignal,
+ // The client's own headers reach the upstream from the record, not from a live request
+ // object: what a provider is allowed to forward is filtered per provider, and the dump
+ // shows what was there to filter.
+ buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))),
+ );
+ } catch (error) {
+ use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) });
+ // A dial that never completed reached no upstream, so nothing was billed and there are
+ // no headers to carry. What it leaves behind is the performance row settlement writes.
+ return move({
+ ...facts,
+ 'response.openaiAudioTranscription.canonical': dialFailure(error),
+ 'response.openaiAudioTranscription.mediaType': null,
+ 'response.openaiAudioTranscription.streamedOutcome': null,
+ 'response.usage.billable': [],
+ 'response.http.status': 502,
+ 'response.http.headers': [],
+ 'response.http.body': spentBody(null),
+ });
+ }
+ const identity = telemetryModelIdentity(candidate, result.modelKey);
+ const format = facts['ingress.openaiAudioTranscription.responseFormat'];
+ const status = result.response.status;
+ const mediaType = result.response.headers.get('content-type');
+ const headers = move([...result.response.headers] as readonly (readonly [string, string])[]);
+ // An upstream that was called and reported nothing is a different situation from one
+ // that reported zero, so the entity is present with no quantities.
+ const called: readonly BillableEntity[] = [{ identity, quantities: {} }];
+
+ if (!result.response.ok) {
+ use.log.warn('upstream refused', { status });
+ // An upstream error body is JSON like any other body, and reading it here is also what
+ // leaves a losing attempt with nothing open behind it.
+ return move({
+ ...facts,
+ 'response.openaiAudioTranscription.canonical': await refusal(status, result.response),
+ 'response.openaiAudioTranscription.mediaType': mediaType,
+ 'response.openaiAudioTranscription.streamedOutcome': null,
+ 'response.usage.billable': called,
+ 'response.http.status': status,
+ 'response.http.headers': headers,
+ 'response.http.body': spentBody(result.response.body),
+ });
+ }
+
+ // A streamed answer is the one the client asked for with `stream`, and it is the media
+ // type that says one arrived: an upstream that ignores `stream` — `whisper-1` does —
+ // answers in the rendering `response_format` named instead.
+ // https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L36325-L36336
+ if (isEventStreamMediaType(mediaType)) {
+ if (result.response.body === null) {
+ return move({
+ ...facts,
+ 'response.openaiAudioTranscription.canonical': { status: 502, message: 'Upstream returned a streaming response with no body.' },
+ 'response.openaiAudioTranscription.mediaType': mediaType,
+ 'response.openaiAudioTranscription.streamedOutcome': null,
+ 'response.usage.billable': called,
+ 'response.http.status': status,
+ 'response.http.headers': headers,
+ 'response.http.body': spentBody(null),
+ });
+ }
+ // What the upstream metered, and whether the transcript finished, are both observed
+ // here — closest to the upstream and on the protocol it spoke — by folding the events
+ // as they pass, so the reading costs one pass and the client's own stream drives it.
+ const metered = meterEvents(result.response.body, identity, use.gateway.abortSignal, use.log);
+ return move({
+ ...facts,
+ // This protocol's stream is bare events rather than protocol frames, so the record is
+ // told how one becomes a frame instead of being left to assume.
+ 'response.openaiAudioTranscription.canonical': recordStream(metered.events, use.gateway.dump, eventFrame),
+ 'response.openaiAudioTranscription.mediaType': mediaType,
+ 'response.openaiAudioTranscription.streamedOutcome': metered.outcome,
+ 'response.usage.billable': called,
+ 'response.http.status': status,
+ 'response.http.headers': headers,
+ // Releasing this body is reading those events to the end: they are one reader over
+ // one connection, and a second reader is not something a `ReadableStream` allows.
+ 'response.http.body': own(result.response.body, async (): Promise => { for await (const _event of metered.events) { /* to end of stream */ } }),
+ });
+ }
+
+ // A 2xx body is the answer whether or not this endpoint could read it, because what the
+ // client is sent is the document that arrived and not something serialized from a parse.
+ // So there is nothing here to fail over from: the reading feeds the record and the usage
+ // row, and the bytes travel either way.
+ const read = readTranscription(format, new Uint8Array(await result.response.arrayBuffer()), use.log);
+ return move({
+ ...facts,
+ 'response.openaiAudioTranscription.canonical': read.canonical,
+ 'response.openaiAudioTranscription.mediaType': mediaType,
+ 'response.openaiAudioTranscription.streamedOutcome': null,
+ 'response.usage.billable': [{ identity, quantities: billed(read.usage) }],
+ 'response.http.status': status,
+ 'response.http.headers': headers,
+ 'response.http.body': spentBody(result.response.body),
+ });
+ },
+});
+
+/** A body this stage has already read to the end, or one the upstream never sent. The record
+ * holds a body as a stream and `failover` releases the losing attempts', so every path hands
+ * one up; what says an answer was unusable is the failure at the canonical key, not this. */
+const spentBody = (body: ReadableStream | null): ReadableStream & Owned =>
+ own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve());
+
+const refusal = async (status: number, response: Response): Promise => {
+ const text = await response.text();
+ try {
+ return { status, message: text, body: JSON.parse(text) as unknown };
+ } catch {
+ // A refusal that is not JSON is still a refusal, and its text is what the client is
+ // told; there is simply no parsed body for a dump reader to open.
+ return { status, message: text };
+ }
+};
+
+/**
+ * What a 2xx body was worth to this endpoint, in two readings that do not depend on each
+ * other.
+ *
+ * Neither can cost the client the answer: the document is carried, so a body no reading
+ * could open is still what goes back, and what a failed reading costs is the transcript in
+ * the record and whatever usage the body would have stated. Nor can either cost the other —
+ * an upstream whose usage block this gateway cannot model still had its transcript read, and
+ * one whose document it could not open is still billed for nothing rather than mis-billed.
+ */
+const readTranscription = (
+ format: OpenAIAudioTranscriptionResponseFormat,
+ document: Uint8Array,
+ log: Logger,
+): { readonly canonical: CanonicalOpenAIAudioTranscription; readonly usage: OpenAIAudioTranscriptionUsage | undefined } => {
+ let canonical: CanonicalOpenAIAudioTranscription;
+ try {
+ canonical = parseOpenAIAudioTranscription(format, document);
+ } catch (error) {
+ log.warn('failed to parse 2xx upstream body for /audio/transcriptions; forwarding it as it arrived', { error: String(error) });
+ return { canonical: { document }, usage: undefined };
+ }
+ // Only an object rendering states usage: `text`, `srt` and `vtt` have nowhere to put it,
+ // and asking them for one is what would warn about every subtitle a client requests.
+ return { canonical, usage: canonical.raw === undefined ? undefined : readUsage(() => parseOpenAIAudioTranscriptionUsage(canonical.raw), log) };
+};
+
+/** A transcription states its usage in two places — the body of an object rendering and the
+ * terminal event of a stream — and either can state it in a shape this gateway cannot read.
+ * That upstream metered something, and the request is recorded saying exactly that: the
+ * entity, and no quantities. */
+const readUsage = (read: () => OpenAIAudioTranscriptionUsage | undefined, log: Logger): OpenAIAudioTranscriptionUsage | undefined => {
+ try {
+ return read();
+ } catch (error) {
+ log.warn('invalid usage in 2xx upstream response; recording the request only', { error: String(error) });
+ return undefined;
+ }
+};
+
+interface MeteredEvents {
+ readonly events: OpenAIAudioTranscriptionEvents;
+ readonly outcome: Deferred;
+}
+
+const meterEvents = (
+ body: ReadableStream,
+ identity: TelemetryModelIdentity,
+ signal: AbortSignal | undefined,
+ log: Logger,
+): MeteredEvents => {
+ let settle!: (outcome: OpenAIAudioTranscriptionStreamOutcome) => void;
+ // Declared as this run's own unfinished work, so the runner waits for it at teardown where
+ // it can see it rather than the reading being started and forgotten.
+ const outcome = defer(new Promise(resolve => { settle = resolve; }));
+ const events = viewOf((async function* () {
+ let usage: OpenAIAudioTranscriptionUsage | undefined;
+ let completed = false;
+ try {
+ for await (const frame of parseSSEStream(body, { signal })) {
+ const event = parseOpenAIAudioTranscriptionStreamEvent(JSON.parse(frame.data) as unknown);
+ if (isOpenAIAudioTranscriptionDoneEvent(event)) {
+ usage = readUsage(() => parseOpenAIAudioTranscriptionStreamUsage(event), log);
+ completed = true;
+ yield event;
+ // The transcript is complete, so there is nothing further to read. An upstream that
+ // holds the connection open past this point would otherwise keep the client's own
+ // stream open with it; returning here closes the read, which cancels the upstream.
+ return;
+ }
+ yield event;
+ }
+ } finally {
+ // Reached however the events ended — the terminal one, a client that stopped reading,
+ // or a broken upstream — because what the upstream already metered is billable
+ // whatever happened to the downstream half.
+ settle({ billable: [{ identity, quantities: billed(usage) }], failed: !completed });
+ }
+ })());
+ return { events, outcome };
+};
+
+const billed = (usage: OpenAIAudioTranscriptionUsage | undefined): UsageQuantities => {
+ if (usage === undefined) return {};
+ if (usage.kind === 'duration') return { input_audio_seconds: parseDecimalString(String(usage.seconds)) };
+ // Audio input is priced apart from text input, so what stays on the general input metric is
+ // what the upstream did not attribute to audio.
+ return {
+ input_tokens: parseDecimalString(String(usage.inputTokens - (usage.inputAudioTokens ?? 0))),
+ ...(usage.inputAudioTokens === undefined ? {} : { input_audio_tokens: parseDecimalString(String(usage.inputAudioTokens)) }),
+ output_tokens: parseDecimalString(String(usage.outputTokens)),
+ };
+};
+
+/** Nothing about this request narrows a candidate further. A transcription's parameters are
+ * form fields the upstream reads for itself, and no endpoint metadata says which renderings
+ * a model can write — so exposing the endpoint is the whole of the test. */
+const narrowing = {
+ kind: 'transcription' as const,
+ reject: (candidate: ModelCandidate): string | null =>
+ candidate.model.endpoints.openaiAudioTranscriptions === undefined
+ ? 'the upstream does not expose an OpenAI Audio Transcriptions endpoint'
+ : null,
+ unsupported: (model: string) => `Model ${model} does not support the /audio/transcriptions endpoint.`,
+ refuse: (status: number, message: string) => ({
+ 'response.openaiAudioTranscription.canonical': { status, message } as Failure,
+ 'response.openaiAudioTranscription.mediaType': null,
+ 'response.openaiAudioTranscription.streamedOutcome': null,
+ }),
+ refuses: [
+ 'response.openaiAudioTranscription.canonical',
+ 'response.openaiAudioTranscription.mediaType',
+ 'response.openaiAudioTranscription.streamedOutcome',
+ ] as const,
+};
+
+export const openaiAudioTranscriptionServePipeline: Pipeline<
+ A<'ingress.http.headers' | 'ingress.openaiAudioTranscription.responseFormat' | 'request.openaiAudioTranscription.form' | 'serve.model'>,
+ A<'response.openaiAudioTranscription.rendered' | 'response.openaiAudioTranscription.mediaType' | 'response.openaiAudioTranscription.streamedOutcome'>
+ & { 'response.http.status': number; 'response.usage.billable': readonly BillableEntity[];
+ 'response.http.headers': readonly (readonly [string, string])[]; }
+> = compose('openaiAudioTranscriptionServe', [
+ emitOpenAIAudioTranscription,
+ writeSettlement(
+ handedUp => isFailure((handedUp as { 'response.openaiAudioTranscription.canonical'?: unknown })['response.openaiAudioTranscription.canonical']),
+ handedUp => (handedUp as { 'response.openaiAudioTranscription.streamedOutcome'?: unknown })['response.openaiAudioTranscription.streamedOutcome'] !== null,
+ ),
+ resolveCandidates(narrowing),
+ failover({
+ failed: handedUp => isFailure((handedUp as { 'response.openaiAudioTranscription.canonical'?: unknown })['response.openaiAudioTranscription.canonical']),
+ owns: ['response.http.body'],
+ }),
+ callOpenAIAudioTranscriptionUpstream,
+]);
diff --git a/packages/gateway/src/data-plane/openai-audio/respond.ts b/packages/gateway/src/data-plane/openai-audio/respond.ts
deleted file mode 100644
index dc838b760c..0000000000
--- a/packages/gateway/src/data-plane/openai-audio/respond.ts
+++ /dev/null
@@ -1,92 +0,0 @@
-import { streamSSE } from 'hono/streaming';
-
-import { measureOpenAIAudioTranscriptionUsage } from './usage.ts';
-import { passthroughApiError } from '../shared/passthrough-serve.ts';
-import type { PassthroughResponseStrategyContext } from '../shared/passthrough-serve.ts';
-import { type StreamCompletion, writeSSEFrames } from '../shared/sse.ts';
-import { settleUsageMeasurement } from '../shared/telemetry/settle.ts';
-import { requestOnlyUsageMeasurement } from '../shared/telemetry/usage.ts';
-import { forwardUpstreamHeaders, forwardUpstreamResponse } from '../shared/upstream-response.ts';
-import { eventFrame, isEventStreamMediaType, isJsonMediaType, parseSSEStream, sseCommentFrame } from '@floway-dev/protocols/common';
-import { isOpenAIAudioTranscriptionDoneEvent } from '@floway-dev/protocols/openai-audio';
-
-const respondNonStreaming = async ({ ctx, sourceApi, response, performance, identity }: PassthroughResponseStrategyContext): Promise => {
- let measurement = requestOnlyUsageMeasurement();
- if (isJsonMediaType(response.headers.get('content-type'))) {
- let parsed: unknown;
- try {
- parsed = await response.clone().json();
- } catch (error) {
- console.warn(
- `audio-transcription: failed to parse 2xx upstream body for ${sourceApi}; usage row will be request-only`,
- error instanceof Error ? error.message : String(error),
- );
- }
- if (parsed !== undefined) {
- measurement = measureOpenAIAudioTranscriptionUsage(parsed, sourceApi);
- }
- }
- ctx.dump?.success(identity, measurement.dumpTokenUsage);
- settleUsageMeasurement(ctx, performance, identity, measurement, false);
- return forwardUpstreamResponse(response, { defaultContentType: null });
-};
-
-const respondStreaming = ({ c, ctx, sourceApi, response, performance, identity }: PassthroughResponseStrategyContext): Response => {
- const upstreamBody = response.body;
- if (!upstreamBody) {
- ctx.dump?.failed(`${sourceApi} streaming upstream returned no body`);
- settleUsageMeasurement(ctx, performance, identity, requestOnlyUsageMeasurement(), true);
- forwardUpstreamHeaders(c, response.headers);
- return passthroughApiError(c, 'Upstream returned a streaming response with no body.', 502);
- }
- forwardUpstreamHeaders(c, response.headers);
- return streamSSE(c, async stream => {
- let completion: StreamCompletion = 'error';
- let streamError: unknown;
- let terminalEventSeen = false;
- let measurement = requestOnlyUsageMeasurement();
- try {
- const frames = (async function* () {
- for await (const frame of parseSSEStream(upstreamBody, { signal: ctx.abortSignal })) {
- let event: unknown;
- try {
- event = JSON.parse(frame.data) as unknown;
- } catch (error) {
- throw new Error(`Malformed upstream ${sourceApi} SSE JSON: ${frame.data}`, { cause: error });
- }
- ctx.dump?.frame(eventFrame(event));
- if (isOpenAIAudioTranscriptionDoneEvent(event)) {
- terminalEventSeen = true;
- measurement = measureOpenAIAudioTranscriptionUsage(event, sourceApi);
- yield frame;
- return;
- }
- yield frame;
- }
- })();
- completion = await writeSSEFrames(stream, frames, {
- keepAlive: { frame: sseCommentFrame('keepalive') },
- downstreamAbortController: ctx.downstreamAbortController,
- });
- } catch (error) {
- streamError = error;
- } finally {
- const failed = streamError !== undefined || completion === 'error' || !terminalEventSeen;
- if (failed) ctx.dump?.failed(streamError ?? `${sourceApi} stream ended with completion=${completion}`);
- else ctx.dump?.success(identity, measurement.dumpTokenUsage);
- settleUsageMeasurement(ctx, performance, identity, measurement, failed);
- }
- });
-};
-
-export const respondOpenAIAudioTranscription = async (context: PassthroughResponseStrategyContext): Promise => {
- const { ctx, response, performance, identity } = context;
- if (!response.ok) {
- settleUsageMeasurement(ctx, performance, identity, requestOnlyUsageMeasurement(), true);
- ctx.dump?.error('upstream', identity.upstream);
- return forwardUpstreamResponse(response, { defaultContentType: null });
- }
- return isEventStreamMediaType(response.headers.get('content-type'))
- ? respondStreaming(context)
- : await respondNonStreaming(context);
-};
diff --git a/packages/gateway/src/data-plane/openai-audio/usage.ts b/packages/gateway/src/data-plane/openai-audio/usage.ts
deleted file mode 100644
index 31dbc13610..0000000000
--- a/packages/gateway/src/data-plane/openai-audio/usage.ts
+++ /dev/null
@@ -1,100 +0,0 @@
-import type { UsageQuantities } from '../../repo/types.ts';
-import { requestOnlyUsageMeasurement, tokenUsage, type UsageMeasurement } from '../shared/telemetry/usage.ts';
-import { parseDecimalString } from '@floway-dev/protocols/common';
-
-// OpenAI transcription responses discriminate usage by `type`. Token-based
-// models split input_token_details into text and audio metrics; without that
-// optional split, the aggregate stays on the general input metric. Duration-
-// based usage exposes seconds, while Whisper verbose JSON reports the same
-// quantity as a top-level `duration`. Unknown breakdowns record the request
-// only, while malformed fields under a known discriminator remain observable.
-// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L36378-L36562
-const audioDurationMeasurement = (seconds: unknown, label: string): UsageMeasurement => {
- if (typeof seconds !== 'number' || !Number.isFinite(seconds) || seconds < 0) {
- throw new Error(`Audio transcription ${label} must be a finite non-negative number`);
- }
- return {
- quantities: { input_audio_seconds: parseDecimalString(String(seconds)) },
- pricingFacts: {},
- dumpTokenUsage: null,
- };
-};
-
-export const openaiAudioTranscriptionUsageMeasurement = (body: unknown): UsageMeasurement => {
- if (!body || typeof body !== 'object') return requestOnlyUsageMeasurement();
- if (!Object.hasOwn(body, 'usage')) {
- if (!Object.hasOwn(body, 'duration')) return requestOnlyUsageMeasurement();
- return audioDurationMeasurement((body as { duration: unknown }).duration, 'duration');
- }
- const usage = (body as { usage: unknown }).usage;
- if (!usage || typeof usage !== 'object' || Array.isArray(usage)) {
- throw new Error('Audio transcription usage must be an object');
- }
- const metric = usage as { type?: unknown; seconds?: unknown; input_tokens?: unknown; input_token_details?: unknown; output_tokens?: unknown; total_tokens?: unknown };
-
- if (metric.type === 'duration') {
- return audioDurationMeasurement(metric.seconds, 'duration usage.seconds');
- }
-
- if (metric.type !== 'tokens') return requestOnlyUsageMeasurement();
- for (const [field, value] of [
- ['input_tokens', metric.input_tokens],
- ['output_tokens', metric.output_tokens],
- ['total_tokens', metric.total_tokens],
- ] as const) {
- if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
- throw new Error(`Audio transcription token usage.${field} must be a non-negative safe integer`);
- }
- }
- const inputTokens = metric.input_tokens as number;
- const outputTokens = metric.output_tokens as number;
- const totalTokens = metric.total_tokens as number;
- if (totalTokens !== inputTokens + outputTokens) {
- throw new Error('Audio transcription token usage.total_tokens must equal input_tokens plus output_tokens');
- }
-
- let inputQuantities: UsageQuantities = { input_tokens: parseDecimalString(String(inputTokens)) };
- if (metric.input_token_details !== undefined) {
- if (!metric.input_token_details || typeof metric.input_token_details !== 'object' || Array.isArray(metric.input_token_details)) {
- throw new Error('Audio transcription token usage.input_token_details must be an object');
- }
- const details = metric.input_token_details as { text_tokens?: unknown; audio_tokens?: unknown };
- for (const [field, value] of [
- ['text_tokens', details.text_tokens],
- ['audio_tokens', details.audio_tokens],
- ] as const) {
- if (value !== undefined && (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0)) {
- throw new Error(`Audio transcription token usage.input_token_details.${field} must be a non-negative safe integer`);
- }
- }
- const textTokens = details.text_tokens as number | undefined;
- const audioTokens = details.audio_tokens as number | undefined;
- if ((textTokens ?? 0) + (audioTokens ?? 0) > inputTokens) {
- throw new Error('Audio transcription token usage.input_token_details must not exceed input_tokens');
- }
- inputQuantities = {
- input_tokens: parseDecimalString(String(inputTokens - (audioTokens ?? 0))),
- ...(audioTokens === undefined ? {} : { input_audio_tokens: parseDecimalString(String(audioTokens)) }),
- };
- }
- return {
- quantities: {
- ...inputQuantities,
- output_tokens: parseDecimalString(String(outputTokens)),
- },
- pricingFacts: { inputTokens },
- dumpTokenUsage: tokenUsage({ input: inputTokens, output: outputTokens }),
- };
-};
-
-export const measureOpenAIAudioTranscriptionUsage = (value: unknown, sourceApi: string): UsageMeasurement => {
- try {
- return openaiAudioTranscriptionUsageMeasurement(value);
- } catch (error) {
- console.warn(
- `audio-transcription: invalid usage in 2xx upstream response for ${sourceApi}; usage row will be request-only`,
- error instanceof Error ? error.message : String(error),
- );
- return requestOnlyUsageMeasurement();
- }
-};
diff --git a/packages/gateway/src/data-plane/openai-completions/http.ts b/packages/gateway/src/data-plane/openai-completions/http.ts
index 7b74fbe483..596c0931ea 100644
--- a/packages/gateway/src/data-plane/openai-completions/http.ts
+++ b/packages/gateway/src/data-plane/openai-completions/http.ts
@@ -1,96 +1,55 @@
-// POST /v1/completions and /completions — OpenAI Completions passthrough.
-// The endpoint sits outside the chat source/target executor:
-// no protocol translation, no interceptor chain, no cross-protocol
-// traversal. The request body is forwarded to the chosen provider's
-// /completions verbatim; the response (single-shot JSON or streaming SSE
-// depending on the client's `stream` flag) flows back through the shared
-// passthroughServe scaffold.
+// POST /v1/completions and /completions, served through the pipeline.
+//
+// The handler is a prologue and an epilogue around `openaiCompletionsServePipeline`: read
+// what the client sent, hand it over, and turn what the run answered with into a response. The
+// one thing it decides for itself is whether the request streams, because that is written in
+// the body and the run has to be opened knowing it.
import type { Context } from 'hono';
-import { tokenUsageFromOpenAICompletionsUsage } from './usage.ts';
-import type { TokenUsage } from '../../repo/types.ts';
-import { backgroundSchedulerFromContext } from '../../runtime/background.ts';
-import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../shared/gateway-ctx.ts';
-import { prepareJsonModelRequest } from '../shared/passthrough-request.ts';
-import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts';
-import { readRequestBody, takeRequestBody } from '../shared/request-body.ts';
-import { isOpenAIUsageOnlyEventShape, type ProtocolFrame } from '@floway-dev/protocols/common';
-import type { ProviderModel } from '@floway-dev/provider';
+import { openaiCompletionsServePipeline } from './pipeline.ts';
+import { isFrames, openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts';
+import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts';
+import { prepareJsonModelRequest } from '../shared/json-model-request.ts';
+import { move } from '@floway-dev/pipeline';
export const openaiCompletions = async (c: Context): Promise => {
- const requestBody = await readRequestBody(c);
- const request = prepareJsonModelRequest(requestBody.bytes, 'OpenAI Completions');
- // `stream` decides the response shape, so the gateway context has to learn
- // it before the invalid branch — which has no body to read it from.
- const wantsStream = request.type === 'ok' && request.body.stream === true;
- const ctx = createGatewayCtxFromHono(c, {
- wantsStream,
- requestBody: takeRequestBody(requestBody),
- backgroundScheduler: backgroundSchedulerFromContext(c),
- });
+ const ingress = await readIngress(c);
+ const request = prepareJsonModelRequest(ingress.body.bytes, 'OpenAI Completions');
if (request.type === 'invalid') {
- ctx.dump?.error('gateway');
- return finalizeGatewayResponse(ctx, passthroughApiError(c, request.message, 400));
+ // A request the gateway could not read never reaches a pipeline: there is no model to
+ // resolve and no attempt to make, so there is nothing for a run to record.
+ const refused = openPrologue(c, ingress, { wantsStream: false });
+ refused.gateway.dump?.error('gateway');
+ return finalizeGatewayResponse(
+ refused.gateway,
+ Response.json({ error: { message: request.message, type: 'api_error' } }, { status: 400 }),
+ );
}
- ctx.dump?.requestedModel(request.model);
+ const wantsStream = request.body.stream === true;
const streamOptions = request.body.stream_options as { include_usage?: unknown } | null | undefined;
- const clientWantsUsageChunk = streamOptions?.include_usage === true;
- // Strip the inbound model; the provider re-stamps the upstream-resolved
- // model id. For streaming requests we force `stream_options.include_usage`
- // on so billing always sees the usage chunk — sibling keys on
- // stream_options (if any) ride through unchanged.
- const { model: _model, ...upstreamBodyBase } = request.body;
- const upstreamBody = wantsStream
- ? { ...upstreamBodyBase, stream_options: { ...(streamOptions ?? {}), include_usage: true } }
- : upstreamBodyBase;
+ const prologue = openPrologue(c, ingress, { wantsStream, model: request.model });
- // Streaming closure: track the usage block (only on the usage-only
- // chunk per OpenAI spec) and service_tier independently — service_tier
- // can ride on any event root, so settling them together at the end
- // lets the tier override land regardless of which chunk carried it.
- let streamingUsageBlock: unknown = null;
- let streamingServiceTier: string | null | undefined;
- // The scaffold picks the upstream after these closures are built, and this
- // endpoint has no interceptor chain to normalize usage on the way in, so
- // the serving model is staked here for the billing read to consult.
- let serving: { model: ProviderModel; upstreamId: string } | undefined;
- const declaredExclusive = (): boolean => serving?.model.enabledFlags.has('usage-exclusive-cached-tokens') === true;
- const servingIdentity = (): string => serving === undefined ? 'unresolved upstream' : `${serving.upstreamId}/${serving.model.id}`;
- const transformFrame = (frame: ProtocolFrame): ProtocolFrame | null => {
- if (frame.type !== 'event') return frame;
- const eventRoot = frame.event as { service_tier?: string | null; usage?: unknown };
- if (eventRoot.service_tier !== undefined) streamingServiceTier = eventRoot.service_tier;
- if (!isOpenAIUsageOnlyEventShape(frame.event)) return frame;
- streamingUsageBlock = eventRoot.usage;
- return clientWantsUsageChunk ? frame : null;
- };
- const settleUsage = (): TokenUsage | null =>
- streamingUsageBlock === null ? null : tokenUsageFromOpenAICompletionsUsage(streamingUsageBlock, streamingServiceTier, declaredExclusive(), servingIdentity());
-
- const response = await passthroughServe({
+ return await serveThrough(
c,
- ctx,
- sourceApi: '/completions',
- operation: 'text_completion',
- model: request.model,
- kind: 'chat',
- modelServesEndpoint: model => model.endpoints.openaiCompletions !== undefined,
- call: (provider, model, opts) => {
- serving = { model, upstreamId: provider.upstreamId };
- return provider.instance.callOpenAICompletions(model, upstreamBody, ctx.abortSignal, opts);
+ prologue,
+ openaiCompletionsServePipeline,
+ move({
+ 'ingress.http.headers': prologue.headers,
+ 'ingress.openaiCompletions.wantsStream': wantsStream,
+ // Whether the client asked to *see* the usage chunk. The edge turns it on upstream
+ // either way so billing always gets one, and drops it here when it was not asked for.
+ 'ingress.openaiCompletions.wantsUsageChunk': streamOptions?.include_usage === true,
+ 'request.openaiCompletions.payload': request.body,
+ 'serve.model': request.model,
+ }) as never,
+ facts => {
+ const rendered = facts['response.openaiCompletions.rendered'];
+ return isFrames(rendered)
+ ? { frames: rendered }
+ : { body: JSON.stringify(rendered), contentType: 'application/json' };
},
- response: wantsStream
- ? { format: 'sse', transformFrame, settleUsage }
- : {
- format: 'json',
- extractBilling: (body: unknown) => {
- if (!body || typeof body !== 'object') return null;
- const { usage, service_tier: tier } = body as { usage?: unknown; service_tier?: string | null };
- return tokenUsageFromOpenAICompletionsUsage(usage, tier, declaredExclusive(), servingIdentity());
- },
- },
- });
- return finalizeGatewayResponse(ctx, response);
+ facts => facts['response.openaiCompletions.streamedUsage'],
+ );
};
diff --git a/packages/gateway/src/data-plane/openai-completions/pipeline.ts b/packages/gateway/src/data-plane/openai-completions/pipeline.ts
new file mode 100644
index 0000000000..1468139696
--- /dev/null
+++ b/packages/gateway/src/data-plane/openai-completions/pipeline.ts
@@ -0,0 +1,420 @@
+// Text completions as a pipeline. One protocol, no translation, and an answer that is a
+// stream whenever the client asked for one — the same request field decides both what the
+// upstream is asked for and what shape comes back.
+//
+// The shape:
+//
+// emitOpenAICompletions the edge: asks the upstream for the usage chunk on the way
+// down, and renders the answer into the client's protocol on
+// the way back, SSE framing included
+// resolveCandidates narrows to the upstreams that expose the endpoint
+// failover runs what follows once per candidate
+// callOpenAICompletionsUpstream the ending: dials, parses what came back, and provides the
+// answer plus what is billable
+
+import { tokenUsageFromOpenAICompletionsUsage } from './usage.ts';
+import { recordStream, streamReferenceOf, type TurnDump } from '../../dump/turn-dump.ts';
+import type { UsageQuantities } from '../../repo/types.ts';
+import { tokenUsageQuantities } from '../../repo/usage-metrics.ts';
+import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts';
+import { isFailure } from '../pipeline/facts.ts';
+import type { StreamOutcome } from '../pipeline/serve.ts';
+import type { GatewayServices } from '../pipeline/services.ts';
+import { writeSettlement } from '../pipeline/settlement.ts';
+import { failover, resolveCandidates } from '../pipeline/stages.ts';
+import { dialFailure } from '../pipeline/upstream-body.ts';
+import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts';
+import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts';
+import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts';
+import { compose, defer, defineStage, move, own, type Deferred, type Owned, type Pipeline } from '@floway-dev/pipeline';
+import { isOpenAIUsageOnlyEventShape, renderErrorEnvelope, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common';
+import {
+ openaiCompletionsProtocolFrameToSSEFrame,
+ parseOpenAICompletionsResult,
+ parseOpenAICompletionsStream,
+ type OpenAICompletionsPayload,
+ type OpenAICompletionsResult,
+ type OpenAICompletionsStreamEvent,
+} from '@floway-dev/protocols/openai-completions';
+import { providerModelOf } from '@floway-dev/provider';
+import type { ModelCandidate, TelemetryModelIdentity } from '@floway-dev/provider';
+
+/** The answer while it is still the upstream's, one frame at a time. */
+export type OpenAICompletionsFrames = AsyncIterable>;
+
+/**
+ * A stream as a value the record can hold: a wrapper around the generator rather than the
+ * generator itself, which is what says where the resource is. There is exactly one resource in
+ * an OpenAI Completions run — the upstream's body, claimed with `own()` — and the wrapper keeps
+ * a frame view from reading as another.
+ *
+ * What comes back is single-shot on purpose: a stream that has been read is a stream that is
+ * over, and a second reader learns that rather than being lied to.
+ */
+const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames });
+
+/** OpenAI Completions' own keys, extending the shared space by intersection. */
+export interface OpenAICompletionsFacts extends GatewayFacts {
+ /** What the client asked for, which is not what the upstream is asked for: the gateway
+ * meters every stream and so always turns the usage chunk on. These stay put for the same
+ * reason every `ingress.*` key does — they describe the request that arrived, and the
+ * answer is rendered back into it. */
+ 'ingress.openaiCompletions.wantsStream': boolean;
+ 'ingress.openaiCompletions.wantsUsageChunk': boolean;
+ 'request.openaiCompletions.payload': OpenAICompletionsPayload;
+ /** The answer, whichever kind it turned out to be. A stream, a value and a failure sit at
+ * one key: telling them apart is reading a value, and each stage does that where it needs
+ * to. */
+ 'response.openaiCompletions.payload': OpenAICompletionsResult | OpenAICompletionsFrames | Failure;
+ /** What the upstream will have reported by the time the frames run out, and `null` on
+ * every path that does not stream. A stream's usage arrives with its last chunk, which is
+ * after this run has answered — so the numbers cannot be in `response.usage.billable`,
+ * which says what had been reported when the ending stage handed up: the entity, and no
+ * quantities. Settling billing from this is the prologue's job, after the drain. */
+ 'response.openaiCompletions.streamedUsage': Deferred | null;
+ /** What the client is actually sent, in its own protocol — a JSON body, or the SSE frames
+ * of a stream. The edge provides it, so a dump shows what the client received rather than
+ * the gateway's own reading of it. */
+ 'response.openaiCompletions.rendered': Record | AsyncIterable;
+}
+
+type C = { [P in K]: OpenAICompletionsFacts[P] };
+
+const isFrames = (answer: OpenAICompletionsFacts['response.openaiCompletions.payload']): answer is OpenAICompletionsFrames =>
+ Symbol.asyncIterator in answer;
+
+/**
+ * The outermost edge, and the only place where what the client asked for and what the
+ * upstream is asked for differ: billing needs the usage chunk on every stream, so it is
+ * turned on going down and taken back out coming up unless the client asked to see it.
+ *
+ * Rendering the answer is the other half. SSE framing is produced here and nowhere else —
+ * below this stage a stream carries protocol frames, so the same assembly would serve
+ * another transport by rendering differently at this one point.
+ */
+const emitOpenAICompletions = defineStage<
+ C<'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload'>,
+ C<'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload'>,
+ C<'ingress.openaiCompletions.wantsUsageChunk' | 'response.openaiCompletions.payload' | 'response.http.headers'>,
+ C<'response.openaiCompletions.rendered' | 'response.http.status' | 'response.http.headers'>
+>({
+ name: 'emitOpenAICompletions',
+ through: {
+ request: {
+ needs: ['ingress.openaiCompletions.wantsStream', 'ingress.openaiCompletions.wantsUsageChunk', 'request.openaiCompletions.payload'],
+ consumes: [],
+ provides: ['request.openaiCompletions.payload'],
+ },
+ response: {
+ needs: ['response.openaiCompletions.payload', 'response.http.headers'],
+ consumes: ['response.openaiCompletions.payload', 'response.http.headers'],
+ provides: ['response.openaiCompletions.rendered', 'response.http.status', 'response.http.headers'],
+ },
+ },
+ execute: async (facts, next) => {
+ const asked = facts['request.openaiCompletions.payload'];
+ const back = await next({
+ ...facts,
+ 'request.openaiCompletions.payload': move(facts['ingress.openaiCompletions.wantsStream']
+ ? { ...asked, stream_options: { ...asked.stream_options, include_usage: true } }
+ : asked),
+ });
+
+ const { 'response.openaiCompletions.payload': answer, 'response.http.headers': headers, ...rest } = back;
+ // Vendor traces and quota state stay visible; what an intermediary must strip, and what
+ // would misdescribe a body this gateway serialized itself, does not. A filter that removed
+ // nothing hands the same array on, so the record shows no change where none happened.
+ const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name));
+ // A refusal keeps the status the upstream gave it. Anything that answered is a 200 the
+ // gateway says itself, because what the client receives is serialized here rather than
+ // relayed — the upstream's own status is a fact further down for whoever wants it.
+ return {
+ ...rest,
+ 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable),
+ 'response.http.status': isFailure(answer) ? answer.status : 200,
+ 'response.openaiCompletions.rendered': move(rendered(answer, back['ingress.openaiCompletions.wantsUsageChunk'])),
+ };
+ },
+});
+
+const rendered = (
+ answer: OpenAICompletionsFacts['response.openaiCompletions.payload'],
+ wantsUsageChunk: boolean,
+): OpenAICompletionsFacts['response.openaiCompletions.rendered'] =>
+ isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body)
+ : isFrames(answer) ? renderSSE(answer, wantsUsageChunk)
+ : answer;
+
+const renderSSE = (frames: OpenAICompletionsFrames, wantsUsageChunk: boolean): AsyncIterable => ({
+ // The frames the client reads are a reframing of the ones the record holds, so this key
+ // points at that same stream rather than at nothing.
+ ...streamReferenceOf(frames),
+ [Symbol.asyncIterator]: () => (async function* () {
+ for await (const frame of frames) {
+ if (!wantsUsageChunk && frame.type === 'event' && isOpenAIUsageOnlyEventShape(frame.event)) continue;
+ yield openaiCompletionsProtocolFrameToSSEFrame(frame);
+ }
+ })(),
+});
+
+/**
+ * The ending. It dials, reads what came back on the shape the request asked for, and
+ * provides the answer, the raw HTTP response beneath it, and what the call is billable for.
+ * A failure is a value: a 429 here is what an earlier stage fails over, and so is a 200 whose
+ * body this protocol cannot read, because the next candidate's path and flags may differ.
+ */
+const callOpenAICompletionsUpstream = defineStage<
+ C<'ingress.openaiCompletions.wantsStream' | 'request.openaiCompletions.payload' | 'route.attempt' | 'ingress.http.headers'>,
+ C<'response.openaiCompletions.payload' | 'response.openaiCompletions.streamedUsage' | 'response.usage.billable'
+ | 'response.http.status' | 'response.http.headers' | 'response.http.body'>,
+ GatewayServices
+>({
+ name: 'callOpenAICompletionsUpstream',
+ return: {
+ provides: [
+ 'response.openaiCompletions.payload',
+ 'response.openaiCompletions.streamedUsage',
+ 'response.usage.billable',
+ 'response.http.status',
+ 'response.http.headers',
+ 'response.http.body',
+ ],
+ },
+ execute: async (facts, use) => {
+ const candidate = use.resolveAttempt(facts['route.attempt']);
+ // The provider re-stamps whatever id it resolved upstream, so the id the client
+ // addressed does not travel with the body.
+ const { model: _addressed, ...body } = facts['request.openaiCompletions.payload'];
+ // Attribution is set before the dial, so an attempt that never completes still names the
+ // candidate it was made against rather than the one tried before it.
+ use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'text_completion');
+
+ let result;
+ try {
+ result = await candidate.provider.instance.callOpenAICompletions(
+ providerModelOf(candidate),
+ body,
+ use.gateway.abortSignal,
+ // The client's own headers reach the upstream from the record, not from a live request
+ // object: what a provider is allowed to forward is filtered per provider, and the dump
+ // shows what was there to filter.
+ buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))),
+ );
+ } catch (error) {
+ use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) });
+ // A dial that never completed reached no upstream, so nothing was billed and there are
+ // no headers to carry. What it leaves behind is the performance row settlement writes.
+ return move({
+ ...facts,
+ 'response.openaiCompletions.payload': dialFailure(error),
+ 'response.openaiCompletions.streamedUsage': null,
+ 'response.usage.billable': [],
+ 'response.http.status': 502,
+ 'response.http.headers': [],
+ 'response.http.body': spentBody(null),
+ });
+ }
+ if (!result.response.ok) use.log.warn('upstream refused', { status: result.response.status });
+
+ const answer = await readUpstream(
+ result.response,
+ facts['ingress.openaiCompletions.wantsStream'],
+ telemetryModelIdentity(candidate, result.modelKey),
+ candidate,
+ use.gateway.abortSignal,
+ use.gateway.dump,
+ );
+ return move({
+ ...facts,
+ 'response.openaiCompletions.payload': answer.payload,
+ 'response.openaiCompletions.streamedUsage': answer.streamedUsage,
+ 'response.usage.billable': answer.billable,
+ 'response.http.status': result.response.status,
+ 'response.http.headers': [...result.response.headers],
+ 'response.http.body': answer.body,
+ });
+ },
+});
+
+interface UpstreamAnswer {
+ readonly payload: OpenAICompletionsFacts['response.openaiCompletions.payload'];
+ readonly streamedUsage: OpenAICompletionsFacts['response.openaiCompletions.streamedUsage'];
+ readonly billable: readonly BillableEntity[];
+ /** Every arm hands one up, this stage's own reading included: the record holds a body as a
+ * stream, and `failover` releases the losing attempts' by consuming that key. */
+ readonly body: ReadableStream & Owned;
+}
+
+/** What the upstream said, on the shape the request asked for. Which of the four this is is
+ * read from the response, never declared: a stream, a value and a failure sit at one key. */
+const readUpstream = async (
+ response: Response,
+ wantsStream: boolean,
+ identity: TelemetryModelIdentity,
+ candidate: ModelCandidate,
+ signal: AbortSignal | undefined,
+ dump: TurnDump | null,
+): Promise => {
+ // An upstream was called and reported nothing — which is what every arm but a read usage
+ // block leaves standing, and is a different statement from reporting zero.
+ const called: readonly BillableEntity[] = [{ identity, quantities: {} }];
+
+ if (!response.ok) {
+ // An upstream error body is JSON like any other body. Reading it here is also what
+ // leaves a losing attempt with nothing open behind it.
+ return { payload: await refusal(response), streamedUsage: null, billable: called, body: spentBody(response.body) };
+ }
+
+ if (!wantsStream) {
+ const read = await readResult(response);
+ return {
+ payload: isFailure(read) ? read : read.value,
+ streamedUsage: null,
+ billable: isFailure(read) ? called : [{ identity, quantities: billed(read.value.usage, read.value.service_tier, candidate) }],
+ body: spentBody(response.body),
+ };
+ }
+
+ if (response.body === null) {
+ return {
+ payload: { status: 502, message: 'Upstream returned a streaming response with no body.' },
+ streamedUsage: null,
+ billable: called,
+ body: spentBody(null),
+ };
+ }
+
+ // Usage is observed here, closest to the upstream and on the protocol it spoke, by folding
+ // the frames as they pass — so the reading costs one pass and the client's own stream is
+ // what drives it. What it finds arrives with the last frame, long after this stage has
+ // handed up, which is why the entity above carries no quantities.
+ const metered = meterFrames(parseOpenAICompletionsStream(response.body, { signal }), identity, candidate);
+ return {
+ payload: recordStream(metered.frames, dump),
+ streamedUsage: metered.outcome,
+ billable: called,
+ // Releasing this body is draining those frames: they are one reader over one connection,
+ // and a second reader is not something a `ReadableStream` allows.
+ body: own(response.body, async (): Promise => { for await (const _frame of metered.frames) { /* to end of stream */ } }),
+ };
+};
+
+/** A body this stage has already read to the end, or one the upstream never sent. The record
+ * holds a body as a stream and `failover` releases the losing attempts', so every path hands
+ * one up; what says an answer was unusable is the failure at the payload key, not this. */
+const spentBody = (body: ReadableStream | null): ReadableStream & Owned =>
+ own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve());
+
+const refusal = async (response: Response): Promise => {
+ const text = await response.text();
+ try {
+ return { status: response.status, message: text, body: JSON.parse(text) as unknown };
+ } catch {
+ // A refusal that is not JSON is still a refusal, and its text is what the client is
+ // told; there is simply no parsed body for a dump reader to open.
+ return { status: response.status, message: text };
+ }
+};
+
+/** A body that answered 200 and cannot be read as this protocol is an attempt that failed,
+ * because the gateway serializes what it sends from the value it parsed and has nothing to
+ * serialize. The next candidate may well answer in the protocol it advertised. */
+const readResult = async (response: Response): Promise<{ readonly value: OpenAICompletionsResult } | Failure> => {
+ try {
+ return { value: parseOpenAICompletionsResult(await response.json()) };
+ } catch (error) {
+ return { status: 502, message: `Upstream answered ${response.status} with a body this endpoint cannot read: ${error instanceof Error ? error.message : String(error)}` };
+ }
+};
+
+interface MeteredFrames {
+ readonly frames: OpenAICompletionsFrames;
+ readonly outcome: Deferred;
+}
+
+const meterFrames = (
+ source: AsyncIterable>,
+ identity: TelemetryModelIdentity,
+ candidate: ModelCandidate,
+): MeteredFrames => {
+ let settle!: (outcome: StreamOutcome) => void;
+ // Declared as this run's own unfinished work, so the runner waits for it at teardown where
+ // it can see it rather than the reading being started and forgotten.
+ const outcome = defer(new Promise(resolve => { settle = resolve; }));
+ // Running out without the terminal frame is what "it did not finish" means, and it is known
+ // at the same moment the usage is.
+ let sawTerminal = false;
+ const frames = view((async function* () {
+ let usage: unknown;
+ let tier: string | null | undefined;
+ try {
+ for await (const frame of source) {
+ if (frame.type === 'event') {
+ // `service_tier` can ride on any chunk's root while the totals only land on the
+ // usage-only one, so the two are tracked apart and settled together.
+ const root = frame.event as { service_tier?: string | null; usage?: unknown };
+ if (root.service_tier !== undefined) tier = root.service_tier;
+ if (isOpenAIUsageOnlyEventShape(frame.event)) usage = root.usage;
+ }
+ // The transport's own terminator is this protocol's end-of-turn.
+ if (frame.type === 'done') sawTerminal = true;
+ yield frame;
+ }
+ } finally {
+ // Reached however the frames ended — the terminal chunk, a client that stopped
+ // reading, or a broken upstream — because tokens the upstream already metered are
+ // billable whatever happened to the downstream half.
+ settle({ billable: [{ identity, quantities: billed(usage, tier, candidate) }], failed: !sawTerminal });
+ }
+ })());
+ return { frames, outcome };
+};
+
+const billed = (usage: unknown, tier: string | null | undefined, candidate: ModelCandidate): UsageQuantities => {
+ const model = providerModelOf(candidate);
+ const tokens = tokenUsageFromOpenAICompletionsUsage(
+ usage,
+ tier,
+ model.enabledFlags.has('usage-exclusive-cached-tokens'),
+ `${candidate.provider.upstreamId}/${model.id}`,
+ );
+ // An upstream that reported nothing leaves no quantities at all, which is a different
+ // statement from reporting zero.
+ //
+ // The service tier survives as far as `TokenUsage.tier` and no further: a billed entity is
+ // an identity and a bag of quantities, and the pricing selector the tier feeds has no seat
+ // there. It comes back when settlement does.
+ return tokens === null ? {} : tokenUsageQuantities(tokens);
+};
+
+/** A candidate that cannot serve this endpoint is not a candidate. The resolver's own filter
+ * is by kind, and a text-completion model is a chat-kind model, so what is left to say is
+ * whether the upstream exposes the endpoint at all. */
+const narrowing = {
+ kind: 'chat' as const,
+ reject: (candidate: ModelCandidate): string | null =>
+ candidate.model.endpoints.openaiCompletions === undefined ? 'the upstream does not expose an OpenAI Completions endpoint' : null,
+ unsupported: (model: string) => `Model ${model} does not support the /completions endpoint.`,
+ refuse: (status: number, message: string): C<'response.openaiCompletions.payload' | 'response.openaiCompletions.streamedUsage'> => ({
+ 'response.openaiCompletions.payload': { status, message },
+ 'response.openaiCompletions.streamedUsage': null,
+ }),
+ refuses: ['response.openaiCompletions.payload', 'response.openaiCompletions.streamedUsage'] as const,
+};
+
+export const openaiCompletionsServePipeline: Pipeline<
+ C<'ingress.http.headers' | 'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload' | 'serve.model'>,
+ C<'response.openaiCompletions.rendered' | 'response.openaiCompletions.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>
+> = compose('openaiCompletionsServe', [
+ emitOpenAICompletions,
+ writeSettlement(
+ handedUp => isFailure((handedUp as { 'response.openaiCompletions.payload'?: unknown })['response.openaiCompletions.payload']),
+ handedUp => (handedUp as { 'response.openaiCompletions.streamedUsage'?: unknown })['response.openaiCompletions.streamedUsage'] !== null,
+ ),
+ resolveCandidates(narrowing),
+ failover({
+ failed: handedUp => isFailure((handedUp as { 'response.openaiCompletions.payload'?: unknown })['response.openaiCompletions.payload']),
+ owns: ['response.http.body'],
+ }),
+ callOpenAICompletionsUpstream,
+]);
diff --git a/packages/gateway/src/data-plane/openai-completions/usage.ts b/packages/gateway/src/data-plane/openai-completions/usage.ts
index f753f8ca47..f3d4332aa5 100644
--- a/packages/gateway/src/data-plane/openai-completions/usage.ts
+++ b/packages/gateway/src/data-plane/openai-completions/usage.ts
@@ -16,11 +16,10 @@ import { billableServiceTier, splitInclusiveInputTokens } from '@floway-dev/prot
// non-streaming /v1/completions body (observed null on a Zhipu/GLM
// fork); the streaming path was observed to omit the field.
//
-// This endpoint is a passthrough with no interceptor chain, so the fold the
-// chat targets apply to the usage chunk itself happens here instead, on the
-// one read that consumes it. `declaredExclusive` carries the serving
-// upstream's `usage-exclusive-cached-tokens` flag and `identity` names it in
-// whatever `foldsExclusiveCacheTokens` raises.
+// This is the one read of a completions usage block, so the fold the chat targets apply to
+// the usage chunk itself happens here instead. `declaredExclusive` carries the serving
+// upstream's `usage-exclusive-cached-tokens` flag and `identity` names it in whatever
+// `foldsExclusiveCacheTokens` raises.
export const tokenUsageFromOpenAICompletionsUsage = (
usage: unknown,
diff --git a/packages/gateway/src/data-plane/openai-embeddings/http.ts b/packages/gateway/src/data-plane/openai-embeddings/http.ts
index e00e0691a7..2882e60f36 100644
--- a/packages/gateway/src/data-plane/openai-embeddings/http.ts
+++ b/packages/gateway/src/data-plane/openai-embeddings/http.ts
@@ -1,38 +1,54 @@
-// POST /v1/embeddings — route embedding requests to the provider that
-// declares the requested model and OpenAI Embeddings capability.
+// POST /v1/embeddings, served through the pipeline.
+//
+// The handler is a prologue and an epilogue around `openaiEmbeddingsServePipeline`: parse what
+// the client sent, hand it over, and turn what the run answered with into a response.
+// Everything between is stages.
import type { Context } from 'hono';
-import { tokenUsageFromOpenAIEmbeddingsBody } from './usage.ts';
-import { backgroundSchedulerFromContext } from '../../runtime/background.ts';
-import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../shared/gateway-ctx.ts';
-import { prepareJsonModelRequest } from '../shared/passthrough-request.ts';
-import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts';
-import { readRequestBody, takeRequestBody } from '../shared/request-body.ts';
+import { openaiEmbeddingsServePipeline } from './pipeline.ts';
+import { openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts';
+import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts';
+import { move } from '@floway-dev/pipeline';
+import { parseOpenAIEmbeddingsRequest, type ParsedOpenAIEmbeddingsRequest } from '@floway-dev/protocols/openai-embeddings';
+
+// The contract reports a malformed request by throwing; what the client is owed is a 400
+// carrying the reason.
+const readRequest = (bytes: Uint8Array): { type: 'ok'; parsed: ParsedOpenAIEmbeddingsRequest } | { type: 'invalid'; message: string } => {
+ try {
+ return { type: 'ok', parsed: parseOpenAIEmbeddingsRequest(JSON.parse(new TextDecoder().decode(bytes)) as unknown) };
+ } catch (error) {
+ return { type: 'invalid', message: error instanceof Error ? error.message : String(error) };
+ }
+};
export const openaiEmbeddings = async (c: Context): Promise => {
- const requestBody = await readRequestBody(c);
- const request = prepareJsonModelRequest(requestBody.bytes, 'OpenAI Embeddings');
- const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) });
- if (request.type === 'invalid') {
- ctx.dump?.error('gateway');
- return finalizeGatewayResponse(ctx, passthroughApiError(c, request.message, 400));
+ const ingress = await readIngress(c);
+ const result = readRequest(ingress.body.bytes);
+ if (result.type === 'invalid') {
+ // A request the gateway could not read never reaches a pipeline: there is no model to
+ // resolve and no attempt to make, so there is nothing for a run to record.
+ const refused = openPrologue(c, ingress, { wantsStream: false });
+ refused.gateway.dump?.error('gateway');
+ return finalizeGatewayResponse(
+ refused.gateway,
+ Response.json({ error: { message: result.message, type: 'api_error' } }, { status: 400 }),
+ );
}
- ctx.dump?.requestedModel(request.model);
- const response = await passthroughServe({
+ const { model, request } = result.parsed;
+ const prologue = openPrologue(c, ingress, { wantsStream: false, model });
+
+ return await serveThrough(
c,
- ctx,
- sourceApi: '/embeddings',
- operation: 'embeddings',
- model: request.model,
- kind: 'embedding',
- modelServesEndpoint: model => model.endpoints.openaiEmbeddings !== undefined,
- call: async (provider, model, opts) => {
- const { model: _model, ...body } = request.body;
- return await provider.instance.callOpenAIEmbeddings(model, body, undefined, opts);
- },
- response: { format: 'json', extractBilling: tokenUsageFromOpenAIEmbeddingsBody },
- });
- return finalizeGatewayResponse(ctx, response);
+ prologue,
+ openaiEmbeddingsServePipeline,
+ move({
+ 'ingress.http.headers': prologue.headers,
+ 'ingress.openaiEmbeddings.encodingFormat': request.encodingFormat,
+ 'request.openaiEmbeddings.canonical': request,
+ 'serve.model': model,
+ }) as never,
+ facts => ({ body: JSON.stringify(facts['response.openaiEmbeddings.rendered']), contentType: 'application/json' }),
+ );
};
diff --git a/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts b/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts
new file mode 100644
index 0000000000..3ed13760b5
--- /dev/null
+++ b/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts
@@ -0,0 +1,219 @@
+// OpenAI Embeddings as a pipeline. The smallest family there is: one protocol, no translation
+// and no stream — so what is left is the four stages every family has, and nothing else.
+//
+// emitOpenAIEmbeddings the edge: writes the answer in the encoding the client asked
+// for
+// resolveCandidates narrows to the upstreams that can serve this request
+// failover runs what follows once per candidate
+// callOpenAIEmbeddingsUpstream the ending: dials, and provides what came back
+//
+// The narrowing is a constant rather than a function of the request, and that is what makes
+// this the simple family: an OpenAI Embeddings request carries nothing a candidate could be
+// incompatible with, so the only question is whether the upstream exposes the endpoint.
+
+import type { UsageQuantities } from '../../repo/types.ts';
+import type { Failure, GatewayFacts } from '../pipeline/facts.ts';
+import { isFailure } from '../pipeline/facts.ts';
+import type { GatewayServices } from '../pipeline/services.ts';
+import { writeSettlement } from '../pipeline/settlement.ts';
+import { failover, resolveCandidates, type Narrowing } from '../pipeline/stages.ts';
+import { dialFailure, readUpstreamBody, unreadableBody } from '../pipeline/upstream-body.ts';
+import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts';
+import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts';
+import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts';
+import type { Pipeline } from '@floway-dev/pipeline';
+import { compose, defineStage, move } from '@floway-dev/pipeline';
+import { parseDecimalString, renderErrorEnvelope, upstreamErrorMessage } from '@floway-dev/protocols/common';
+import {
+ parseOpenAIEmbeddingsResponse,
+ renderOpenAIEmbeddingsResponse,
+ serializeOpenAIEmbeddingsRequest,
+ type CanonicalOpenAIEmbeddingsRequest,
+ type CanonicalOpenAIEmbeddingsResponse,
+ type CanonicalOpenAIEmbeddingsUsage,
+ type OpenAIEmbeddingsEncodingFormat,
+} from '@floway-dev/protocols/openai-embeddings';
+import { providerModelOf } from '@floway-dev/provider';
+
+/** OpenAI Embeddings' own keys. They extend the shared space and never merge into it, so a stage
+ * written against the gateway alone cannot name one. */
+export interface OpenAIEmbeddingsFacts extends GatewayFacts {
+ /** Which encoding the client is able to read. It belongs to the ingress and stays put:
+ * the answer is written in it whichever encoding the upstream chose to answer in, and
+ * the two differ whenever an upstream ignores the field. */
+ 'ingress.openaiEmbeddings.encodingFormat': OpenAIEmbeddingsEncodingFormat;
+ 'request.openaiEmbeddings.canonical': CanonicalOpenAIEmbeddingsRequest;
+ 'response.openaiEmbeddings.canonical': CanonicalOpenAIEmbeddingsResponse | Failure;
+ /** What the client is actually sent. The edge provides it, so a dump shows the bytes the
+ * client received rather than the gateway's canonical form. */
+ 'response.openaiEmbeddings.rendered': Record;
+}
+
+type E = { [P in K]: OpenAIEmbeddingsFacts[P] };
+
+/**
+ * The outermost edge. Writes the vectors in the encoding the client asked for — which is
+ * why the encoding is an ingress fact and not a request one: it has to survive an upstream
+ * that answered in the other one. A client on an official OpenAI SDK asked for `base64`
+ * without its caller choosing to, and will decode as base64 whatever it is handed.
+ */
+const emitOpenAIEmbeddings = defineStage<
+ E<'ingress.openaiEmbeddings.encodingFormat'>,
+ E<'ingress.openaiEmbeddings.encodingFormat'>,
+ E<'ingress.openaiEmbeddings.encodingFormat' | 'response.openaiEmbeddings.canonical' | 'response.http.headers'>,
+ E<'response.openaiEmbeddings.rendered' | 'response.http.status' | 'response.http.headers'>
+>({
+ name: 'emitOpenAIEmbeddings',
+ through: {
+ request: {
+ needs: ['ingress.openaiEmbeddings.encodingFormat'],
+ consumes: [],
+ provides: [],
+ },
+ response: {
+ needs: ['response.openaiEmbeddings.canonical', 'response.http.headers'],
+ consumes: ['response.openaiEmbeddings.canonical', 'response.http.headers'],
+ provides: ['response.openaiEmbeddings.rendered', 'response.http.status', 'response.http.headers'],
+ },
+ },
+ execute: async (facts, next) => {
+ const back = await next(facts);
+ const { 'response.openaiEmbeddings.canonical': answer, 'response.http.headers': headers, ...rest } = back;
+ // Vendor traces and quota state stay visible; what an intermediary must strip, and what
+ // would misdescribe a body this gateway serialized itself, does not. A filter that removed
+ // nothing hands the same array on, so the record shows no change where none happened.
+ const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name));
+ const forClient = forwardable.length === headers.length ? headers : move(forwardable);
+ if (isFailure(answer)) {
+ return {
+ ...rest,
+ 'response.http.headers': forClient,
+ 'response.openaiEmbeddings.rendered': move(renderErrorEnvelope(answer.message, answer.body)),
+ // The upstream's own status, or the gateway's own when it refused before dialling.
+ // A client is not owed the upstream's exact bytes; it is owed the truth about what
+ // happened, and a 429 arriving as a 200 is not that.
+ 'response.http.status': answer.status,
+ };
+ }
+ return {
+ ...rest,
+ 'response.http.headers': forClient,
+ 'response.http.status': 200,
+ 'response.openaiEmbeddings.rendered': move(renderOpenAIEmbeddingsResponse(back['ingress.openaiEmbeddings.encodingFormat'], answer)),
+ };
+ },
+});
+
+/**
+ * The ending. It dials, reads the upstream's body, and provides the canonical answer and
+ * what the call is billable for. A failure is a value: a 429 here is what an earlier stage
+ * fails over, and even a 400 can be, because the next candidate's path and flags may differ.
+ */
+const callOpenAIEmbeddingsUpstream = defineStage<
+ E<'request.openaiEmbeddings.canonical' | 'route.attempt' | 'ingress.http.headers' | 'serve.model'>,
+ E<'response.openaiEmbeddings.canonical' | 'response.http.headers' | 'response.usage.billable'>,
+ GatewayServices
+>({
+ name: 'callOpenAIEmbeddingsUpstream',
+ return: {
+ provides: ['response.openaiEmbeddings.canonical', 'response.http.headers', 'response.usage.billable'],
+ },
+ execute: async (facts, use) => {
+ const candidate = use.resolveAttempt(facts['route.attempt']);
+ // Attribution is set before the dial, so an attempt that never completes still names the
+ // candidate it was made against rather than the one tried before it.
+ use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'embeddings');
+
+ let result;
+ try {
+ result = await candidate.provider.instance.callOpenAIEmbeddings(
+ providerModelOf(candidate),
+ serializeOpenAIEmbeddingsRequest(facts['request.openaiEmbeddings.canonical']),
+ use.gateway.abortSignal,
+ // The client's own headers reach the upstream from the record, not from a live
+ // request object: what a provider is allowed to forward is filtered per provider,
+ // and the dump shows what was there to filter.
+ buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))),
+ );
+ } catch (error) {
+ use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) });
+ // A dial that never completed reached no upstream, so nothing was billed and there are
+ // no headers to carry. What it leaves behind is the performance row settlement writes.
+ return move({
+ ...facts,
+ 'response.openaiEmbeddings.canonical': dialFailure(error),
+ 'response.http.headers': [],
+ 'response.usage.billable': [],
+ });
+ }
+
+ const identity = telemetryModelIdentity(candidate, result.modelKey);
+ // What came back, unfiltered: the edge is where a client's view of it is decided.
+ const headers = [...result.response.headers];
+ const body = await readUpstreamBody(result.response);
+ // The upstream was called and reported nothing, which is a different situation from
+ // reporting zero — so the entity is present with no quantities.
+ const answered = (canonical: CanonicalOpenAIEmbeddingsResponse | Failure, quantities: UsageQuantities) => move({
+ ...facts,
+ 'response.openaiEmbeddings.canonical': canonical,
+ 'response.http.headers': headers,
+ 'response.usage.billable': [{ identity, quantities }],
+ });
+ const reportedNothing: UsageQuantities = {};
+
+ if (!result.response.ok) {
+ use.log.warn('upstream refused', { status: result.response.status });
+ return answered({
+ status: result.response.status,
+ message: upstreamErrorMessage(body.json) ?? body.text,
+ ...('json' in body ? { body: body.json } : {}),
+ }, reportedNothing);
+ }
+ if (!('json' in body)) {
+ return answered(unreadableBody(result.response, body, 'the OpenAI Embeddings protocol'), reportedNothing);
+ }
+
+ // Every protocol the gateway carries is one it fully understands: the body is parsed
+ // here and written again at the edge, in whichever encoding the client can read.
+ let canonical: CanonicalOpenAIEmbeddingsResponse;
+ try {
+ canonical = parseOpenAIEmbeddingsResponse(body.json, facts['serve.model']);
+ } catch (error) {
+ use.log.warn('upstream answered with a body the OpenAI Embeddings protocol cannot read', { error: String(error) });
+ return answered(unreadableBody(result.response, body, 'the OpenAI Embeddings protocol'), reportedNothing);
+ }
+ return answered(canonical, billed(canonical.usage));
+ },
+});
+
+// An OpenAI Embeddings call has no output side, so `prompt_tokens` is the whole of what the
+// upstream metered and `total_tokens` restates it. An upstream that reported nothing bills
+// nothing, and says so by leaving the entity's quantities empty.
+const billed = (usage: CanonicalOpenAIEmbeddingsUsage | undefined): UsageQuantities =>
+ usage === undefined ? {} : { input_tokens: parseDecimalString(String(usage.promptTokens)) };
+
+/** A candidate that cannot serve *this* request is not a candidate. Saying why is what
+ * turns an empty list into a 400 a client can act on. */
+const narrowing: Narrowing> = {
+ kind: 'embedding',
+ reject: candidate => candidate.model.endpoints.openaiEmbeddings === undefined
+ ? 'the upstream does not expose an OpenAI Embeddings endpoint'
+ : null,
+ unsupported: model => `Model ${model} does not support the /embeddings endpoint.`,
+ refuse: (status, message) => ({ 'response.openaiEmbeddings.canonical': { status, message } }),
+ refuses: ['response.openaiEmbeddings.canonical'],
+};
+
+export const openaiEmbeddingsServePipeline: Pipeline<
+ E<'ingress.http.headers' | 'ingress.openaiEmbeddings.encodingFormat' | 'request.openaiEmbeddings.canonical' | 'serve.model'>,
+ E<'response.openaiEmbeddings.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>
+> = compose('openaiEmbeddingsServe', [
+ emitOpenAIEmbeddings,
+ writeSettlement(handedUp => isFailure((handedUp as { 'response.openaiEmbeddings.canonical'?: unknown })['response.openaiEmbeddings.canonical'])),
+ resolveCandidates(narrowing),
+ failover({
+ failed: handedUp => isFailure((handedUp as { 'response.openaiEmbeddings.canonical'?: unknown })['response.openaiEmbeddings.canonical']),
+ owns: [],
+ }),
+ callOpenAIEmbeddingsUpstream,
+]);
diff --git a/packages/gateway/src/data-plane/openai-images/http.ts b/packages/gateway/src/data-plane/openai-images/http.ts
index 76a68bd627..6ecac99d63 100644
--- a/packages/gateway/src/data-plane/openai-images/http.ts
+++ b/packages/gateway/src/data-plane/openai-images/http.ts
@@ -1,174 +1,75 @@
-// POST /v1/images/generations and POST /v1/images/edits — route image
-// requests to the provider that declares the requested model and the
-// matching image endpoint capability.
+// POST /v1/images/generations and POST /v1/images/edits, served through the pipeline.
//
-// The edits handler accepts multipart uploads and JSON `images` references.
-// Both are buffered once for dump capture and normalized into a semantic
-// request; each provider owns the final JSON or multipart serialization.
-// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620
+// Two endpoints and one family. What the endpoint decides is which contract reads the body —
+// generations is always JSON, edits is JSON or a multipart form carrying the files — and
+// after that both hand the same canonical request to the same run.
import type { Context } from 'hono';
-import { backgroundSchedulerFromContext } from '../../runtime/background.ts';
-import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../shared/gateway-ctx.ts';
-import { prepareJsonModelRequest } from '../shared/passthrough-request.ts';
-import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts';
-import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts';
-import { tokenUsageFromOpenAIImagesBody } from '../shared/telemetry/usage.ts';
-import { isJsonMediaType, isMultipartFormDataMediaType } from '@floway-dev/protocols/common';
-import type { OpenAIImageEditReference } from '@floway-dev/protocols/openai-images';
-import { isBase64ImageDataUrl, type OpenAIImagesEditsRequest, type OpenAIImagesEditsSource } from '@floway-dev/provider';
+import { openaiImagesServePipeline } from './pipeline.ts';
+import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../pipeline/serve.ts';
+import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts';
+import { move } from '@floway-dev/pipeline';
+import { openaiImagesRequestWantsStream, parseOpenAIImagesEditsRequest, parseOpenAIImagesGenerationsRequest, type ParsedOpenAIImagesRequest } from '@floway-dev/protocols/openai-images';
-type PreparedOpenAIImagesEdit =
- | { type: 'ok'; request: OpenAIImagesEditsRequest }
- | { type: 'invalid'; message: string };
-
-const openaiImageEditSource = (value: unknown, path: string): OpenAIImagesEditsSource | string => {
- if (value === null || typeof value !== 'object' || Array.isArray(value)) {
- return `${path} must be an object.`;
- }
- const reference = value as { image_url?: unknown; file_id?: unknown };
- const { image_url: imageUrl, file_id: fileId } = reference;
- if (typeof imageUrl === 'string' && fileId === undefined) {
- const imageReference = reference as OpenAIImageEditReference & { image_url: string };
- return isBase64ImageDataUrl(imageUrl)
- ? { type: 'inline', reference: imageReference }
- : { type: 'reference', reference: imageReference };
- }
- if (typeof fileId === 'string' && imageUrl === undefined) {
- return { type: 'reference', reference: reference as OpenAIImageEditReference };
- }
- return `${path} must contain exactly one string field: image_url or file_id.`;
-};
-
-const prepareJsonOpenAIImagesEdit = (body: Record): PreparedOpenAIImagesEdit => {
- if (!Array.isArray(body.images)) {
- return { type: 'invalid', message: 'OpenAI Images Edits request body must include an images array.' };
- }
- const images: OpenAIImagesEditsSource[] = [];
- for (const [index, value] of body.images.entries()) {
- const source = openaiImageEditSource(value, `OpenAI Images Edits images[${index}]`);
- if (typeof source === 'string') return { type: 'invalid', message: source };
- images.push(source);
- }
- let mask: OpenAIImagesEditsSource | undefined;
- if (body.mask !== undefined) {
- const source = openaiImageEditSource(body.mask, 'OpenAI Images Edits mask');
- if (typeof source === 'string') return { type: 'invalid', message: source };
- mask = source;
+/** The half both endpoints share: whatever the contract read, hand it over and turn what the
+ * run answered with into a response. The contract reports a malformed request by throwing;
+ * what the client is owed is a 400 carrying the reason. */
+const serveOpenAIImages = async (
+ c: Context,
+ ingress: Ingress,
+ read: () => ParsedOpenAIImagesRequest | Promise,
+): Promise => {
+ let parsed: ParsedOpenAIImagesRequest;
+ try {
+ parsed = await read();
+ } catch (error) {
+ // A request the gateway could not read never reaches a pipeline: there is no model to
+ // resolve and no attempt to make, so there is nothing for a run to record.
+ const refused = openPrologue(c, ingress, { wantsStream: false });
+ refused.gateway.dump?.error('gateway');
+ return finalizeGatewayResponse(
+ refused.gateway,
+ Response.json(
+ { error: { message: error instanceof Error ? error.message : String(error), type: 'api_error' } },
+ { status: 400 },
+ ),
+ );
}
- const { model: _model, images: _images, mask: _mask, ...parameters } = body;
- return {
- type: 'ok',
- request: {
- images,
- ...(mask === undefined ? {} : { mask }),
- parameters,
- },
- };
-};
-export const openaiImagesGenerations = async (c: Context): Promise => {
- const requestBody = await readRequestBody(c);
- const request = prepareJsonModelRequest(requestBody.bytes, 'OpenAI Images Generations');
- const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) });
- if (request.type === 'invalid') {
- ctx.dump?.error('gateway');
- return finalizeGatewayResponse(ctx, passthroughApiError(c, request.message, 400));
- }
+ const { model, request } = parsed;
+ // Whether the answer streams is written in the request the client sent, and the run has to be
+ // opened knowing it: the abort controller a streaming run cancels its read with is minted
+ // from this flag.
+ const wantsStream = openaiImagesRequestWantsStream(request);
+ const prologue = openPrologue(c, ingress, { wantsStream, model });
- ctx.dump?.requestedModel(request.model);
- const response = await passthroughServe({
+ return await serveThrough(
c,
- ctx,
- sourceApi: '/images/generations',
- operation: 'image_generation',
- model: request.model,
- kind: 'image',
- modelServesEndpoint: model => model.endpoints.openaiImagesGenerations !== undefined,
- call: (provider, model, opts) => {
- const { model: _model, ...body } = request.body;
- return provider.instance.callOpenAIImagesGenerations(model, body, undefined, opts);
+ prologue,
+ openaiImagesServePipeline(request),
+ move({
+ 'ingress.http.headers': prologue.headers,
+ 'ingress.openaiImages.wantsStream': wantsStream,
+ 'request.openaiImages.canonical': request,
+ 'serve.model': model,
+ }) as never,
+ facts => {
+ const answer = facts['response.openaiImages.rendered'];
+ return isFrames(answer) ? { frames: answer } : { body: JSON.stringify(answer), contentType: 'application/json' };
},
- response: { format: 'json', extractBilling: tokenUsageFromOpenAIImagesBody },
- });
- return finalizeGatewayResponse(ctx, response);
+ facts => facts['response.openaiImages.streamedUsage'],
+ );
};
-const serveOpenAIImagesEditRequest = async (
- c: Context,
- requestBody: RequestBody,
- model: string,
- request: OpenAIImagesEditsRequest,
-): Promise => {
- const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) });
- ctx.dump?.requestedModel(model);
- const response = await passthroughServe({
- c,
- ctx,
- sourceApi: '/images/edits',
- operation: 'image_edit',
- model,
- kind: 'image',
- modelServesEndpoint: model => model.endpoints.openaiImagesEdits !== undefined,
- call: (provider, model, opts) => provider.instance.callOpenAIImagesEdits(model, request, undefined, opts),
- response: { format: 'json', extractBilling: tokenUsageFromOpenAIImagesBody },
- });
- return finalizeGatewayResponse(ctx, response);
+export const openaiImagesGenerations = async (c: Context): Promise => {
+ const ingress = await readIngress(c);
+ return await serveOpenAIImages(c, ingress, () => parseOpenAIImagesGenerationsRequest(ingress.body.bytes));
};
export const openaiImagesEdits = async (c: Context): Promise => {
- const requestBody = await readRequestBody(c);
- const invalid = (message: string): Response => {
- const errorCtx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) });
- errorCtx.dump?.error('gateway');
- return finalizeGatewayResponse(errorCtx, passthroughApiError(c, message, 400));
- };
-
- const contentType = c.req.header('content-type');
- if (contentType === undefined) {
- return invalid('OpenAI Images Edits request body must use application/json or multipart/form-data.');
- }
- if (isJsonMediaType(contentType)) {
- const body = prepareJsonModelRequest(requestBody.bytes, 'OpenAI Images Edits');
- if (body.type === 'invalid') return invalid(body.message);
- const request = prepareJsonOpenAIImagesEdit(body.body);
- if (request.type === 'invalid') return invalid(request.message);
- return await serveOpenAIImagesEditRequest(c, requestBody, body.model, request.request);
- }
-
- if (!isMultipartFormDataMediaType(contentType)) {
- return invalid('OpenAI Images Edits request body must use application/json or multipart/form-data.');
- }
- let form: FormData;
- try {
- form = await new Response(requestBody.bytes as BodyInit, { headers: { 'content-type': contentType } }).formData();
- } catch {
- return invalid('OpenAI Images Edits request body must be valid multipart/form-data.');
- }
- const model = form.get('model');
- if (typeof model !== 'string' || model.length === 0) {
- return invalid('OpenAI Images Edits request body must include a model field.');
- }
- const images: File[] = [];
- let mask: File | undefined;
- const parameters: Record = {};
- for (const [name, value] of form.entries()) {
- if (name === 'model') continue;
- if (name === 'image' || name === 'image[]') {
- if (!(value instanceof File)) return invalid(`OpenAI Images Edits ${name} fields must be files.`);
- images.push(value);
- } else if (name === 'mask') {
- if (!(value instanceof File)) return invalid('OpenAI Images Edits mask field must be a file.');
- mask = value;
- } else {
- if (typeof value !== 'string') return invalid(`OpenAI Images Edits ${name} field must be text.`);
- parameters[name] = value;
- }
- }
- return await serveOpenAIImagesEditRequest(c, requestBody, model, {
- images: images.map(file => ({ type: 'upload', file })),
- ...(mask === undefined ? {} : { mask: { type: 'upload' as const, file: mask } }),
- parameters,
- });
+ const ingress = await readIngress(c);
+ // Which of the two bodies arrived is a header's statement and not the payload's, so the
+ // contract is handed the media type alongside the bytes.
+ return await serveOpenAIImages(c, ingress, () => parseOpenAIImagesEditsRequest(c.req.header('content-type'), ingress.body.bytes));
};
diff --git a/packages/gateway/src/data-plane/openai-images/pipeline.ts b/packages/gateway/src/data-plane/openai-images/pipeline.ts
new file mode 100644
index 0000000000..4151a93971
--- /dev/null
+++ b/packages/gateway/src/data-plane/openai-images/pipeline.ts
@@ -0,0 +1,429 @@
+// OpenAI Images as a pipeline. Two endpoints and one family: `generations` sends JSON and
+// `edits` sends either JSON or a multipart form, so what the operation decides is the request
+// fact and the provider method the ending calls, and the four stages are the same either way.
+// Both endpoints accept `stream: true`, so the answer is a stream whenever the client asked for
+// one — the same request field decides both what the upstream is asked for and what shape comes
+// back.
+//
+// emitOpenAIImages the edge: serializes the answer into the OpenAI Images protocol,
+// SSE framing included
+// resolveCandidates narrows to the upstreams that expose this operation's endpoint
+// failover runs what follows once per candidate
+// callOpenAIImagesUpstream the ending: dials, and provides what came back
+
+import { recordStream, streamReferenceOf } from '../../dump/turn-dump.ts';
+import type { UsageQuantities } from '../../repo/types.ts';
+import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts';
+import { isFailure } from '../pipeline/facts.ts';
+import type { StreamOutcome } from '../pipeline/serve.ts';
+import type { GatewayServices } from '../pipeline/services.ts';
+import { writeSettlement } from '../pipeline/settlement.ts';
+import { failover, resolveCandidates } from '../pipeline/stages.ts';
+import { dialFailure, readUpstreamBody } from '../pipeline/upstream-body.ts';
+import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts';
+import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts';
+import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts';
+import { compose, defer, defineStage, move, own, type Deferred, type Owned, type Pipeline } from '@floway-dev/pipeline';
+import {
+ eventFrame,
+ isEventStreamMediaType,
+ mediaTypeEssence,
+ parseDecimalString,
+ renderErrorEnvelope,
+ upstreamErrorMessage,
+ type ModelEndpointKey,
+ type SseFrame,
+} from '@floway-dev/protocols/common';
+import {
+ OPENAI_IMAGES_MISSING_TERMINAL_MESSAGE,
+ openaiImagesStreamEventToSSEFrame,
+ isOpenAIImagesTerminalEvent,
+ parseOpenAIImagesResponse,
+ parseOpenAIImagesStream,
+ parseOpenAIImagesUsage,
+ renderOpenAIImagesResponse,
+ type CanonicalOpenAIImagesEditsRequest,
+ type CanonicalOpenAIImagesRequest,
+ type CanonicalOpenAIImagesResponse,
+ type CanonicalOpenAIImagesUsage,
+ type OpenAIImageEditReference,
+ type OpenAIImagesEditImage,
+ type OpenAIImagesOperation,
+ type OpenAIImagesStreamEvent,
+} from '@floway-dev/protocols/openai-images';
+import { isBase64ImageDataUrl, providerModelOf } from '@floway-dev/provider';
+import type { OpenAIImagesEditsRequest, OpenAIImagesEditsSource, ModelCandidate, PerformanceOperation, TelemetryModelIdentity } from '@floway-dev/provider';
+
+/** The answer while it is still the upstream's, one event at a time. There is no transport
+ * sentinel below the events — this protocol ends at its completed event — so what travels is
+ * the events themselves rather than frames with a terminal arm the protocol has not got. */
+export type OpenAIImagesFrames = AsyncIterable;
+
+/** A stream as a value the record can hold, as a wrapper around the generator rather than the
+ * generator itself. What the wrapper says is where the resource is: the one resource in an
+ * OpenAI Images run is the upstream's body at `response.http.body`, claimed with `own()`, and
+ * this keeps a frame view from reading as another. */
+const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames });
+
+/** OpenAI Images' own keys. They extend the shared space and never merge into it, so a stage
+ * written against the gateway alone cannot name one. */
+export interface OpenAIImagesFacts extends GatewayFacts {
+ /** Whether the client asked for the answer as a stream. It stays put, as every `ingress.*`
+ * key does: the same flag travels to the upstream inside the parameters, and the answer is
+ * written back in the shape it asked for. */
+ 'ingress.openaiImages.wantsStream': boolean;
+ 'request.openaiImages.canonical': CanonicalOpenAIImagesRequest;
+ /** The answer, whichever kind it turned out to be. A stream, a value and a failure sit at
+ * one key: telling them apart is reading a value, and each stage does that where it needs
+ * to. */
+ 'response.openaiImages.canonical': CanonicalOpenAIImagesResponse | OpenAIImagesFrames | Failure;
+ /** What the upstream will have reported by the time the events run out, and `null` on every
+ * path that does not stream. A streamed image states its usage in the completed event,
+ * which is after this run has answered — so the numbers cannot be in
+ * `response.usage.billable`, which says what had been reported when the ending stage handed
+ * up: the entity, and no quantities. Settling from this is the epilogue's job, after the
+ * drain. */
+ 'response.openaiImages.streamedUsage': Deferred | null;
+ /** What the client is actually sent, in the OpenAI Images protocol — a JSON body, or the SSE
+ * frames of a stream. The edge provides it, so a dump shows the body the client received
+ * rather than the gateway's canonical form. */
+ 'response.openaiImages.rendered': Record | AsyncIterable;
+}
+
+type I = { [P in K]: OpenAIImagesFacts[P] };
+
+const ENDPOINT = {
+ generations: 'openaiImagesGenerations',
+ edits: 'openaiImagesEdits',
+} as const satisfies Record;
+
+const PERFORMANCE_OPERATION = {
+ generations: 'image_generation',
+ edits: 'image_edit',
+} as const satisfies Record;
+
+const isFrames = (answer: CanonicalOpenAIImagesResponse | OpenAIImagesFrames): answer is OpenAIImagesFrames =>
+ Symbol.asyncIterator in answer;
+
+/**
+ * The outermost edge. It names nothing on the way down — the family has one protocol, so there
+ * is no request to read to know how to answer — and coming back it renders the answer, keeps
+ * the upstream headers a client may see, and decides the status. The status is decided here
+ * rather than carried up because a refusal that never reached an upstream has none to carry.
+ *
+ * SSE framing is produced here and nowhere else: below this stage a stream carries protocol
+ * events, so the same assembly would serve another transport by rendering differently at this
+ * one point.
+ */
+const emitOpenAIImages = defineStage<
+ Record,
+ Record,
+ I<'response.openaiImages.canonical' | 'response.http.headers'>,
+ I<'response.openaiImages.rendered' | 'response.http.status' | 'response.http.headers'>
+>({
+ name: 'emitOpenAIImages',
+ through: {
+ request: { needs: [], consumes: [], provides: [] },
+ response: {
+ needs: ['response.openaiImages.canonical', 'response.http.headers'],
+ consumes: ['response.openaiImages.canonical', 'response.http.headers'],
+ provides: ['response.openaiImages.rendered', 'response.http.status', 'response.http.headers'],
+ },
+ },
+ execute: async (facts, next) => {
+ const back = await next(facts);
+ const { 'response.openaiImages.canonical': answer, 'response.http.headers': headers, ...rest } = back;
+ // Vendor traces and quota state stay visible; what an intermediary must strip, and what
+ // would misdescribe a body this gateway serialized itself, does not. A filter that removed
+ // nothing hands the same array on, so the record shows no change where none happened.
+ const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name));
+ return {
+ ...rest,
+ 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable),
+ 'response.http.status': isFailure(answer) ? answer.status : 200,
+ 'response.openaiImages.rendered': move(rendered(answer)),
+ };
+ },
+});
+
+const rendered = (answer: OpenAIImagesFacts['response.openaiImages.canonical']): OpenAIImagesFacts['response.openaiImages.rendered'] =>
+ isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body)
+ : isFrames(answer) ? renderSSE(answer)
+ : renderOpenAIImagesResponse(answer);
+
+const renderSSE = (frames: OpenAIImagesFrames): AsyncIterable => ({
+ // The frames the client reads are a reframing of the ones the record holds, so this key
+ // points at that same stream rather than at nothing.
+ ...streamReferenceOf(frames),
+ [Symbol.asyncIterator]: () => (async function* () {
+ for await (const event of frames) yield openaiImagesStreamEventToSSEFrame(event);
+ })(),
+});
+
+/**
+ * The ending. It dials, reads what came back on the shape the request asked for, and provides
+ * the canonical answer, the headers that came with it, the raw HTTP body beneath it, and what
+ * the call is billable for. A failure is a value: an upstream that refused, and one that
+ * answered with something this protocol cannot read, are both outcomes the fork above can take
+ * to the next candidate rather than faults that end the run.
+ */
+const callOpenAIImagesUpstream = defineStage<
+ I<'ingress.openaiImages.wantsStream' | 'request.openaiImages.canonical' | 'route.attempt' | 'ingress.http.headers'>,
+ I<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.headers'
+ | 'response.http.body' | 'response.usage.billable'>,
+ GatewayServices
+>({
+ name: 'callOpenAIImagesUpstream',
+ return: {
+ provides: [
+ 'response.openaiImages.canonical',
+ 'response.openaiImages.streamedUsage',
+ 'response.http.headers',
+ 'response.http.body',
+ 'response.usage.billable',
+ ],
+ },
+ execute: async (facts, use) => {
+ const candidate = use.resolveAttempt(facts['route.attempt']);
+ const request = facts['request.openaiImages.canonical'];
+ const options = buildUpstreamCallOptions(
+ candidate,
+ use.gateway,
+ // The client's own headers reach the upstream from the record, not from a live request
+ // object: what a provider is allowed to forward is filtered per provider, and the dump
+ // shows what was there to filter.
+ new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])),
+ );
+ const model = providerModelOf(candidate);
+ // Attribution is set before the dial, so an attempt that never completes still names the
+ // candidate it was made against rather than the one tried before it.
+ use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, PERFORMANCE_OPERATION[request.operation]);
+
+ let result;
+ try {
+ // No abort signal, as on this family's endpoints from the beginning: an image the upstream
+ // has already begun is charged for whether or not the client waited for it, so dropping the
+ // call would lose the usage reading and save nothing. Reading a stream is the other half of
+ // that and does take the signal — what it drops there is a connection, not an image.
+ result = request.operation === 'generations'
+ ? await candidate.provider.instance.callOpenAIImagesGenerations(model, request.parameters, undefined, options)
+ : await candidate.provider.instance.callOpenAIImagesEdits(model, providerEditsRequest(request), undefined, options);
+ } catch (error) {
+ use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) });
+ // A dial that never completed reached no upstream, so nothing was billed and there are
+ // no headers to carry. What it leaves behind is the performance row settlement writes.
+ return move({
+ ...facts,
+ 'response.openaiImages.canonical': dialFailure(error),
+ 'response.openaiImages.streamedUsage': null,
+ 'response.http.headers': [],
+ 'response.http.body': spentBody(null),
+ 'response.usage.billable': [],
+ });
+ }
+
+ const identity = telemetryModelIdentity(candidate, result.modelKey);
+ // What came back, unfiltered: the edge is where a client's view of it is decided.
+ const headers = [...result.response.headers];
+ // An entity with no quantities is how "the upstream was called and reported nothing" is
+ // said, which is a different situation from reporting zero.
+ const called: readonly BillableEntity[] = [{ identity, quantities: {} }];
+ const read = (canonical: CanonicalOpenAIImagesResponse | Failure, billable: readonly BillableEntity[]) => move({
+ ...facts,
+ 'response.openaiImages.canonical': canonical,
+ 'response.openaiImages.streamedUsage': null,
+ 'response.http.headers': headers,
+ 'response.http.body': spentBody(result.response.body),
+ 'response.usage.billable': billable,
+ });
+
+ if (!result.response.ok) {
+ use.log.warn('upstream refused', { status: result.response.status });
+ // An upstream error body is JSON like any other body. Reading it here is also what
+ // leaves a losing attempt with nothing open behind it.
+ const body = await readUpstreamBody(result.response);
+ return read({
+ status: result.response.status,
+ message: upstreamErrorMessage(body.json) ?? body.text,
+ ...('json' in body ? { body: body.json } : {}),
+ }, called);
+ }
+
+ const mediaType = result.response.headers.get('content-type');
+ // Both halves have to hold. `stream` is what the client asked for, and an upstream that
+ // ignores it answers the single JSON body the arm below reads — so what settles which
+ // shape arrived is the media type, and what settles which shape the client is owed is the
+ // request.
+ if (facts['ingress.openaiImages.wantsStream'] && isEventStreamMediaType(mediaType)) {
+ if (result.response.body === null) {
+ return read({ status: 502, message: 'Upstream returned a streaming response with no body.' }, called);
+ }
+ // Usage is observed here, closest to the upstream and on the protocol it spoke, by
+ // folding the events as they pass — so the reading costs one pass and the client's own
+ // stream is what drives it. What it finds arrives with the completed event, long after
+ // this stage has handed up, which is why the entity above carries no quantities.
+ const metered = meterFrames(result.response.body, identity, use.gateway.abortSignal);
+ return move({
+ ...facts,
+ // This protocol's stream is bare events rather than protocol frames, so the record is
+ // told how one becomes a frame instead of being left to assume.
+ 'response.openaiImages.canonical': recordStream(metered.frames, use.gateway.dump, eventFrame),
+ 'response.openaiImages.streamedUsage': metered.outcome,
+ 'response.http.headers': headers,
+ // Releasing this body is reading those events to the end: they are one reader over one
+ // connection, and a second reader is not something a `ReadableStream` allows.
+ 'response.http.body': own(result.response.body, async (): Promise => { for await (const _event of metered.frames) { /* to end of stream */ } }),
+ 'response.usage.billable': called,
+ });
+ }
+
+ const body = await readUpstreamBody(result.response);
+ if (!('json' in body)) {
+ // Every protocol the gateway carries is one it fully understands, so a body it cannot
+ // read is not handed on unread.
+ const essence = mediaTypeEssence(mediaType) ?? 'no media type';
+ use.log.warn('upstream answered with a body that is not JSON', { status: result.response.status, mediaType: essence });
+ return read({
+ status: 502,
+ message: `The upstream answered ${result.response.status} with ${essence}, and the OpenAI Images protocol is JSON.`,
+ body: body.text,
+ }, called);
+ }
+
+ let canonical: CanonicalOpenAIImagesResponse;
+ try {
+ canonical = parseOpenAIImagesResponse(body.json);
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ use.log.warn('upstream answered with a body the OpenAI Images protocol cannot read', { message });
+ return read({ status: 502, message, body: body.json }, called);
+ }
+ return read(canonical, [{ identity, quantities: billed(canonical.usage) }]);
+ },
+});
+
+/** A body this stage has already read to the end, or one the upstream never sent. The record
+ * holds a body as a stream and `failover` releases the losing attempts', so every path hands
+ * one up; what says an answer was unusable is the failure at the canonical key, not this. */
+const spentBody = (body: ReadableStream | null): ReadableStream & Owned =>
+ own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve());
+
+interface MeteredFrames {
+ readonly frames: OpenAIImagesFrames;
+ readonly outcome: Deferred;
+}
+
+const meterFrames = (
+ body: ReadableStream,
+ identity: TelemetryModelIdentity,
+ signal: AbortSignal | undefined,
+): MeteredFrames => {
+ let settle!: (outcome: StreamOutcome) => void;
+ // Declared as this run's own unfinished work, so the runner waits for it at teardown where
+ // it can see it rather than the reading being started and forgotten.
+ const outcome = defer(new Promise(resolve => { settle = resolve; }));
+ // Running out without the completed event is what "it did not finish" means, and it is known
+ // at the same moment the usage is.
+ let sawTerminal = false;
+ const frames = view((async function* () {
+ let usage: CanonicalOpenAIImagesUsage | undefined;
+ try {
+ for await (const event of parseOpenAIImagesStream(body, { signal })) {
+ if (isOpenAIImagesTerminalEvent(event)) {
+ usage = parseOpenAIImagesUsage(event);
+ sawTerminal = true;
+ yield event;
+ // The image is complete, so there is nothing further to read. An upstream that holds
+ // the connection open past this point would otherwise keep the client's own stream
+ // open with it; returning here closes the read, which cancels the upstream.
+ return;
+ }
+ yield event;
+ }
+ } finally {
+ // Reached however the events ended — the completed one, a client that stopped reading,
+ // or a broken upstream — because what the upstream already metered is billable whatever
+ // happened to the downstream half.
+ settle({ billable: [{ identity, quantities: billed(usage) }], failed: !sawTerminal });
+ }
+ // Only an upstream that ended its body without ever completing the image reaches here: the
+ // arm above returns on the terminal event. A client has been sent partial images and no
+ // image, which is a failed answer however far it got.
+ throw new Error(OPENAI_IMAGES_MISSING_TERMINAL_MESSAGE);
+ })());
+ return { frames, outcome };
+};
+
+/** An image is billed by the tokens its upstream reports: `BILLING_METRICS` names no per-image
+ * or per-size unit, and per-size pricing is a selector coordinate rather than a metric, so
+ * there is nothing else here to record a count against. A reported zero is kept — it says the
+ * upstream reported, which an absent metric would not. */
+const billed = (usage: CanonicalOpenAIImagesUsage | undefined): UsageQuantities => {
+ const quantities: UsageQuantities = {};
+ if (usage?.inputTokens !== undefined) quantities.input_tokens = parseDecimalString(String(usage.inputTokens));
+ if (usage?.inputImageTokens !== undefined) quantities.input_image_tokens = parseDecimalString(String(usage.inputImageTokens));
+ if (usage?.outputTokens !== undefined) quantities.output_tokens = parseDecimalString(String(usage.outputTokens));
+ if (usage?.outputImageTokens !== undefined) quantities.output_image_tokens = parseDecimalString(String(usage.outputImageTokens));
+ return quantities;
+};
+
+/** The provider's own shape, built where the dial happens. What it needs from a reference is
+ * whether the data URL inside it can become a file, because that decides whether the edit can
+ * ride as a multipart form; the canonical fact holds the reference the client wrote and leaves
+ * that question to the serializer that has it. */
+const providerEditsRequest = (request: CanonicalOpenAIImagesEditsRequest): OpenAIImagesEditsRequest => ({
+ images: request.images.map(providerEditsSource),
+ ...(request.mask === undefined ? {} : { mask: providerEditsSource(request.mask) }),
+ parameters: request.parameters,
+});
+
+const providerEditsSource = (image: OpenAIImagesEditImage): OpenAIImagesEditsSource => {
+ if (image.kind === 'file') {
+ return { type: 'upload', file: new File([image.file.bytes], image.file.fileName, { type: image.file.mediaType }) };
+ }
+ const { reference } = image;
+ return typeof reference.image_url === 'string' && isBase64ImageDataUrl(reference.image_url)
+ ? { type: 'inline', reference: reference as OpenAIImageEditReference & { image_url: string } }
+ : { type: 'reference', reference };
+};
+
+/** A candidate that cannot serve *this* request is not a candidate. One family covers two
+ * endpoints and an upstream may expose either without the other, so which one is asked for is
+ * what narrows the list. */
+const narrowing = (request: CanonicalOpenAIImagesRequest) => ({
+ kind: 'image' as const,
+ reject: (candidate: ModelCandidate) => candidate.model.endpoints[ENDPOINT[request.operation]] === undefined
+ ? `the upstream does not expose the OpenAI Images ${request.operation} endpoint`
+ : null,
+ unsupported: (model: string) => `Model ${model} does not support the /images/${request.operation} endpoint.`,
+ refuse: (status: number, message: string) => ({
+ 'response.openaiImages.canonical': { status, message } as Failure,
+ 'response.openaiImages.streamedUsage': null,
+ }),
+ refuses: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage'] as const,
+});
+
+/** What a caller must bring. `ingress.http.headers`, `ingress.openaiImages.wantsStream` and
+ * `request.openaiImages.canonical` are in it although `compose` cannot derive them — the ending
+ * stage reads all three and a return-only stage declares no request side — so this type is the
+ * whole statement and `entryNeeds` is part of it. */
+export type OpenAIImagesServeEntry = I<'ingress.http.headers' | 'ingress.openaiImages.wantsStream' | 'request.openaiImages.canonical' | 'serve.model'>;
+
+export type OpenAIImagesServeExit = I<
+ 'response.openaiImages.rendered' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'
+>;
+
+export const openaiImagesServePipeline = (request: CanonicalOpenAIImagesRequest): Pipeline =>
+ compose('openaiImagesServe', [
+ emitOpenAIImages,
+ writeSettlement(
+ handedUp => isFailure((handedUp as { 'response.openaiImages.canonical'?: unknown })['response.openaiImages.canonical']),
+ handedUp => (handedUp as { 'response.openaiImages.streamedUsage'?: unknown })['response.openaiImages.streamedUsage'] !== null,
+ ),
+ resolveCandidates(narrowing(request)),
+ failover({
+ failed: handedUp => isFailure((handedUp as { 'response.openaiImages.canonical'?: unknown })['response.openaiImages.canonical']),
+ owns: ['response.http.body'],
+ }),
+ callOpenAIImagesUpstream,
+ ]);
diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts
new file mode 100644
index 0000000000..e2a86227e7
--- /dev/null
+++ b/packages/gateway/src/data-plane/pipeline/facts.ts
@@ -0,0 +1,92 @@
+// The keys every family's pipeline shares. A family extends this space with its own
+// protocol keys by intersection and never merges into it, so a stage written here drops
+// into any family's pipeline and a family's own keys are unreachable from a stage that
+// was not written against them.
+//
+// Keys are namespaced, camelCase and family-first. `ingress.*` is what the client sent and
+// stays put across a protocol switch; `serve.*` belongs to the served request as a whole
+// and outlives an attempt; `request.*` and `response.*` are the two directions, mirrored
+// key for key and semantically disjoint.
+
+import type { UsageQuantities } from '../../repo/types.ts';
+import type { Secret, Owned } from '@floway-dev/pipeline';
+import type { PricingRuntimeFacts } from '@floway-dev/protocols/common';
+import type { TelemetryModelIdentity } from '@floway-dev/provider';
+
+/** Everything about an attempt that is data: which upstream, which model row on it, and the
+ * flags that row carries. Enough to choose, to record and to price — and to look the live
+ * candidate back up when the time comes to dial. */
+export interface AttemptSelector {
+ readonly upstreamId: string;
+ readonly modelId: string;
+ /** Snapshotted rather than referenced, because the record must show what was true when
+ * the attempt was made rather than what the row says now. */
+ readonly flags: readonly string[];
+}
+
+/** What an upstream call is answerable for. Keyed by billed entity, because one call can
+ * bill in units that are not commensurable, and an entity with no quantities at all is
+ * how "the upstream was called and reported nothing" is said. */
+export interface BillableEntity {
+ readonly identity: TelemetryModelIdentity;
+ readonly quantities: UsageQuantities;
+ /** What pricing needs beyond the quantities, when a rate depends on more than how much
+ * there was. Absent is a real reading and not a missing one: most families price on the
+ * quantities alone. Observed where the reading is, because settlement is the last reader
+ * and not a second observer. */
+ readonly pricingFacts?: PricingRuntimeFacts;
+}
+
+/** A failure is a value, never a throw, so an earlier stage can fail over a later stage's
+ * fault — even a 400, because the path and the flags may differ on the next candidate. */
+export interface Failure {
+ readonly status: number;
+ readonly message: string;
+ /** The upstream's own body, when there was one. A client is not owed the upstream's
+ * exact bytes, but a dump reader is owed what actually came back. */
+ readonly body?: unknown;
+}
+
+export const isFailure = (value: unknown): value is Failure =>
+ typeof value === 'object' && value !== null && 'status' in value && 'message' in value;
+
+export interface GatewayFacts {
+ /** What the client sent, before anything read it. Every family hands these over, because
+ * every ending forwards what a provider is allowed to forward of them. */
+ 'ingress.http.headers': readonly (readonly [string, string])[];
+
+ /** The public model id the client asked for, and the candidates it resolves to. Nothing
+ * consumes these: they outlive an attempt. */
+ 'serve.model': string;
+ 'serve.candidates': readonly AttemptSelector[];
+
+ /** Which upstream this attempt targets. Provided per attempt by the stage that forks.
+ *
+ * A **selector**, not the candidate itself. A `ModelCandidate` carries the provider's
+ * live instance, its fetcher and its models cache, and a live handle is never a fact —
+ * the test being whether it can be rendered into the dump. Putting one in the record
+ * deep-freezes all three, and the writes the provider relies on then fail *silently*,
+ * because a frozen write only throws in strict mode and the provider's own code is not
+ * the caller. The SWR models cache would stop refreshing with nothing to see.
+ *
+ * So the resolver is a service and the selector is the fact, which is the ruling as
+ * written. What travels is what identifies the attempt; what dials is injected. */
+ 'route.attempt': AttemptSelector;
+
+ /** There is exactly one url and one headers. Headers are rewritten the whole way down,
+ * so the dump shows a header's entire history in one place, and a value may be secret. */
+ 'request.http.url': string;
+ 'request.http.headers': readonly (readonly [string, string | Secret])[];
+
+ 'response.http.status': number;
+ 'response.http.headers': readonly (readonly [string, string])[];
+ /** The upstream's body, still open, and marked as something the run answers for. `Owned`
+ * rather than `AsyncDisposable`, because a structural type would say what the host happens
+ * to mark rather than what this run answers for — and what a host marks differs between the
+ * Node versions this ships on. Ownership is claimed, so the type says so too. */
+ 'response.http.body': ReadableStream & Owned;
+
+ /** The authoritative reading, provided closest to the upstream on the dialect it
+ * actually spoke. Every step that changes usage re-provides it. */
+ 'response.usage.billable': readonly BillableEntity[];
+}
diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts
new file mode 100644
index 0000000000..45b03e5c56
--- /dev/null
+++ b/packages/gateway/src/data-plane/pipeline/serve.ts
@@ -0,0 +1,253 @@
+// The seam. Every family's route handler does the same things around its pipeline — read
+// the body, build what the run is given, run it, turn the exit facts into a response — so
+// it is written once here and each family adds only its own keys.
+//
+// This is also where "entering the pipeline system requires no capability" is concrete: a
+// handler calls `run` like any other caller. What makes a run *recorded* is the prologue
+// resolving a dump sink, not a flag anywhere below it.
+
+import type { Context } from 'hono';
+import { streamSSE } from 'hono/streaming';
+import type { ContentfulStatusCode } from 'hono/utils/http-status';
+
+import type { AttemptSelector, BillableEntity, GatewayFacts } from './facts.ts';
+import type { GatewayServices } from './services.ts';
+import { settleBillable } from './settlement.ts';
+import { openRunDump } from '../../dump/run-sink.ts';
+import { apiKeyFromContext, type AuthedContext } from '../../middleware/auth.ts';
+import { internalErrorResponse } from '../../middleware/internal-error-response.ts';
+import { backgroundSchedulerFromContext } from '../../runtime/background.ts';
+import { consoleLogSink } from '../../runtime/log.ts';
+import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts';
+import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts';
+import { writeSSEFrames } from '../shared/sse.ts';
+import { run, type Deferred, type Pipeline } from '@floway-dev/pipeline';
+import { sseCommentFrame, type SseFrame } from '@floway-dev/protocols/common';
+import type { ModelCandidate } from '@floway-dev/provider';
+
+type Slice = { [P in K]: GatewayFacts[P] };
+
+/** What the client sent, read once, before anything has decided what it means. */
+export interface Ingress {
+ readonly body: RequestBody;
+ readonly headers: readonly (readonly [string, string])[];
+}
+
+/**
+ * Reads the request.
+ *
+ * This is separate from opening the run because whether a request streams is written *in*
+ * the request — `stream: true` in a JSON body, a form field in a multipart upload — and the
+ * body can only be read once. A run opened before the read would have to guess, and guessing
+ * `false` is not harmless: the abort controller a streaming run cancels its upstream with is
+ * minted from that flag, so a client that disconnected would stop cancelling anything.
+ */
+export const readIngress = async (c: Context): Promise => ({
+ body: await readRequestBody(c),
+ headers: [...c.req.raw.headers],
+});
+
+export interface Prologue {
+ readonly services: GatewayServices;
+ readonly gateway: GatewayCtx;
+ readonly headers: readonly (readonly [string, string])[];
+}
+
+/**
+ * Opens a run: the request context the telemetry stages read, and the services the stages
+ * are given. It takes the bytes the handler has already read and hands them to the dump,
+ * which is what leaves the handler's own copy free to be released.
+ *
+ * The candidate store is the resolver the ruling names — "the resolver is the service and
+ * the selector is a fact". A `ModelCandidate` carries the provider's instance, its fetcher
+ * and its models cache; those never enter the record, because `move()` would freeze them
+ * and the provider's own cache refresh would break. So the stage that enumerates hands the
+ * live ones here, and the stage that dials asks for one back by selector.
+ */
+export const openPrologue = (
+ c: AuthedContext,
+ ingress: Ingress,
+ options: { readonly wantsStream: boolean; readonly model?: string },
+): Prologue => {
+ const backgroundScheduler = backgroundSchedulerFromContext(c);
+ // The shape follows the endpoint. A pipelined turn is recorded as its whole run — every
+ // stage, both directions — so it opens that recording here instead of the edge one, and
+ // no turn is ever written twice.
+ const runDump = openRunDump(
+ apiKeyFromContext(c),
+ { method: c.req.method, path: new URL(c.req.raw.url).pathname, body: ingress.body },
+ backgroundScheduler,
+ );
+ const gateway = createGatewayCtxFromHono(c, {
+ wantsStream: options.wantsStream,
+ ...(options.model === undefined ? {} : { model: options.model }),
+ requestBody: takeRequestBody(ingress.body),
+ backgroundScheduler,
+ dump: runDump,
+ });
+
+ const live = new Map();
+
+ return {
+ gateway,
+ headers: ingress.headers,
+ services: {
+ gateway,
+ log: consoleLogSink,
+ background: work => { gateway.backgroundScheduler(work); },
+ rememberCandidates: candidates => {
+ for (const candidate of candidates) live.set(candidate.provider.upstreamId, candidate);
+ },
+ // Absent when this key has no retention configured, which is what keeps recording
+ // conditional: the runner does none of it rather than doing it and discarding.
+ ...(runDump === null ? {} : { dump: runDump.sink }),
+ resolveAttempt: (selector: AttemptSelector) => {
+ const candidate = live.get(selector.upstreamId);
+ if (candidate === undefined) {
+ throw new Error(`resolveAttempt: nothing live for ${selector.upstreamId}; the selector did not come from this run`);
+ }
+ return candidate;
+ },
+ },
+ };
+};
+
+/** What a family writes for the client. The status and the upstream's own headers are facts
+ * the pipeline already carries, so what is left here is the answer itself. */
+export type Rendered =
+ | {
+ readonly body: BodyInit;
+ /** Owned by whichever stage serialized the body, never by the upstream — a media type
+ * describing bytes the gateway wrote itself is the one thing an upstream cannot say.
+ * `null` where a family carries an upstream's own body and the upstream declared none:
+ * inventing one would describe bytes nobody described. */
+ readonly contentType: string | null;
+ }
+ | {
+ /** An answer that *is* a stream. The frames go out as they arrive, which is why nothing
+ * above waits for them and why what they billed is settled afterwards. */
+ readonly frames: AsyncIterable;
+ };
+
+/** Which of the two an answer turned out to be. A family's rendered fact carries whichever
+ * shape its run produced, and only the family knows the key it is under. */
+export const isFrames = (rendered: unknown): rendered is AsyncIterable =>
+ typeof rendered === 'object' && rendered !== null && Symbol.asyncIterator in rendered;
+
+/** How a streaming turn ended, once its frames ran out. Both halves arrive together because
+ * both are known at the same moment — the last chunk states the usage, and running out
+ * without a terminal event is what "it did not finish" means. */
+export interface StreamOutcome {
+ readonly billable: readonly BillableEntity[];
+ readonly failed: boolean;
+}
+
+/** What a streaming family will have been billed, and whether it got there. A stream's usage
+ * arrives with its last chunk, which is after the run has answered — so the run hands up a
+ * reading it has started and not finished, and settlement of it belongs here, after the answer
+ * is on its way. It is `Deferred` because that is what it is: the runner sees it in the record
+ * and waits for it at teardown, so a family that hands up a reading that never settles is
+ * reported rather than silently never billed. */
+export type DeferredUsage = (facts: Exit) => Deferred | null;
+
+/**
+ * Runs a family's pipeline and turns what it answered with into a response.
+ *
+ * The drain is scheduled rather than awaited. A streaming family's answer *is* the stream,
+ * so draining before returning would consume the frames the client is waiting for. Release
+ * is not cancel — an aborted connection cannot be reused and leaves the upstream's own
+ * billing unsettled — so it still happens, just after the answer is on its way.
+ *
+ * A run that threw is answered here rather than by the app's own handler, because the record
+ * is closed here. Letting the throw out would lose the whole record: nothing above this knows
+ * a dump is open, so the turn that most needs explaining would be the one that left nothing
+ * behind. What goes to the client is the same envelope the app's handler writes, from the same
+ * function — the stack included, which is what a gateway owes an operator for its own fault.
+ */
+export const serveThrough = async <
+ Entry extends object,
+ Exit extends Slice<'response.http.status' | 'response.http.headers'>,
+>(
+ c: Context,
+ prologue: Prologue,
+ pipeline: Pipeline,
+ entry: Entry,
+ render: (facts: Exit) => Rendered,
+ deferredUsage?: DeferredUsage,
+): Promise => {
+ try {
+ return await serveRun(c, prologue, pipeline, entry, render, deferredUsage);
+ } catch (error) {
+ // `run` drains what it opened before it rethrows, so there is nothing left outstanding to
+ // release here — only the record to close, and the reason to put on it.
+ prologue.gateway.dump?.failed(error);
+ return finalizeGatewayResponse(prologue.gateway, internalErrorResponse(asError(error), c));
+ }
+};
+
+/** Anything can be thrown; only an `Error` carries a stack. A throw that was not one is
+ * reported as what it was rather than being dressed up as something with a call site. */
+const asError = (thrown: unknown): Error =>
+ thrown instanceof Error ? thrown : new Error(String(thrown));
+
+const serveRun = async <
+ Entry extends object,
+ Exit extends Slice<'response.http.status' | 'response.http.headers'>,
+>(
+ c: Context,
+ prologue: Prologue,
+ pipeline: Pipeline,
+ entry: Entry,
+ render: (facts: Exit) => Rendered,
+ deferredUsage?: DeferredUsage,
+): Promise => {
+ const { facts, drain } = await run(pipeline, entry, prologue.services as never);
+ const answer = render(facts);
+
+ const pending = deferredUsage?.(facts) ?? null;
+ // Registered while the request is still live, so the platform binds the write to it — and
+ // resolved only when the stream ends, which is the one moment both what it billed and
+ // whether it finished are known. A turn that stopped short is not recorded as one that
+ // produced what it said it would.
+ if (pending !== null) {
+ prologue.services.background(pending.then(outcome => {
+ settleBillable({ ...prologue.services, log: consoleLogSink }, outcome.billable, outcome.failed);
+ }));
+ }
+
+ const status = facts['response.http.status'] as ContentfulStatusCode;
+ if ('frames' in answer) {
+ // Hono's streamSSE builds the response itself, so what the client is to see has to be
+ // staged on the context before it is called rather than passed to a constructor.
+ for (const [name, value] of facts['response.http.headers']) c.header(name, value);
+ c.status(status);
+ // The dump is closed the same way on both paths. Hono builds the streaming response
+ // itself, so what is finalized is what it returned rather than one constructed here.
+ return finalizeGatewayResponse(prologue.gateway, streamSSE(c, async stream => {
+ try {
+ await writeSSEFrames(stream, answer.frames, {
+ keepAlive: { frame: sseCommentFrame('keepalive') },
+ ...(prologue.gateway.downstreamAbortController === undefined
+ ? {}
+ : { downstreamAbortController: prologue.gateway.downstreamAbortController }),
+ });
+ } finally {
+ // Reading the frames to the client *is* releasing the body they came from, so the
+ // drain waits for that to finish. Draining alongside it would take frames out of the
+ // client's own stream — one connection has one reader. A client that stopped reading
+ // still gets here, which is what leaves nothing open behind it.
+ await drain();
+ }
+ }));
+ }
+
+ // Nothing is left to read either: what the client is sent was serialized from facts the run
+ // already held, so releasing can start at once.
+ prologue.services.background(drain());
+ const headers = new Headers(facts['response.http.headers'].map(([name, value]): [string, string] => [name, value]));
+ // An upstream that declared no media type is answered without one, rather than having one
+ // invented for bytes nobody described.
+ if (answer.contentType !== null) headers.set('content-type', answer.contentType);
+ else headers.delete('content-type');
+ return finalizeGatewayResponse(prologue.gateway, new Response(answer.body, { status, headers }));
+};
diff --git a/packages/gateway/src/data-plane/pipeline/services.ts b/packages/gateway/src/data-plane/pipeline/services.ts
new file mode 100644
index 0000000000..000ce02049
--- /dev/null
+++ b/packages/gateway/src/data-plane/pipeline/services.ts
@@ -0,0 +1,35 @@
+// What a run is given beside its facts. Services are wiring: they are fixed for the run at
+// the prologue and never change on a handoff, because what a stage hands to `next` is the
+// next segment's facts — if it also supplied capabilities, the same pipeline value would
+// run with different capabilities depending on who called it.
+//
+// Everything in facts is dumpable, and that is the test: a live handle dumps as nothing,
+// so a live handle is never a fact.
+
+import type { AttemptSelector } from './facts.ts';
+import type { GatewayCtx } from '../shared/gateway-ctx.ts';
+import type { Event, Logger } from '@floway-dev/pipeline';
+import type { ModelCandidate } from '@floway-dev/provider';
+
+export interface GatewayServices {
+ /** The global sink. Every stage's lines reach it, tagged with the stage's name. */
+ readonly log?: Logger;
+ /** Present only when this request is being dumped, which is what keeps recording
+ * conditional: with no sink resolved here, the runner does none of it. */
+ readonly dump?: (event: Event) => void;
+
+ /** The request-scoped context the settlement and telemetry stages read. It is a service
+ * and not a fact because it holds live handles — the scheduler, the abort signal. */
+ readonly gateway: GatewayCtx;
+ /** Turns a selector back into the thing that dials. The resolver is the service and the
+ * selector is the fact: a per-upstream transport is not a fact and is not pinned at the
+ * prologue either, so what is injected is the thing that resolves one and what travels
+ * is the identifier it resolves from. */
+ readonly resolveAttempt: (selector: AttemptSelector) => ModelCandidate;
+ /** The other half of the resolver: the stage that enumerates hands the live candidates
+ * here, so the ones that travel can be selectors. Per run, because a candidate list is. */
+ readonly rememberCandidates: (candidates: readonly ModelCandidate[]) => void;
+ /** Binds a promise to the request's lifetime: `waitUntil` on Workers, the event loop on
+ * Node. What the drain is handed to, so an answer is not held up by it. */
+ readonly background: (work: Promise) => void;
+}
diff --git a/packages/gateway/src/data-plane/pipeline/settlement.ts b/packages/gateway/src/data-plane/pipeline/settlement.ts
new file mode 100644
index 0000000000..5773694e2b
--- /dev/null
+++ b/packages/gateway/src/data-plane/pipeline/settlement.ts
@@ -0,0 +1,109 @@
+// Settlement, as a stage. It sits **above** the fork, so a run bills once however many
+// candidates it tried — repetition passes through the stage that observes usage, and not
+// through this one.
+//
+// It is unconditional: a run that measured rather than generated still writes, and its row
+// simply names no billed entity. Emptiness is observed, not declared — which is why there
+// is no "unknown" anywhere here. The situations are concrete and the list is open: the
+// upstream reported zero, we did not call an upstream, we failed before reaching the
+// upstream's usage, the upstream did not report.
+
+import type { BillableEntity, GatewayFacts } from './facts.ts';
+import type { GatewayServices } from './services.ts';
+import type { TokenUsage, UsageQuantities } from '../../repo/types.ts';
+import { recordPerformance } from '../shared/telemetry/performance.ts';
+import { recordUsage } from '../shared/telemetry/usage.ts';
+import { defineStage } from '@floway-dev/pipeline';
+import type { Logger } from '@floway-dev/pipeline';
+import type { BillingMetric } from '@floway-dev/protocols/common';
+
+type Slice = { [P in K]: GatewayFacts[P] };
+
+/** The dump's own two columns, read back out of what was billed.
+ *
+ * The dump sums every input category into one number and keeps output separate, and it
+ * distinguishes "not measured" from "measured zero" — so an entity that reported nothing
+ * contributes no keys and stays null rather than becoming a zero the upstream never said. */
+const dumpUsage = (quantities: UsageQuantities): TokenUsage => {
+ const count = (metric: BillingMetric): number | undefined =>
+ quantities[metric] === undefined ? undefined : Number(quantities[metric]);
+ return {
+ ...(count('input_tokens') === undefined ? {} : { input: count('input_tokens') }),
+ ...(count('input_cache_read_tokens') === undefined ? {} : { input_cache_read: count('input_cache_read_tokens') }),
+ ...(count('input_cache_write_tokens') === undefined ? {} : { input_cache_write: count('input_cache_write_tokens') }),
+ ...(count('input_image_tokens') === undefined ? {} : { input_image: count('input_image_tokens') }),
+ ...(count('output_tokens') === undefined ? {} : { output: count('output_tokens') }),
+ ...(count('output_image_tokens') === undefined ? {} : { output_image: count('output_image_tokens') }),
+ };
+};
+
+/**
+ * Writes what was billed, and the performance sample that goes with it.
+ *
+ * The usage write is scheduled rather than awaited. A transient repository failure must not
+ * turn an upstream's already-flowing response into a 502, and the run's own answer does not
+ * depend on the row — so it is handed to the background scheduler, which binds it to the
+ * request's lifetime on both deployment targets.
+ *
+ * A streaming family settles through this too, from the prologue after the drain: its
+ * numbers arrive with the stream's last chunk, which is after the run has answered.
+ */
+export const settleBillable = (
+ services: Pick & { readonly log: Logger },
+ billable: readonly BillableEntity[],
+ failed: boolean,
+ outputTokens = 0,
+ finishedAt: number = performance.now(),
+): void => {
+ for (const entity of billable) {
+ // The dump names the upstream that answered and what it metered, which is the same
+ // reading the row is written from rather than a second one taken separately.
+ if (!failed) services.gateway.dump?.success(entity.identity, dumpUsage(entity.quantities));
+ services.background(recordUsage(
+ services.gateway.apiKeyId,
+ entity.identity,
+ entity.quantities,
+ entity.pricingFacts ?? {},
+ ).catch((error: unknown) => {
+ services.log.error('failed to record usage', { error: String(error) });
+ }));
+ }
+ recordPerformance(services.gateway, services.gateway.attempt.telemetry, failed, outputTokens, finishedAt);
+};
+
+/**
+ * Prices what was observed and writes it, once per run.
+ *
+ * `stillReading` is how a streaming family says its numbers have not arrived yet: they come
+ * with the stream's last chunk, which is after this stage has handed up. Settling here as
+ * well would write the row twice — once for the entity that had reported nothing and once
+ * for what the stream turned out to say — so the run settles wherever the numbers are, and
+ * for a stream that is the epilogue.
+ */
+export const writeSettlement = (
+ failed: (handedUp: Record) => boolean,
+ stillReading?: (handedUp: Record) => boolean,
+) => defineStage<
+ Record,
+ Record,
+ Slice<'response.usage.billable'>,
+ Slice<'response.usage.billable'>,
+ GatewayServices
+>({
+ name: 'writeSettlement',
+ through: {
+ // Nothing on the way down: what is settled is what came back, and naming a request key
+ // here would put it in the entry contract of a family that never resolved a model.
+ request: { needs: [], consumes: [], provides: [] },
+ // It reads the authoritative reading and hands it on untouched: settlement is the last
+ // reader, not another writer, and a stage that changed usage re-provided it below.
+ response: { needs: ['response.usage.billable'], consumes: [], provides: [] },
+ },
+ execute: async (facts, next, use) => {
+ const back = await next(facts);
+ if (stillReading?.(back as Record) === true) return back;
+ // One performance sample per run, attributed to the attempt that answered.
+ settleBillable(use, back['response.usage.billable'], failed(back as Record));
+ return back;
+ },
+});
diff --git a/packages/gateway/src/data-plane/pipeline/stages.ts b/packages/gateway/src/data-plane/pipeline/stages.ts
new file mode 100644
index 0000000000..c370adedcc
--- /dev/null
+++ b/packages/gateway/src/data-plane/pipeline/stages.ts
@@ -0,0 +1,180 @@
+// Resolving a model to the upstreams that can serve it, and running the suffix once per
+// candidate until one answers. Both are ordinary stages: nothing in the framework knows
+// what a retry is, and array position is the whole of the statement that what follows
+// `failover` is the per-attempt segment.
+//
+// Neither names a family's own key. Branching is a capability of the framework and what
+// something branches on is a concept in the domain, so the family hands in the two
+// domain-shaped things — how to narrow a candidate, and how to read an attempt's outcome —
+// and these stages stay written against the shared space alone. That is also what lets
+// them compose into a pipeline over any family's larger space with no variance question to
+// lose: assembly reasons over declarations, which are strings.
+
+import type { AttemptSelector, GatewayFacts } from './facts.ts';
+import type { GatewayServices } from './services.ts';
+import { enumerateModelCandidates } from '../providers/resolution.ts';
+import { appendFailedUpstreams } from '../shared/failed-upstreams.ts';
+import { defineStage, move } from '@floway-dev/pipeline';
+import type { Facts } from '@floway-dev/pipeline';
+import type { ModelKind } from '@floway-dev/protocols/common';
+import { providerModelOf } from '@floway-dev/provider';
+import type { ModelCandidate } from '@floway-dev/provider';
+
+type Slice = { [P in K]: GatewayFacts[P] };
+
+/** Everything about a candidate that is data. The live half — the provider instance, the
+ * fetcher, the models cache — stays out of the record and is looked back up by the
+ * resolver service at the moment of the call. */
+const selectorFor = (candidate: ModelCandidate): AttemptSelector => ({
+ upstreamId: candidate.provider.upstreamId,
+ modelId: candidate.model.id,
+ flags: [...providerModelOf(candidate).enabledFlags],
+});
+
+/** What a family narrows its candidates by, and what it says when nothing is left. A
+ * candidate that resolves but cannot serve this request — no endpoint for the kind, or a
+ * payload the target protocol cannot express — is not a candidate, and saying why is what
+ * turns an empty list into a usable 400 rather than a bare 404. */
+export interface Narrowing {
+ readonly kind: ModelKind;
+ /** Keeps a candidate, or says in one phrase why it cannot serve this request. */
+ readonly reject: (candidate: ModelCandidate) => string | null;
+ /** What the client is told when the model resolved but nothing can serve the request.
+ * `reasons` holds what `reject` said, and is empty when no candidate of this kind was
+ * found at all — which is the difference between "this model is not an embeddings model"
+ * and "this reranker cannot do what this request asks". Families differ in how much of
+ * that they spell out, so the sentence is the family's rather than this stage's. */
+ readonly unsupported: (model: string, reasons: readonly string[]) => string;
+ /** What this family answers with when there is no candidate to try. The family decides
+ * which of its own keys carries the failure, which is why the refusal slice is a type
+ * parameter rather than a shared key: a reusable stage whose short-circuit provides
+ * family-specific keys cannot be written against the shared space alone. */
+ readonly refuse: (status: number, message: string) => Refusal;
+ /** The same keys as strings, so assembly can check that a short-circuit here covers what
+ * the stages above it need. */
+ readonly refuses: readonly (keyof Refusal)[];
+}
+
+/**
+ * Provides `serve.candidates`, or answers with the failure that says why there are none —
+ * both traits, because "no upstream serves this model" is an answer this stage already
+ * holds and nothing below it could produce one.
+ */
+export const resolveCandidates = (narrowing: Narrowing) => defineStage<
+ Slice<'serve.model'>, // what arrives
+ Slice<'serve.model' | 'serve.candidates'>, // what it hands down
+ Slice<'response.usage.billable' | 'response.http.headers'>, // what comes back
+ Slice<'response.usage.billable' | 'response.http.headers'>, // what it hands up, having descended
+ Slice<'response.usage.billable' | 'response.http.headers'> & Refusal, // and what it answers with instead
+ GatewayServices
+>({
+ name: 'resolveCandidates',
+ through: {
+ request: { needs: ['serve.model'], consumes: [], provides: ['serve.candidates'] },
+ response: { needs: ['response.usage.billable', 'response.http.headers'], consumes: [], provides: [] },
+ },
+ return: { provides: ['response.usage.billable', 'response.http.headers', ...narrowing.refuses] },
+ execute: async (facts, next, use) => {
+ const model = facts['serve.model'];
+ const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({
+ upstreamIds: use.gateway.upstreamIds,
+ model,
+ kind: narrowing.kind,
+ scheduler: use.gateway.backgroundScheduler,
+ runtimeLocation: use.gateway.runtimeLocation,
+ });
+
+ // An empty billed set is what "we did not call an upstream" looks like, and an empty
+ // header list is the same statement on the other key. The settlement stages still run
+ // and still write; the row simply names no billed entity.
+ const refuse = (status: number, message: string) =>
+ move({
+ ...facts,
+ 'response.usage.billable': [],
+ 'response.http.headers': [],
+ ...narrowing.refuse(status, message),
+ });
+
+ if (candidates.length === 0) {
+ const missing = sawModel
+ ? narrowing.unsupported(model, [])
+ : `Model ${model} is not available on any configured upstream.`;
+ return refuse(sawModel ? 400 : 404, appendFailedUpstreams(missing, failedUpstreams));
+ }
+
+ const refused = new Set();
+ const viable = candidates.filter(candidate => {
+ const why = narrowing.reject(candidate);
+ if (why !== null) refused.add(why);
+ return why === null;
+ });
+ // The live half stays with the resolver; only selectors travel.
+ use.rememberCandidates(viable);
+ if (viable.length === 0) {
+ use.log.debug('no viable candidate', { model, refused: [...refused] });
+ return refuse(400, appendFailedUpstreams(narrowing.unsupported(model, [...refused]), failedUpstreams));
+ }
+
+ use.log.debug('resolved candidates', { model, viable: viable.length, resolved: candidates.length });
+ return await next({ ...facts, 'serve.candidates': move(viable.map(selectorFor)) });
+ },
+});
+
+/**
+ * Runs what follows it once per candidate and returns the first that did not fail.
+ *
+ * The losing attempts' bodies are its own — that is what `consumes` declares on the way up
+ * — and the winner's rides onward, which is what `provides` declares. Failover never
+ * breaks a stream: once one has opened there is nothing to fail over to, and the family's
+ * edge is what settles that by sitting above this stage rather than below it.
+ */
+export interface Forking {
+ /** How this family reads an attempt's outcome. Branching is the framework's; what
+ * something branches on is the domain's. */
+ readonly failed: (handedUp: Facts) => boolean;
+ /** The keys at which this family's attempts hand up something the run owns — an upstream
+ * body still open, most often. A family whose ending reads its answer to the end owns
+ * nothing and names nothing here, and one that streams names the key it streams at.
+ *
+ * It cannot be a fixed key. Declaring `provides` for a key a family never produces makes
+ * the runner throw on the first real request, and declaring `consumes` for one it does
+ * produce and hands up makes it throw the other way. Which keys carry a resource is a
+ * statement only the family can make. */
+ readonly owns: readonly string[];
+}
+
+export const failover = ({ failed, owns }: Forking) => defineStage<
+ Slice<'serve.candidates'>,
+ Slice<'serve.candidates' | 'route.attempt'>,
+ Slice<'response.usage.billable'>,
+ Slice<'response.usage.billable'>,
+ GatewayServices
+>({
+ name: 'failover',
+ through: {
+ request: { needs: ['serve.candidates'], consumes: [], provides: ['route.attempt'] },
+ response: {
+ needs: ['response.usage.billable'],
+ // Owned on the way up and handed onward: every attempt's is this stage's to release,
+ // and the one it adopts rides up with ownership going with it.
+ consumes: owns as never,
+ provides: owns as never,
+ },
+ },
+ execute: async (facts, next, use) => {
+ let last: Slice<'response.usage.billable'> | undefined;
+ for (const candidate of facts['serve.candidates']) {
+ // Per-attempt telemetry state, cleared before control leaves, so a mid-attempt throw
+ // still attributes its performance row to the candidate that was being tried.
+ use.gateway.attempt.upstreamCallStartedAt = null;
+ use.gateway.attempt.firstOutputTokenAt = null;
+ last = await next({ ...facts, 'route.attempt': move(candidate) });
+ if (!failed(last as Facts)) return last;
+ use.log.info('candidate failed, trying the next', { upstream: candidate.upstreamId });
+ }
+ if (last === undefined) throw new Error('failover: assembly handed it an empty candidate list');
+ // Every candidate failed, and the last failure is the base — so the client sees real
+ // upstream telemetry rather than a synthesized gateway envelope.
+ return last;
+ },
+});
diff --git a/packages/gateway/src/data-plane/pipeline/upstream-body.ts b/packages/gateway/src/data-plane/pipeline/upstream-body.ts
new file mode 100644
index 0000000000..84f5484345
--- /dev/null
+++ b/packages/gateway/src/data-plane/pipeline/upstream-body.ts
@@ -0,0 +1,52 @@
+// Reading what an upstream answered with, for the families whose protocol is JSON.
+//
+// Every protocol the gateway carries is one it fully understands, so a body it cannot read
+// is not handed on unread: the family synthesizes an error from what it did see. That is
+// also why the text is kept alongside the parse — a dump reader is owed what actually came
+// back, and a message that quotes the body is the only thing a client can act on when the
+// upstream answered with something the protocol does not admit.
+
+import type { Failure } from './facts.ts';
+
+export interface UpstreamBody {
+ readonly text: string;
+ /** Absent when the body was not JSON at all, which is itself the answer to a protocol that
+ * requires JSON. */
+ readonly json?: unknown;
+}
+
+export const readUpstreamBody = async (response: Response): Promise => {
+ const text = await response.text();
+ try {
+ return { text, json: JSON.parse(text) as unknown };
+ } catch {
+ return { text };
+ }
+};
+
+/**
+ * What an upstream that answered in something other than this protocol is worth to a client.
+ *
+ * A gateway that cannot read the answer has not served the request, whatever status came with
+ * it, so this is 502 rather than the upstream's own — and it is a value, because the fork
+ * above has another candidate to try and an answer nobody can read is exactly the outcome it
+ * exists to move past.
+ */
+export const unreadableBody = (response: Response, body: UpstreamBody, protocolName: string): Failure => ({
+ status: 502,
+ message: `The upstream answered ${response.status} with a body ${protocolName} cannot read: ${body.text.slice(0, 200)}`,
+});
+
+/**
+ * What the platform raised while dialling, as a value.
+ *
+ * Nothing in the domain throws. A connection that was refused, timed out or was reset is an
+ * outcome failover has to be able to see — a run that ends there has more candidates to try —
+ * so the ending catches whatever the platform raised and hands it up like any other failure.
+ * The status says what happened rather than what the error was: the gateway reached no
+ * upstream at all, which is what 502 states.
+ */
+export const dialFailure = (error: unknown): Failure => ({
+ status: 502,
+ message: error instanceof Error ? error.message : String(error),
+});
diff --git a/packages/gateway/src/data-plane/rerank/attempt.ts b/packages/gateway/src/data-plane/rerank/attempt.ts
deleted file mode 100644
index 1842a852dc..0000000000
--- a/packages/gateway/src/data-plane/rerank/attempt.ts
+++ /dev/null
@@ -1,42 +0,0 @@
-import type { Context } from 'hono';
-
-import type { GatewayCtx } from '../shared/gateway-ctx.ts';
-import { inboundHeaders } from '../shared/inbound-headers.ts';
-import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts';
-import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts';
-import type { RerankTarget } from '@floway-dev/protocols/common';
-import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank';
-import { providerModelOf } from '@floway-dev/provider';
-import type { ModelCandidate, PerformanceTelemetryContext, ProviderRerankCallResult, TelemetryModelIdentity } from '@floway-dev/provider';
-
-export interface RerankAttemptResult {
- readonly type: 'plain';
- readonly status: number;
- readonly response: Response;
- readonly target: RerankTarget;
- readonly performance: PerformanceTelemetryContext;
- readonly identity: TelemetryModelIdentity;
-}
-
-export const rerankAttempt = async (
- c: Context,
- ctx: GatewayCtx,
- candidate: ModelCandidate,
- request: CanonicalRerankRequest,
-): Promise => {
- const model = providerModelOf(candidate);
- const result: ProviderRerankCallResult = await candidate.provider.instance.callRerank(
- model,
- request,
- ctx.abortSignal,
- buildUpstreamCallOptions(candidate, ctx, inboundHeaders(c)),
- );
- return {
- type: 'plain',
- status: result.response.status,
- response: result.response,
- target: result.target,
- performance: upstreamPerformanceContext(ctx, candidate, 'rerank'),
- identity: telemetryModelIdentity(candidate, result.modelKey),
- };
-};
diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts
new file mode 100644
index 0000000000..17e62b8cf3
--- /dev/null
+++ b/packages/gateway/src/data-plane/rerank/pipeline.ts
@@ -0,0 +1,303 @@
+// Rerank as a pipeline. The family with a canonical contract already — parse, render,
+// serialize and a usage reading — so it is the one that proves the shared stages on real
+// business before the families that need their contracts written first.
+//
+// The shape every family repeats:
+//
+// emitRerank the edge: serializes the answer into the client's protocol
+// writeSettlement above the fork, so a run bills once however many attempts it made
+// resolveCandidates narrows to the upstreams that can serve this request
+// failover runs what follows once per candidate
+// callRerankUpstream the ending: dials, and provides what came back
+
+import type { UsageQuantities } from '../../repo/types.ts';
+import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts';
+import { isFailure } from '../pipeline/facts.ts';
+import type { GatewayServices } from '../pipeline/services.ts';
+import { writeSettlement } from '../pipeline/settlement.ts';
+import { failover, resolveCandidates } from '../pipeline/stages.ts';
+import { dialFailure, readUpstreamBody, unreadableBody } from '../pipeline/upstream-body.ts';
+import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts';
+import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts';
+import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts';
+import type { Pipeline } from '@floway-dev/pipeline';
+import { defineStage, move, compose } from '@floway-dev/pipeline';
+import { parseDecimalString, renderErrorEnvelope, upstreamErrorMessage, type RerankProtocol, type RerankSourceProtocol } from '@floway-dev/protocols/common';
+import {
+ parseRerankResponse,
+ parseRerankUsage,
+ renderRerankResponse,
+ rerankRequestIncompatibility,
+ type CanonicalRerankRequest,
+ type CanonicalRerankResponse,
+} from '@floway-dev/protocols/rerank';
+import { providerModelOf } from '@floway-dev/provider';
+
+/** Rerank's own keys. They extend the shared space and never merge into it, so a stage
+ * written against the gateway alone cannot name one. */
+export interface RerankFacts extends GatewayFacts {
+ /** Which of the four rerank protocols the client spoke. It belongs to the ingress and
+ * stays put: the answer is rendered back into it whatever the upstream spoke. */
+ 'ingress.rerank.sourceProtocol': RerankSourceProtocol;
+ 'request.rerank.canonical': CanonicalRerankRequest;
+ 'response.rerank.canonical': CanonicalRerankResponse | Failure;
+ /** Which protocol the attempt spoke. A response fact, because an upstream picks the target
+ * it answers in and the edge needs it to render back into the client's — a run that never
+ * reached one carries the target its request was serialized for. */
+ 'response.rerank.targetProtocol': RerankProtocol;
+ /** What the client is actually sent, in its own protocol. The edge provides it, so a
+ * dump shows the bytes the client received rather than the gateway's canonical form. */
+ 'response.rerank.rendered': Record;
+}
+
+type R = { [P in K]: RerankFacts[P] };
+
+/**
+ * The outermost edge. Renders the canonical answer into the protocol the client spoke —
+ * which is why `ingress.rerank.sourceProtocol` is an ingress fact and not a request one:
+ * it survives the switch to whatever protocol the upstream turned out to speak.
+ */
+const emitRerank = defineStage<
+ R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>,
+ R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>,
+ R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.headers'>,
+ R<'response.rerank.rendered' | 'response.http.status' | 'response.http.headers'>
+>({
+ name: 'emitRerank',
+ through: {
+ request: {
+ needs: ['ingress.rerank.sourceProtocol', 'request.rerank.canonical'],
+ consumes: [],
+ provides: [],
+ },
+ response: {
+ needs: ['response.rerank.canonical', 'response.http.headers'],
+ consumes: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.headers'],
+ provides: ['response.rerank.rendered', 'response.http.status', 'response.http.headers'],
+ },
+ },
+ execute: async (facts, next) => {
+ const back = await next(facts);
+ const { 'response.rerank.canonical': answer, 'response.rerank.targetProtocol': target, 'response.http.headers': headers, ...rest } = back;
+ // Vendor traces and quota state stay visible; what an intermediary must strip, and what
+ // would misdescribe a body this gateway serialized itself, does not. A filter that removed
+ // nothing hands the same array on, so the record shows no change where none happened.
+ const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name));
+ const forClient = forwardable.length === headers.length ? headers : move(forwardable);
+ if (isFailure(answer)) {
+ return {
+ ...rest,
+ 'response.http.headers': forClient,
+ 'response.rerank.rendered': move(renderErrorEnvelope(answer.message, answer.body)),
+ // The upstream's own status, or the gateway's own when it refused before dialling.
+ // A client is not owed the upstream's exact bytes; it is owed the truth about what
+ // happened, and a 429 arriving as a 200 is not that.
+ 'response.http.status': answer.status,
+ };
+ }
+ // Translating can fail on an answer that parsed: a result may index a document the
+ // request never sent. The upstream answered and the gateway cannot put that answer in the
+ // protocol the client speaks, which is the gateway failing to serve rather than anything
+ // the client can fix — so it is 502, and a value like every other failure here.
+ let rendered: Record;
+ try {
+ rendered = renderRerankResponse(
+ back['ingress.rerank.sourceProtocol'],
+ target,
+ answer,
+ back['request.rerank.canonical'],
+ );
+ } catch (error) {
+ return {
+ ...rest,
+ 'response.http.headers': forClient,
+ 'response.rerank.rendered': move(renderErrorEnvelope(error instanceof Error ? error.message : String(error))),
+ 'response.http.status': 502,
+ };
+ }
+ return {
+ ...rest,
+ 'response.http.headers': forClient,
+ 'response.http.status': 200,
+ 'response.rerank.rendered': move(rendered),
+ };
+ },
+});
+
+/**
+ * The ending. It dials, reads the upstream's body, and provides the canonical answer and
+ * what the call is billable for. A failure is a value: a 429 here is what an earlier stage
+ * fails over, and even a 400 can be, because the next candidate's path and flags may differ.
+ */
+const callRerankUpstream = defineStage<
+ R<'request.rerank.canonical' | 'route.attempt' | 'ingress.http.headers' | 'ingress.rerank.sourceProtocol'>,
+ R<'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.headers' | 'response.usage.billable'>,
+ GatewayServices
+>({
+ name: 'callRerankUpstream',
+ return: {
+ provides: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.headers', 'response.usage.billable'],
+ },
+ execute: async (facts, use) => {
+ const candidate = use.resolveAttempt(facts['route.attempt']);
+ const request = facts['request.rerank.canonical'];
+ const model = providerModelOf(candidate);
+ // Configuration refuses a rerank model without a target, so this holds for every candidate
+ // the narrowing kept; saying so here is what lets a dial that never answered still name
+ // the protocol it spoke.
+ const configuredTarget = model.rerankTarget;
+ if (configuredTarget === undefined) {
+ throw new Error(`${candidate.provider.upstreamId} serves rerank for ${candidate.model.id} without a target protocol`);
+ }
+ // Attribution is set before the dial, so an attempt that never completes still names the
+ // candidate it was made against rather than the one tried before it.
+ use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'rerank');
+
+ let result;
+ try {
+ result = await candidate.provider.instance.callRerank(
+ model,
+ request,
+ use.gateway.abortSignal,
+ // The client's own headers reach the upstream from the record, not from a live
+ // request object: what a provider is allowed to forward is filtered per provider,
+ // and the dump shows what was there to filter.
+ buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))),
+ );
+ } catch (error) {
+ use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) });
+ // A dial that never completed reached no upstream, so nothing was billed and there are
+ // no headers to carry. What it leaves behind is the performance row settlement writes.
+ return move({
+ ...facts,
+ 'response.rerank.canonical': dialFailure(error),
+ 'response.rerank.targetProtocol': configuredTarget.protocol,
+ 'response.http.headers': [],
+ 'response.usage.billable': [],
+ });
+ }
+
+ const identity = telemetryModelIdentity(candidate, result.modelKey);
+ // What came back, unfiltered: the edge is where a client's view of it is decided.
+ const headers = [...result.response.headers];
+ const body = await readUpstreamBody(result.response);
+
+ if (!result.response.ok) {
+ use.log.warn('upstream refused', { status: result.response.status });
+ return move({
+ ...facts,
+ 'response.rerank.canonical': {
+ status: result.response.status,
+ message: upstreamErrorMessage(body.json) ?? body.text,
+ ...('json' in body ? { body: body.json } : {}),
+ },
+ 'response.rerank.targetProtocol': result.target.protocol,
+ 'response.http.headers': headers,
+ // The upstream was called and reported nothing, which is a different situation
+ // from reporting zero — so the entity is present with no quantities.
+ 'response.usage.billable': [{ identity, quantities: {} }],
+ });
+ }
+
+ if (!('json' in body)) {
+ return move({
+ ...facts,
+ 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'),
+ 'response.rerank.targetProtocol': result.target.protocol,
+ 'response.http.headers': headers,
+ 'response.usage.billable': [{ identity, quantities: {} }],
+ });
+ }
+
+ // A usage block the reader cannot make sense of is a report we cannot parse, which from
+ // here is no report. It is read before the results, so an answer this gateway could not
+ // model still bills for what the upstream did meter — the two readings are independent
+ // and one of them failing is not a reason to discard the other.
+ let usage: Pick;
+ try {
+ usage = parseRerankUsage(result.target.protocol, body.json);
+ } catch (error) {
+ use.log.warn('upstream reported usage the rerank protocol cannot read', { error: String(error) });
+ usage = {};
+ }
+ const metered: readonly BillableEntity[] = [{
+ identity,
+ quantities: billed(usage),
+ // A rerank rate can depend on how large the input was and not only on how much of it
+ // there was, so the token total is a pricing input as well as a quantity.
+ ...(usage.totalTokens === undefined ? {} : { pricingFacts: { inputTokens: usage.totalTokens } }),
+ }];
+
+ // An answer the client's own protocol produced is what that client already reads, so the
+ // edge renders it back out unchanged and nothing here has to model it. Reading the results
+ // is what a *translation* needs, and only a cross-protocol run does one — so a result item
+ // this gateway cannot model is a failure there and a field it simply carries here.
+ const translating = facts['ingress.rerank.sourceProtocol'] !== result.target.protocol;
+ let canonical: CanonicalRerankResponse;
+ try {
+ canonical = parseRerankResponse(result.target.protocol, body.json);
+ } catch (error) {
+ if (translating) {
+ use.log.warn('upstream answered with results the rerank protocol cannot read', { error: String(error) });
+ return move({
+ ...facts,
+ 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'),
+ 'response.rerank.targetProtocol': result.target.protocol,
+ 'response.http.headers': headers,
+ 'response.usage.billable': metered,
+ });
+ }
+ use.log.debug('same-protocol answer carries results this gateway does not model', { error: String(error) });
+ canonical = { raw: body.json as Record, results: [] };
+ }
+
+ return move({
+ ...facts,
+ 'response.rerank.canonical': canonical,
+ 'response.rerank.targetProtocol': result.target.protocol,
+ 'response.http.headers': headers,
+ 'response.usage.billable': metered,
+ });
+ },
+});
+
+const billed = (usage: Pick | undefined): UsageQuantities => {
+ const quantities: UsageQuantities = {};
+ if (usage?.searchUnits !== undefined) quantities.rerank_searches = parseDecimalString(String(usage.searchUnits));
+ if (usage?.totalTokens !== undefined) quantities.input_tokens = parseDecimalString(String(usage.totalTokens));
+ return quantities;
+};
+
+/** A candidate that cannot serve *this* request is not a candidate. Saying why is what
+ * turns an empty list into a 400 a client can act on. */
+const narrowing = (request: CanonicalRerankRequest) => ({
+ kind: 'rerank' as const,
+ reject: (candidate: Parameters[0]) => {
+ const model = providerModelOf(candidate);
+ if (candidate.model.endpoints.rerank === undefined || model.rerankTarget === undefined) {
+ return 'the upstream does not expose a rerank endpoint';
+ }
+ return rerankRequestIncompatibility(model.rerankTarget.protocol, request);
+ },
+ unsupported: (model: string, reasons: readonly string[]) => reasons.length === 0
+ ? `Model ${model} does not support rerank.`
+ : `Model ${model} does not support this rerank request: ${reasons.join('; ')}.`,
+ refuse: (status: number, message: string) => ({ 'response.rerank.canonical': { status, message } }),
+ refuses: ['response.rerank.canonical'] as const,
+});
+
+export const rerankServePipeline = (request: CanonicalRerankRequest): Pipeline<
+ R<'ingress.http.headers' | 'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'serve.model'>,
+ R<'response.rerank.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>
+> => compose('rerankServe', [
+ emitRerank,
+ writeSettlement(handedUp => isFailure((handedUp as { 'response.rerank.canonical'?: unknown })['response.rerank.canonical'])),
+ resolveCandidates(narrowing(request)),
+ failover({
+ failed: handedUp => isFailure((handedUp as { 'response.rerank.canonical'?: unknown })['response.rerank.canonical']),
+ owns: [],
+ }),
+ callRerankUpstream,
+]);
+
+export type { BillableEntity };
diff --git a/packages/gateway/src/data-plane/rerank/serve.ts b/packages/gateway/src/data-plane/rerank/serve.ts
index 7d64ba63d2..8c5d440b41 100644
--- a/packages/gateway/src/data-plane/rerank/serve.ts
+++ b/packages/gateway/src/data-plane/rerank/serve.ts
@@ -1,183 +1,67 @@
-import type { Context } from 'hono';
-import type { ContentfulStatusCode } from 'hono/utils/http-status';
+// POST /v1/rerank, /v2/rerank, /jina/v1/rerank and /voyage/v1/rerank, served through the
+// pipeline.
+//
+// One family behind four routes. Which protocol the client spoke is not in the body — the
+// route it arrived on is the whole of that statement — so the mount passes it in and the
+// handler carries it into the record as an ingress fact. Everything after the parse is
+// stages.
-import { rerankAttempt, type RerankAttemptResult } from './attempt.ts';
-import type { UsageQuantities } from '../../repo/types.ts';
-import { backgroundSchedulerFromContext } from '../../runtime/background.ts';
-import { enumerateModelCandidates } from '../providers/resolution.ts';
-import { appendFailedUpstreams } from '../shared/failed-upstreams.ts';
-import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts';
-import { iterateCandidates } from '../shared/iterate-candidates.ts';
-import { readRequestBody, takeRequestBody } from '../shared/request-body.ts';
-import { recordFailedRequest, recordPerformance, type PerformanceTelemetryContext } from '../shared/telemetry/performance.ts';
-import { recordUsage } from '../shared/telemetry/usage.ts';
-import { forwardUpstreamResponse } from '../shared/upstream-response.ts';
-import { parseDecimalString, type RerankSourceProtocol } from '@floway-dev/protocols/common';
-import { parseRerankRequest, parseRerankResponse, parseRerankUsage, renderRerankResponse, rerankRequestIncompatibility, type CanonicalRerankResponse, type ParsedRerankRequest } from '@floway-dev/protocols/rerank';
-import { httpResponseToResponse, ProviderModelsUnavailableError, providerModelOf, toInternalDebugError } from '@floway-dev/provider';
-import type { TelemetryModelIdentity } from '@floway-dev/provider';
+import type { Context } from 'hono';
-const apiError = (c: Context, message: string, status: ContentfulStatusCode): Response =>
- c.json({ error: { message, type: 'api_error' } }, status);
+import { rerankServePipeline } from './pipeline.ts';
+import { openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts';
+import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts';
+import { move } from '@floway-dev/pipeline';
+import type { RerankSourceProtocol } from '@floway-dev/protocols/common';
+import { parseRerankRequest, type ParsedRerankRequest } from '@floway-dev/protocols/rerank';
-const parseJson = (bytes: Uint8Array): unknown => {
+// The contract reports a malformed request by throwing; what the client is owed is a 400
+// carrying the reason. `JSON.parse`'s own wording names a byte offset in a body the client
+// already has, so the reason for that one is written here instead.
+const readRequest = (
+ sourceProtocol: RerankSourceProtocol,
+ bytes: Uint8Array,
+): { type: 'ok'; parsed: ParsedRerankRequest } | { type: 'invalid'; message: string } => {
+ let body: unknown;
try {
- return JSON.parse(new TextDecoder().decode(bytes)) as unknown;
+ body = JSON.parse(new TextDecoder().decode(bytes)) as unknown;
} catch {
- throw new Error('Rerank request body must be valid JSON');
+ return { type: 'invalid', message: 'Rerank request body must be valid JSON' };
}
-};
-
-const settleRerank = (
- ctx: GatewayCtx,
- performanceContext: PerformanceTelemetryContext,
- identity: TelemetryModelIdentity,
- usage: Pick | undefined,
- failed: boolean,
-): void => {
- const quantities: UsageQuantities = {};
- if (usage?.searchUnits !== undefined) quantities.rerank_searches = parseDecimalString(String(usage.searchUnits));
- if (usage?.totalTokens !== undefined) quantities.input_tokens = parseDecimalString(String(usage.totalTokens));
- const pricingFacts = usage?.totalTokens === undefined ? {} : { inputTokens: usage.totalTokens };
- ctx.backgroundScheduler(recordUsage(ctx.apiKeyId, identity, quantities, pricingFacts).catch(error => {
- console.error('Failed to record rerank usage:', error);
- }));
- recordPerformance(ctx, performanceContext, failed, 0, performance.now());
-};
-
-const unsupportedMessage = (model: string): string => `Model ${model} does not support rerank.`;
-
-export const rerank = (sourceProtocol: RerankSourceProtocol) => async (c: Context): Promise => {
- const requestBody = await readRequestBody(c);
- let parsedRequest: ParsedRerankRequest;
try {
- parsedRequest = parseRerankRequest(sourceProtocol, parseJson(requestBody.bytes));
+ return { type: 'ok', parsed: parseRerankRequest(sourceProtocol, body) };
} catch (error) {
- const ctx = createGatewayCtxFromHono(c, {
- wantsStream: false,
- requestBody: takeRequestBody(requestBody),
- backgroundScheduler: backgroundSchedulerFromContext(c),
- });
- ctx.dump?.error('gateway');
- return finalizeGatewayResponse(ctx, apiError(c, error instanceof Error ? error.message : String(error), 400));
+ return { type: 'invalid', message: error instanceof Error ? error.message : String(error) };
}
+};
- const { model, request } = parsedRequest;
- const ctx = createGatewayCtxFromHono(c, {
- wantsStream: false,
- model,
- requestBody: takeRequestBody(requestBody),
- backgroundScheduler: backgroundSchedulerFromContext(c),
- });
-
- let terminal: RerankAttemptResult | undefined;
- let measuredUsage: Pick | undefined;
- let usageSettled = false;
- try {
- const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({
- upstreamIds: ctx.upstreamIds,
- model,
- kind: 'rerank',
- scheduler: ctx.backgroundScheduler,
- runtimeLocation: ctx.runtimeLocation,
- });
- if (candidates.length === 0) {
- ctx.dump?.error('gateway');
- const message = sawModel
- ? unsupportedMessage(model)
- : `Model ${model} is not available on any configured upstream.`;
- return finalizeGatewayResponse(ctx, apiError(c, appendFailedUpstreams(message, failedUpstreams), sawModel ? 400 : 404));
- }
-
- const routable = candidates.flatMap(candidate => {
- const providerModel = providerModelOf(candidate);
- return candidate.model.endpoints.rerank === undefined || providerModel.rerankTarget === undefined
- ? []
- : [{ candidate, target: providerModel.rerankTarget }];
- });
- if (routable.length === 0) {
- ctx.dump?.error('gateway');
- return finalizeGatewayResponse(ctx, apiError(c, appendFailedUpstreams(unsupportedMessage(model), failedUpstreams), 400));
- }
- const viable = routable.filter(({ target }) => rerankRequestIncompatibility(target.protocol, request) === null);
- if (viable.length === 0) {
- const reasons = [...new Set(routable.flatMap(({ target }) => {
- const reason = rerankRequestIncompatibility(target.protocol, request);
- return reason === null ? [] : [reason];
- }))];
- ctx.dump?.error('gateway');
- return finalizeGatewayResponse(ctx, apiError(c, `Model ${model} does not support this rerank request: ${reasons.join('; ')}.`, 400));
- }
-
- terminal = await iterateCandidates(
- viable.map(({ candidate }) => candidate),
- 'rerank',
- ctx,
- 'rerank',
- candidate => rerankAttempt(c, ctx, candidate, request),
+export const rerank = (sourceProtocol: RerankSourceProtocol) => async (c: Context): Promise => {
+ const ingress = await readIngress(c);
+ const result = readRequest(sourceProtocol, ingress.body.bytes);
+ if (result.type === 'invalid') {
+ // A request the gateway could not read never reaches a pipeline: there is no model to
+ // resolve and no attempt to make, so there is nothing for a run to record.
+ const refused = openPrologue(c, ingress, { wantsStream: false });
+ refused.gateway.dump?.error('gateway');
+ return finalizeGatewayResponse(
+ refused.gateway,
+ Response.json({ error: { message: result.message, type: 'api_error' } }, { status: 400 }),
);
+ }
- if (!terminal.response.ok) {
- ctx.dump?.error('upstream', terminal.identity.upstream);
- settleRerank(ctx, terminal.performance, terminal.identity, undefined, true);
- usageSettled = true;
- return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response));
- }
+ const { model, request } = result.parsed;
+ const prologue = openPrologue(c, ingress, { wantsStream: false, model });
- const sameProtocol = sourceProtocol === terminal.target.protocol;
- let upstreamBody: unknown;
- try {
- upstreamBody = await terminal.response.clone().json() as unknown;
- } catch (error) {
- if (!sameProtocol) throw error;
- console.warn(
- `rerank: failed to parse same-protocol 2xx upstream body for ${sourceProtocol}; usage row will be request-only`,
- error instanceof Error ? error.message : String(error),
- );
- ctx.dump?.success(terminal.identity, null);
- settleRerank(ctx, terminal.performance, terminal.identity, undefined, false);
- usageSettled = true;
- return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response));
- }
- try {
- measuredUsage = parseRerankUsage(terminal.target.protocol, upstreamBody);
- } catch (error) {
- if (!sameProtocol) throw error;
- console.warn(
- `rerank: failed to parse same-protocol usage for ${sourceProtocol}; usage row will be request-only`,
- error instanceof Error ? error.message : String(error),
- );
- ctx.dump?.success(terminal.identity, null);
- settleRerank(ctx, terminal.performance, terminal.identity, undefined, false);
- usageSettled = true;
- return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response));
- }
- if (sameProtocol) {
- ctx.dump?.success(terminal.identity, null);
- settleRerank(ctx, terminal.performance, terminal.identity, measuredUsage, false);
- usageSettled = true;
- return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response));
- }
- const canonical = parseRerankResponse(terminal.target.protocol, upstreamBody);
- const rendered = renderRerankResponse(sourceProtocol, terminal.target.protocol, canonical, request);
- ctx.dump?.success(terminal.identity, null);
- settleRerank(ctx, terminal.performance, terminal.identity, measuredUsage, false);
- usageSettled = true;
- return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response, { body: JSON.stringify(rendered) }));
- } catch (error) {
- if (terminal !== undefined && !usageSettled) {
- settleRerank(ctx, terminal.performance, terminal.identity, measuredUsage, true);
- } else if (terminal === undefined) {
- recordFailedRequest(ctx, ctx.attempt.telemetry);
- }
- if (error instanceof ProviderModelsUnavailableError) {
- const forwarded = httpResponseToResponse(error.httpResponse);
- if (forwarded) {
- ctx.dump?.error('upstream');
- return finalizeGatewayResponse(ctx, forwarded);
- }
- }
- ctx.dump?.failed(error);
- return finalizeGatewayResponse(ctx, c.json({ error: toInternalDebugError(error) }, 502));
- }
+ return await serveThrough(
+ c,
+ prologue,
+ rerankServePipeline(request),
+ move({
+ 'ingress.http.headers': prologue.headers,
+ 'ingress.rerank.sourceProtocol': sourceProtocol,
+ 'request.rerank.canonical': request,
+ 'serve.model': model,
+ }) as never,
+ facts => ({ body: JSON.stringify(facts['response.rerank.rendered']), contentType: 'application/json' }),
+ );
};
diff --git a/packages/gateway/src/data-plane/shared/api-names.ts b/packages/gateway/src/data-plane/shared/api-names.ts
deleted file mode 100644
index 174855734f..0000000000
--- a/packages/gateway/src/data-plane/shared/api-names.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-// Routing-side API name primitives shared across the data plane.
-//
-// `PassthroughServeApiName` is the set of API names served by the
-// passthroughServe helper (see ./passthrough-serve.ts) rather than the
-// chat source/target executor. It groups by transport shape (the body /
-// frames are forwarded verbatim, possibly with a usage-extraction step),
-// not by whether the endpoint is chat-shaped — `/completions` is a
-// non-chat endpoint that lives here because there is nothing to translate
-// to or from. The value is the public URL fragment, so it can be used
-// directly in error messages and route comparisons without a lookup table.
-export type PassthroughServeApiName = '/completions' | '/embeddings' | '/images/generations' | '/images/edits' | '/audio/transcriptions';
diff --git a/packages/gateway/src/data-plane/shared/gateway-ctx.ts b/packages/gateway/src/data-plane/shared/gateway-ctx.ts
index 6dd3e50a1c..fb8af7900f 100644
--- a/packages/gateway/src/data-plane/shared/gateway-ctx.ts
+++ b/packages/gateway/src/data-plane/shared/gateway-ctx.ts
@@ -1,5 +1,6 @@
import type { RequestBody } from './request-body.ts';
-import { type DumpAccumulator, openDumpAccumulator } from '../../dump/accumulator.ts';
+import { openDumpAccumulator } from '../../dump/accumulator.ts';
+import type { TurnDump } from '../../dump/turn-dump.ts';
import { apiKeyFromContext, type AuthedContext, effectiveUpstreamIdsFromContext } from '../../middleware/auth.ts';
import { getRuntimeLocation } from '../../runtime/runtime-info.ts';
import type { BackgroundScheduler } from '@floway-dev/platform';
@@ -40,11 +41,15 @@ export interface GatewayCtx {
// Null when the api key has no dump retention configured, in which case
// `finalizeGatewayResponse` short-circuits the dump tee and returns the
// response untouched.
- readonly dump: DumpAccumulator | null;
+ readonly dump: TurnDump | null;
}
export interface CreateGatewayCtxOptions {
wantsStream: boolean;
+ // What this turn is recorded as. The shape follows the endpoint: a pipelined one hands in
+ // its run recording, and everything else lets the factory open the edge accumulator.
+ // Absent and null differ — absent means "open the usual one", null means "record nothing".
+ dump?: TurnDump | null;
// WebSocket-style call sites own the AbortController (so the upgrade
// handler can cancel mid-stream); HTTP call sites let the factory mint one
// when wantsStream is true.
@@ -80,7 +85,7 @@ export const createGatewayCtxFromHono = (c: AuthedContext, opts: CreateGatewayCt
const controller = opts.downstreamAbortController ?? (opts.wantsStream ? new AbortController() : undefined);
const apiKey = apiKeyFromContext(c);
const upstreamIds = effectiveUpstreamIdsFromContext(c);
- const dump = openDumpAccumulator(c, opts.method ?? c.req.method, apiKey, opts.requestBody, opts.backgroundScheduler);
+ const dump = 'dump' in opts ? opts.dump ?? null : openDumpAccumulator(c, opts.method ?? c.req.method, apiKey, opts.requestBody, opts.backgroundScheduler);
if (opts.model !== undefined) dump?.requestedModel(opts.model);
return {
apiKeyId: apiKey.id,
diff --git a/packages/gateway/src/data-plane/shared/passthrough-request.ts b/packages/gateway/src/data-plane/shared/json-model-request.ts
similarity index 100%
rename from packages/gateway/src/data-plane/shared/passthrough-request.ts
rename to packages/gateway/src/data-plane/shared/json-model-request.ts
diff --git a/packages/gateway/src/data-plane/shared/passthrough-attempt.ts b/packages/gateway/src/data-plane/shared/passthrough-attempt.ts
deleted file mode 100644
index db997affe0..0000000000
--- a/packages/gateway/src/data-plane/shared/passthrough-attempt.ts
+++ /dev/null
@@ -1,82 +0,0 @@
-// Per-candidate passthrough attempt: does the upstream HTTP call for one
-// resolved candidate and hands the serve loop back a `plain`-shaped result
-// the shared `iterateCandidates` iterator can drive.
-//
-// Chat protocols run each attempt through a translation + interceptor
-// stack that yields an `ExecuteResult` discriminated union. Passthrough
-// endpoints have no translation — the request body is forwarded to the
-// upstream's matching endpoint and the raw upstream Response is returned
-// verbatim. The `plain` discriminant is enlarged here to carry that
-// Response plus the per-call telemetry the serve site needs when it
-// forwards the winning attempt (2xx) or the last failure (exhausted).
-
-import type { GatewayCtx } from './gateway-ctx.ts';
-import { inboundHeaders } from './inbound-headers.ts';
-import { telemetryModelIdentity, upstreamPerformanceContext } from './telemetry/attribution.ts';
-import type { PerformanceTelemetryContext } from './telemetry/performance.ts';
-import { buildUpstreamCallOptions } from './upstream-call-options.ts';
-import type { AuthedContext } from '../../middleware/auth.ts';
-import { providerModelOf } from '@floway-dev/provider';
-import type { ModelCandidate, PerformanceOperation, Provider, ProviderCallResult, ProviderModel, TelemetryModelIdentity, UpstreamCallOptions } from '@floway-dev/provider';
-
-// Enlarged `plain` shape: `iterateCandidates` reads `type` + `status`;
-// the passthrough serve reads the rest to forward the response and
-// attribute dumps. `identity` carries the upstream id alongside the
-// model/pricing metadata the dump and usage-record paths already consume
-// together.
-export interface PassthroughAttemptResult {
- readonly type: 'plain';
- readonly status: number;
- readonly response: Response;
- readonly performance: PerformanceTelemetryContext;
- readonly identity: TelemetryModelIdentity;
-}
-
-export interface PassthroughAttemptArgs {
- readonly c: AuthedContext;
- readonly ctx: GatewayCtx;
- readonly candidate: ModelCandidate;
- readonly operation: PerformanceOperation;
- // Delegated to the passthrough caller so each endpoint keeps its
- // request-body shaping (`{ model: _, ...body }`) local. Any throw here
- // is preserved and the serve layer turns it into a 502 with the
- // internal-debug envelope.
- readonly call: (provider: Provider, model: ProviderModel, opts: UpstreamCallOptions) => Promise;
-}
-
-// Statuses whose response is defined to carry no body; constructing a Response
-// with one throws.
-// https://fetch.spec.whatwg.org/#null-body-status
-const NULL_BODY_STATUSES = new Set([101, 103, 204, 205, 304]);
-
-// The fallback loop keeps only the most recent failure and drops the rest, and
-// on the direct-connect egress a dropped response strands its socket — the
-// transport is released only when the body is read to its end or cancelled.
-// Reading a failed attempt's body now makes the result inert, so no later owner
-// has to remember to release it. The bytes, status and headers still forward
-// verbatim when this turns out to be the last candidate. This is what the chat
-// path already does through `readUpstreamApiError`.
-const materializeFailure = async (response: Response): Promise => {
- const bytes = await response.arrayBuffer();
- return new Response(NULL_BODY_STATUSES.has(response.status) ? null : bytes, {
- status: response.status,
- statusText: response.statusText,
- headers: response.headers,
- });
-};
-
-export const passthroughAttempt = async (args: PassthroughAttemptArgs): Promise => {
- const { c, ctx, candidate, operation, call } = args;
- const { response, modelKey } = await call(
- candidate.provider,
- providerModelOf(candidate),
- buildUpstreamCallOptions(candidate, ctx, inboundHeaders(c)),
- );
- return {
- type: 'plain',
- status: response.status,
- response: response.ok ? response : await materializeFailure(response),
- performance: upstreamPerformanceContext(ctx, candidate, operation),
- identity: telemetryModelIdentity(candidate, modelKey),
- };
-};
diff --git a/packages/gateway/src/data-plane/shared/passthrough-serve.ts b/packages/gateway/src/data-plane/shared/passthrough-serve.ts
deleted file mode 100644
index b98c4f7eb7..0000000000
--- a/packages/gateway/src/data-plane/shared/passthrough-serve.ts
+++ /dev/null
@@ -1,258 +0,0 @@
-// Shared serve scaffold for passthrough data-plane endpoints. These
-// bypass the chat source/target executor because they have no protocol
-// translation — the request body is forwarded to the chosen provider's
-// matching endpoint and the upstream response is passed through back to
-// the client. OpenAI Embeddings and OpenAI Images run the `json` branch
-// (single-shot body, OpenAI-shape `usage` block); /v1/completions runs the
-// `sse` branch (frame-level transformFrame closure + settleUsage).
-// Endpoint-owned response strategies handle specialized media-type state
-// machines. Usage and request-performance writes are scheduled through the
-// runtime's background scheduler so transient repo failures cannot turn a
-// successful 200 from upstream into a 502.
-
-import type { Context } from 'hono';
-import { streamSSE } from 'hono/streaming';
-import type { ContentfulStatusCode } from 'hono/utils/http-status';
-
-import type { PassthroughServeApiName } from './api-names.ts';
-import { appendFailedUpstreams } from './failed-upstreams.ts';
-import type { GatewayCtx } from './gateway-ctx.ts';
-import { iterateCandidates } from './iterate-candidates.ts';
-import { passthroughAttempt } from './passthrough-attempt.ts';
-import { type StreamCompletion, writeSSEFrames } from './sse.ts';
-import { recordFailedRequest } from './telemetry/performance.ts';
-import { settle } from './telemetry/settle.ts';
-import { forwardUpstreamHeaders, forwardUpstreamResponse } from './upstream-response.ts';
-import type { AuthedContext } from '../../middleware/auth.ts';
-import type { TokenUsage } from '../../repo/types.ts';
-import { enumerateModelCandidates } from '../providers/resolution.ts';
-import { doneFrame, eventFrame, type ModelKind, parseSSEStream, parseTargetStreamFrames, type ProtocolFrame, sseCommentFrame, sseFrame } from '@floway-dev/protocols/common';
-import { httpResponseToResponse, ProviderModelsUnavailableError, toInternalDebugError } from '@floway-dev/provider';
-import type { PerformanceOperation, PerformanceTelemetryContext, InternalModel, Provider, ProviderCallResult, ProviderModel, TelemetryModelIdentity, UpstreamCallOptions } from '@floway-dev/provider';
-
-// `json` (OpenAI Embeddings, OpenAI Images): single-shot body,
-// `extractBilling` reads usage / metadata off the parsed root. `sse`
-// (/v1/completions): frame stream, `transformFrame` mutates or drops frames
-// (return null), then `settleUsage` reports billing once the stream ends.
-// `strategy` delegates response handling after candidate selection to the
-// owning endpoint.
-type PassthroughResponseHandling =
- | {
- readonly format: 'json';
- readonly extractBilling: (body: unknown) => TokenUsage | null;
- }
- | {
- readonly format: 'sse';
- readonly transformFrame: (frame: ProtocolFrame) => ProtocolFrame | null;
- readonly settleUsage: () => TokenUsage | null;
- }
- | {
- readonly format: 'strategy';
- readonly respond: (context: PassthroughResponseStrategyContext) => Promise;
- };
-
-export interface PassthroughResponseStrategyContext {
- readonly c: AuthedContext;
- readonly ctx: GatewayCtx;
- readonly sourceApi: PassthroughServeApiName;
- readonly response: Response;
- readonly performance: PerformanceTelemetryContext;
- readonly identity: TelemetryModelIdentity;
-}
-
-interface PassthroughServeContext {
- readonly c: AuthedContext;
- readonly ctx: GatewayCtx;
- readonly sourceApi: PassthroughServeApiName;
- readonly operation: PerformanceOperation;
- // Already-validated public model id the client requested. The helper
- // resolves it against the provider registry; if no upstream serves the
- // id with the requested kind, the client sees a 404 with the standard
- // wording.
- readonly model: string;
- // The model kind this endpoint serves. The resolver filters candidates
- // to `model.kind === kind`; `sawModel=true && candidates=[]` becomes
- // the "model exists but doesn't support this endpoint" 400.
- readonly kind: ModelKind;
- // Endpoint-availability gate against a resolved candidate's `InternalModel`.
- // Reads `.endpoints` on the candidate — the row narrows to exactly one
- // contributing upstream, so those endpoints come verbatim from the emitting
- // upstream's `ProviderModel`.
- readonly modelServesEndpoint: (model: InternalModel) => boolean;
- // Any throw here is preserved and becomes a 502 with the internal-debug
- // envelope. `model` is the emitting upstream's `ProviderModel`.
- readonly call: (provider: Provider, model: ProviderModel, opts: UpstreamCallOptions) => Promise;
- readonly response: PassthroughResponseHandling;
-}
-
-// Uniform error envelope for this endpoint family.
-export const passthroughApiError = (c: Context, message: string, status: ContentfulStatusCode): Response =>
- c.json({ error: { message, type: 'api_error' } }, status);
-
-export const passthroughServe = async (input: PassthroughServeContext): Promise => {
- const { c, ctx, sourceApi, operation, model, kind, modelServesEndpoint, call, response: responseHandling } = input;
-
- try {
- // The shared resolver returns every candidate of the requested kind:
- // unprefixed + prefixed addressable surfaces fan out across upstreams,
- // a dated-suffix retry catches `-YYYYMMDD` ids the catalog only lists
- // in base form, and the kind filter rejects models of the wrong
- // family before they reach the endpoint check below. Iteration order
- // follows configured sort_order across upstreams, with the unprefixed
- // branch pushed before the prefixed one within a single upstream.
- // The first candidate whose endpoint-key check passes wins.
- //
- // Alias resolution is a top-of-chain step inside the resolver: an alias
- // id walks every target in `selection` order, tags each returned
- // candidate with that target's rule overlay, and dedups across the
- // flattened list. Passthrough endpoints never consult that overlay, so
- // whatever rules a target carries are inert here — the alias flow only
- // changes which id the gateway addresses upstream.
- const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({
- upstreamIds: ctx.upstreamIds,
- model,
- kind,
- scheduler: ctx.backgroundScheduler,
- runtimeLocation: ctx.runtimeLocation,
- });
- if (candidates.length === 0) {
- ctx.dump?.error('gateway');
- // `sawModel === false` means no upstream catalog knew the inbound id
- // at all (404); `sawModel === true` with zero candidates means the
- // id is known but every match was the wrong kind for this endpoint
- // (400), which mirrors the empty-viable case below.
- return sawModel
- ? passthroughApiError(c, appendFailedUpstreams(`Model ${model} does not support the ${sourceApi} endpoint.`, failedUpstreams), 400)
- : passthroughApiError(c, appendFailedUpstreams(`Model ${model} is not available on any configured upstream.`, failedUpstreams), 404);
- }
-
- // Endpoint-level pre-filter: drop candidates whose upstream model
- // exists for the requested kind but doesn't expose this endpoint's
- // specific capability (e.g. an embedding-kind model on an upstream
- // that only exposes chat). An empty viable set is the same "model
- // exists but no upstream serves this endpoint" 400 the empty-candidate
- // branch above surfaces.
- const viable = candidates.filter(c => modelServesEndpoint(c.model));
- if (viable.length === 0) {
- ctx.dump?.error('gateway');
- return passthroughApiError(c, appendFailedUpstreams(`Model ${model} does not support the ${sourceApi} endpoint.`, failedUpstreams), 400);
- }
-
- // Iterate the viable list. Each candidate's attempt runs the upstream
- // HTTP call and records performance telemetry; the shared
- // iterator returns the first 2xx or, on exhaustion, the last non-2xx
- // result. Request-perf and dump attribution wait until this point so
- // they land against the terminal candidate.
- const result = await iterateCandidates(
- viable,
- 'passthroughServe',
- ctx,
- operation,
- candidate => passthroughAttempt({
- c, ctx, candidate, operation,
- call,
- }),
- );
- const { response, performance: performanceContext, identity } = result;
-
- if (responseHandling.format === 'strategy') {
- return await responseHandling.respond({ c, ctx, sourceApi, response, performance: performanceContext, identity });
- }
-
- if (!response.ok) {
- // Exhausted — forward the last upstream response verbatim so clients
- // still see real upstream telemetry (status, retry-after, request-id,
- // ...) rather than a synthetic gateway envelope.
- recordFailedRequest(ctx, performanceContext);
- ctx.dump?.error('upstream', identity.upstream);
- return forwardUpstreamResponse(response);
- }
-
- if (responseHandling.format === 'json') {
- // A 2xx body that fails to parse must not 502 a client whose
- // upstream call already succeeded; we skip usage extraction and
- // log so missing rows stay traceable.
- let parsed: unknown;
- try {
- parsed = await response.clone().json();
- } catch (e) {
- console.warn(`passthrough-serve: failed to parse 2xx upstream body for ${sourceApi}; usage row will be skipped`, e instanceof Error ? e.message : String(e));
- parsed = undefined;
- }
- const usage = parsed !== undefined ? responseHandling.extractBilling(parsed) : null;
- ctx.dump?.success(identity, usage);
- settle(ctx, performanceContext, identity, usage, false);
- return forwardUpstreamResponse(response);
- }
-
- // Hono's streamSSE owns the response — forwardable upstream
- // headers must be staged on `c` *before* the streamSSE call so
- // they survive its internal newResponse.
- const upstreamBody = response.body;
- if (!upstreamBody) {
- ctx.dump?.failed(`${sourceApi} streaming upstream returned no body`);
- recordFailedRequest(ctx, performanceContext);
- // Preserve upstream correlation headers (x-request-id, cf-ray, ...)
- // on the synthesized 502 so this rare edge case is still traceable.
- forwardUpstreamHeaders(c, response.headers);
- return passthroughApiError(c, 'Upstream returned a streaming response with no body.', 502);
- }
- forwardUpstreamHeaders(c, response.headers);
- return streamSSE(c, async stream => {
- let completion: StreamCompletion = 'error';
- let streamError: unknown;
- // Tracks whether the upstream's terminal (`done`) frame arrived
- // before the writer settled. A client cancel after the terminal
- // frame is graceful (upstream already finished its work); a
- // mid-stream cancel or EOF without terminal is a real failure.
- // Mirrors SourceStreamState.failedAfter on the chat endpoints.
- let terminalFrameSeen = false;
- try {
- const frames = (async function* () {
- const sseFramesIn = parseSSEStream(upstreamBody, { signal: ctx.abortSignal });
- for await (const parsed of parseTargetStreamFrames(sseFramesIn, { protocol: sourceApi })) {
- const inputFrame: ProtocolFrame = parsed.type === 'done' ? doneFrame() : eventFrame(parsed.data);
- // Dump pre-transform, so forensics see upstream truth even
- // when the caller drops a frame from the client-facing stream.
- ctx.dump?.frame(inputFrame);
- if (inputFrame.type === 'done') terminalFrameSeen = true;
- const outputFrame = responseHandling.transformFrame(inputFrame);
- if (outputFrame === null) continue;
- yield outputFrame.type === 'done' ? sseFrame('[DONE]') : sseFrame(JSON.stringify(outputFrame.event));
- }
- })();
- completion = await writeSSEFrames(stream, frames, {
- keepAlive: { frame: sseCommentFrame('keepalive') },
- downstreamAbortController: ctx.downstreamAbortController,
- });
- } catch (e) {
- streamError = e;
- } finally {
- const usage = responseHandling.settleUsage();
- const failed = streamError !== undefined || completion === 'error' || !terminalFrameSeen;
- if (failed) {
- ctx.dump?.failed(streamError ?? `${sourceApi} stream ended with completion=${completion}`);
- } else {
- ctx.dump?.success(identity, usage);
- }
- // Record any accumulated usage regardless of the failed flag —
- // tokens already metered upstream should bill even when the
- // downstream half of the round-trip turned out badly. The chat
- // streaming endpoints follow the same rule.
- settle(ctx, performanceContext, identity, usage, failed);
- }
- });
- } catch (e) {
- if (e instanceof ProviderModelsUnavailableError) {
- const forwarded = httpResponseToResponse(e.httpResponse);
- if (forwarded) {
- ctx.dump?.error('upstream');
- return forwarded;
- }
- }
- // Attributes to whichever candidate iterateCandidates was on (or short-circuits if none started).
- recordFailedRequest(ctx, ctx.attempt.telemetry);
- ctx.dump?.failed(e);
- return c.json({ error: toInternalDebugError(e) }, 502);
- }
-};
diff --git a/packages/gateway/src/data-plane/tools/web-search/alpha-search/relay-response.ts b/packages/gateway/src/data-plane/tools/web-search/alpha-search/relay-response.ts
deleted file mode 100644
index d990344a7a..0000000000
--- a/packages/gateway/src/data-plane/tools/web-search/alpha-search/relay-response.ts
+++ /dev/null
@@ -1,36 +0,0 @@
-// Alpha-search upstream Fetch decodes a response's content coding before exposing its body stream,
-// but keeps the upstream Content-Encoding and Content-Length headers. Relaying
-// that stream with the stale representation headers makes the next Fetch
-// consumer decode plain bytes a second time. Rebuild the response around the
-// decoded stream and preserve every header that still describes it.
-
-const BLOCKED_RELAY_HEADERS: ReadonlySet = new Set([
- // Hop-by-hop headers (RFC 9110 §7.6.1).
- // https://www.rfc-editor.org/rfc/rfc9110#section-7.6.1
- 'connection',
- 'keep-alive',
- 'proxy-authenticate',
- 'proxy-authorization',
- 'te',
- 'trailer',
- 'transfer-encoding',
- 'upgrade',
- // Fetch owns the decoded body's representation framing.
- 'content-encoding',
- 'content-length',
- // Upstream session cookies must not bind a gateway client.
- 'set-cookie',
- 'set-cookie2',
-]);
-
-export const relayFetchedResponse = (response: Response): Response => {
- const headers = new Headers();
- for (const [name, value] of response.headers) {
- if (!BLOCKED_RELAY_HEADERS.has(name.toLowerCase())) headers.set(name, value);
- }
- return new Response(response.body, {
- status: response.status,
- statusText: response.statusText,
- headers,
- });
-};
diff --git a/packages/gateway/src/dump/accumulator.ts b/packages/gateway/src/dump/accumulator.ts
index e026ec9a2c..deebb7e391 100644
--- a/packages/gateway/src/dump/accumulator.ts
+++ b/packages/gateway/src/dump/accumulator.ts
@@ -1,23 +1,26 @@
-// Per-request dump pipeline. Opens the dump session (request snapshot +
-// opt-in decision) and exposes the mid-flight hooks the respond layer
-// calls to record outcomes and frames. When the api key has no retention
-// configured, opening returns null and the data plane pays no per-request
-// cost.
+// Per-request dump pipeline for an endpoint served by the onion: the record is
+// the turn's two **edges** — what the client sent, what the client got back.
+// Opens the dump session (request snapshot + opt-in decision) and exposes the
+// mid-flight hooks the respond layer calls to record outcomes and frames. When
+// the api key has no retention configured, opening returns null and the data
+// plane pays no per-request cost.
+//
+// A pipelined endpoint records the whole run instead; `run-sink.ts` is that
+// half, and both fill the same `DumpMetadata` through `DumpAttribution`.
import type { Context } from 'hono';
+import { DumpAttribution, oneLineError, streamReadError } from './attribution.ts';
import { getDumpBroker, getDumpStore } from './registry.ts';
+import type { StreamRecording } from './turn-dump.ts';
import type {
- DumpErrorMeta,
DumpMetadata,
DumpStreamEvent,
- DumpUpstreamRef,
DumpWriteRecord,
PreparedDumpRequestBody,
StoredDumpResponseBody,
} from './types.ts';
import type { RequestBody } from '../data-plane/shared/request-body.ts';
-import { getRepo } from '../repo/index.ts';
import type { ApiKey, TokenUsage } from '../repo/types.ts';
import { ulid } from '../shared/ulid.ts';
import type { BackgroundScheduler } from '@floway-dev/platform';
@@ -43,68 +46,16 @@ interface ResponseSnapshot {
readonly streamError: string | null;
}
-// Four independent attribution slots the mid-flight hooks fill: `model` and
-// `upstreamId` identify what the turn was about, `inputTokens` /
-// `outputTokens` quantify what the upstream reported. They're independent
-// because different outcomes set different subsets:
-//
-// • Every protocol handler calls `requestedModel(model)` immediately after
-// parsing the payload, so `model` is set regardless of outcome.
-// • `success(identity, usage)` fills all four; the upstream-resolved model
-// id may overwrite what `requestedModel` had.
-// • `error(kind, upstream?)` records a categorized api-error envelope
-// (`kind` matches `ApiErrorResult.source`). Real upstream non-2xx pass
-// `upstream` so a 4xx/5xx row in the dashboard names the upstream that
-// rejected the call; the gateway arm may also pass it when a candidate
-// was already chosen (item-not-found rewrite, server-tool input
-// rejection).
-// • `failed(reason)` records an uncategorized terminal failure: a thrown
-// exception (caught by the respond layer or passthrough-serve), a
-// source-emitted error frame, a downstream cancel, or a writer error.
-// Caller passes a string or Error; the accumulator one-line-formats
-// it (`.message` only — never the stack, which lives in the response
-// body's debug envelope).
-//
-// `requestedModel`-set model survives across both error variants so even an
-// outright-failed turn carries model attribution.
-
-// Anthropic-style disjoint per-category counts: input excludes cache reads
-// and cache writes; sum the present ones onto the dump's single inputTokens
-// column. Missing categories stay null (not measured) instead of zero so a
-// recorded zero genuinely means "upstream said zero".
-const tokenUsageInput = (usage: TokenUsage | null): number | null => {
- if (!usage) return null;
- const { input, input_cache_read, input_cache_write } = usage;
- if (input === undefined && input_cache_read === undefined && input_cache_write === undefined) return null;
- return (input ?? 0) + (input_cache_read ?? 0) + (input_cache_write ?? 0);
-};
-
-const oneLineError = (err: unknown): string => {
- const msg = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, ' ').trim();
- return msg.length > 500 ? `${msg.slice(0, 497)}…` : msg;
-};
-
const headerPairs = (headers: Headers): Array<[string, string]> => {
const pairs: Array<[string, string]> = [];
headers.forEach((value, name) => { pairs.push([name, value]); });
return pairs;
};
-const resolveUpstreamRef = async (id: string | null): Promise => {
- if (!id) return null;
- const upstream = await getRepo().upstreams.getById(id);
- if (!upstream) return null;
- return { id: upstream.id, name: upstream.name, kind: upstream.kind, hue: upstream.hue };
-};
-
export class DumpAccumulator {
+ private readonly attribution = new DumpAttribution();
private readonly events: DumpStreamEvent[] = [];
private sentPayloadBytes = 0;
- private model: string | null = null;
- private upstreamId: string | null = null;
- private inputTokens: number | null = null;
- private outputTokens: number | null = null;
- private errorMeta: DumpErrorMeta | null = null;
private readonly preparedRequestBody: Promise;
constructor(
@@ -124,16 +75,15 @@ export class DumpAccumulator {
// --- mid-flight hooks (called from per-protocol respond layer) ---
requestedModel(model: string): void {
- this.model = model;
+ this.attribution.requestedModel(model);
}
error(kind: 'upstream' | 'gateway', upstream?: string): void {
- this.errorMeta = { kind };
- if (upstream !== undefined) this.upstreamId = upstream;
+ this.attribution.error(kind, upstream);
}
failed(reason: unknown): void {
- this.errorMeta = { kind: 'failed', reason: typeof reason === 'string' ? reason : oneLineError(reason) };
+ this.attribution.failed(reason);
}
// Records one protocol frame. Stored as the canonical ProtocolFrame so
@@ -144,15 +94,20 @@ export class DumpAccumulator {
this.events.push({ frame, ts: Date.now() - this.startedAt });
}
+ /** This shape has one frame log and no way to name anything in it, so every stream lands in
+ * the same place, there is no terminator to write, and no fact can point at one. A turn
+ * that opened two would have them interleaved — which the endpoints on this shape do not do,
+ * and which the run shape is what fixes. */
+ openStream(): StreamRecording {
+ return { frame: frame => { this.frame(frame); }, end: () => {}, fact: null };
+ }
+
recordSentPayloadBytes(byteLength: number): void {
this.sentPayloadBytes += byteLength;
}
success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void {
- this.model = identity.model;
- this.upstreamId = identity.upstream;
- this.inputTokens = tokenUsageInput(usage);
- this.outputTokens = usage?.output ?? null;
+ this.attribution.success(identity, usage);
}
// --- response-side: handler exit ---
@@ -244,8 +199,8 @@ export class DumpAccumulator {
const recordId = ulid(completedAt);
// Prefer the accumulator's frame log so dumps reflect the gateway's
- // frame sequence regardless of negotiated wire shape; passthrough
- // endpoints with no frames fall back to captured bytes.
+ // frame sequence regardless of negotiated wire shape; a turn with no
+ // frames falls back to captured bytes.
const responseBody: StoredDumpResponseBody = this.events.length > 0
? { type: 'stream', events: this.events }
: response.bytes.byteLength > 0 || response.streamError !== null
@@ -254,32 +209,24 @@ export class DumpAccumulator {
: { type: 'bytes', body: response.bytes }
: { type: 'none' };
- const meta: DumpMetadata = {
+ const meta: DumpMetadata = await this.attribution.metadata({
id: recordId,
startedAt: this.startedAt,
completedAt,
method: this.requestSnapshot.method,
path: this.requestSnapshot.path,
status: response.status,
- upstream: await resolveUpstreamRef(this.upstreamId),
- model: this.model,
- inputTokens: this.inputTokens,
- outputTokens: this.outputTokens,
requestBytes: this.requestSnapshot.bodyByteLength,
responseBytes: response.payloadBytes,
- durationMs: completedAt - this.startedAt,
- // Precedence: an explicit error stamp from the respond path wins;
- // otherwise a request-body read failure (operator-side payload didn't
- // arrive intact) outranks a response-body read failure. Both stream-
- // read failures surface as `kind: 'failed'`.
- error: this.errorMeta
- ?? (this.requestSnapshot.streamError !== null ? { kind: 'failed', reason: this.requestSnapshot.streamError } : null)
- ?? (response.streamError !== null ? { kind: 'failed', reason: response.streamError } : null),
- };
+ // Precedence: an explicit error stamp from the respond path wins — the
+ // assembler applies this only when there is none.
+ fallbackError: streamReadError(this.requestSnapshot.streamError, response.streamError),
+ });
// Commit the row before publishing so subscribers fetching detail off the meta frame find it.
try {
const record: DumpWriteRecord = {
+ shape: 'edge',
meta,
request: {
method: this.requestSnapshot.method,
diff --git a/packages/gateway/src/dump/attribution.ts b/packages/gateway/src/dump/attribution.ts
new file mode 100644
index 0000000000..9118a21efe
--- /dev/null
+++ b/packages/gateway/src/dump/attribution.ts
@@ -0,0 +1,131 @@
+// What a turn's record says about the turn, and what fills it.
+//
+// `DumpMetadata` is common to both record shapes — the dashboard lists a run
+// and a pair of edges in the same list, and a turn's model, upstream and token
+// counts do not depend on which mechanism served it — so the hooks that stamp
+// attribution and the assembly that turns it into metadata are one thing rather
+// than one per shape.
+//
+// Four independent slots the mid-flight hooks fill: `model` and `upstreamId`
+// identify what the turn was about, `inputTokens` / `outputTokens` quantify
+// what the upstream reported. They're independent because different outcomes
+// set different subsets:
+//
+// • Every protocol handler calls `requestedModel(model)` immediately after
+// parsing the payload, so `model` is set regardless of outcome.
+// • `success(identity, usage)` fills all four; the upstream-resolved model
+// id may overwrite what `requestedModel` had.
+// • `error(kind, upstream?)` records a categorized api-error envelope
+// (`kind` matches `ApiErrorResult.source`). Real upstream non-2xx pass
+// `upstream` so a 4xx/5xx row in the dashboard names the upstream that
+// rejected the call; the gateway arm may also pass it when a candidate
+// was already chosen (item-not-found rewrite, server-tool input
+// rejection).
+// • `failed(reason)` records an uncategorized terminal failure: a thrown
+// exception, a source-emitted error frame, a downstream cancel, or a
+// writer error. Caller passes a string or Error; this one-line-formats it
+// (`.message` only — never the stack, which lives in the response body's
+// debug envelope).
+//
+// `requestedModel`-set model survives across both error variants so even an
+// outright-failed turn carries model attribution.
+
+import type { DumpErrorMeta, DumpMetadata, DumpUpstreamRef } from './types.ts';
+import { getRepo } from '../repo/index.ts';
+import type { TokenUsage } from '../repo/types.ts';
+import type { TelemetryModelIdentity } from '@floway-dev/provider';
+
+export const oneLineError = (err: unknown): string => {
+ const msg = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, ' ').trim();
+ return msg.length > 500 ? `${msg.slice(0, 497)}…` : msg;
+};
+
+// Anthropic-style disjoint per-category counts: input excludes cache reads
+// and cache writes; sum the present ones onto the dump's single inputTokens
+// column. Missing categories stay null (not measured) instead of zero so a
+// recorded zero genuinely means "upstream said zero".
+const tokenUsageInput = (usage: TokenUsage | null): number | null => {
+ if (!usage) return null;
+ const { input, input_cache_read, input_cache_write } = usage;
+ if (input === undefined && input_cache_read === undefined && input_cache_write === undefined) return null;
+ return (input ?? 0) + (input_cache_read ?? 0) + (input_cache_write ?? 0);
+};
+
+const resolveUpstreamRef = async (id: string | null): Promise => {
+ if (!id) return null;
+ const upstream = await getRepo().upstreams.getById(id);
+ if (!upstream) return null;
+ return { id: upstream.id, name: upstream.name, kind: upstream.kind, hue: upstream.hue };
+};
+
+// What only the recording side knows: identity, timing and the measured sizes
+// of the two edges. Everything else on the metadata comes from the hooks.
+export interface DumpTurnOutcome {
+ readonly id: string;
+ readonly startedAt: number;
+ readonly completedAt: number;
+ readonly method: string;
+ readonly path: string;
+ readonly status: number | null;
+ readonly requestBytes: number;
+ readonly responseBytes: number;
+ // Applied only when no hook stamped an error, so an explicit stamp from the
+ // respond path always outranks a transport-level read failure.
+ readonly fallbackError: DumpErrorMeta | null;
+}
+
+// A request-body read failure — the operator-side payload did not arrive intact
+// — outranks a failure reading back what was answered. Both surface as
+// `kind: 'failed'`.
+export const streamReadError = (request: string | null, response: string | null): DumpErrorMeta | null => {
+ if (request !== null) return { kind: 'failed', reason: request };
+ if (response !== null) return { kind: 'failed', reason: response };
+ return null;
+};
+
+export class DumpAttribution {
+ private model: string | null = null;
+ private upstreamId: string | null = null;
+ private inputTokens: number | null = null;
+ private outputTokens: number | null = null;
+ private errorMeta: DumpErrorMeta | null = null;
+
+ requestedModel(model: string): void {
+ this.model = model;
+ }
+
+ error(kind: 'upstream' | 'gateway', upstream?: string): void {
+ this.errorMeta = { kind };
+ if (upstream !== undefined) this.upstreamId = upstream;
+ }
+
+ failed(reason: unknown): void {
+ this.errorMeta = { kind: 'failed', reason: typeof reason === 'string' ? reason : oneLineError(reason) };
+ }
+
+ success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void {
+ this.model = identity.model;
+ this.upstreamId = identity.upstream;
+ this.inputTokens = tokenUsageInput(usage);
+ this.outputTokens = usage?.output ?? null;
+ }
+
+ async metadata(outcome: DumpTurnOutcome): Promise {
+ return {
+ id: outcome.id,
+ startedAt: outcome.startedAt,
+ completedAt: outcome.completedAt,
+ method: outcome.method,
+ path: outcome.path,
+ status: outcome.status,
+ upstream: await resolveUpstreamRef(this.upstreamId),
+ model: this.model,
+ inputTokens: this.inputTokens,
+ outputTokens: this.outputTokens,
+ requestBytes: outcome.requestBytes,
+ responseBytes: outcome.responseBytes,
+ durationMs: outcome.completedAt - outcome.startedAt,
+ error: this.errorMeta ?? outcome.fallbackError,
+ };
+ }
+}
diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts
new file mode 100644
index 0000000000..2e9c36489f
--- /dev/null
+++ b/packages/gateway/src/dump/run-sink.ts
@@ -0,0 +1,224 @@
+// The pipeline's half of the dump: the record of a **whole run**.
+//
+// A run emits events — every stage, both directions — and this is where they
+// go. What the runner hands `services.dump` is `sink`; what it accumulates is
+// the encoded stream, folded event by event by `createRunEncoder` so a run is
+// written down as it happens rather than re-walked at the end. At the terminal
+// point the stream becomes NDJSON and one record goes through `DumpStore.put`,
+// which is the same contract the edge record is written under: the stream is
+// one more gzipped body file, retained and swept by the same row.
+//
+// Recording is conditional and that is structural: with no retention configured
+// there is no sink to hand to `run`, so `services.dump` is absent and the
+// runner does none of the recording — not a no-op that accumulates and throws
+// the result away.
+
+import { DumpAttribution, oneLineError, streamReadError } from './attribution.ts';
+import { getDumpBroker, getDumpStore } from './registry.ts';
+import type { StreamRecording } from './turn-dump.ts';
+import type { DumpMetadata } from './types.ts';
+import type { RequestBody } from '../data-plane/shared/request-body.ts';
+import type { ApiKey, TokenUsage } from '../repo/types.ts';
+import { ulid } from '../shared/ulid.ts';
+import { createRunEncoder, streamFact, toNdjson, type DumpEvent, type Event } from '@floway-dev/pipeline';
+import type { BackgroundScheduler } from '@floway-dev/platform';
+import type { ProtocolFrame } from '@floway-dev/protocols/common';
+import type { TelemetryModelIdentity } from '@floway-dev/provider';
+
+// What the client sent, as the metadata needs it. The headers and the body are
+// facts the run itself records, so nothing is snapshotted here beyond what a
+// list row shows without opening the record.
+interface RequestSnapshot {
+ readonly method: string;
+ readonly path: string;
+ readonly bodyByteLength: number;
+ readonly streamError: string | null;
+}
+
+export class RunDump {
+ private readonly attribution = new DumpAttribution();
+ private readonly encode = createRunEncoder();
+ private readonly events: DumpEvent[] = [];
+ private sentPayloadBytes = 0;
+ private streams = 0;
+ private answerStream: StreamRecording | undefined;
+
+ constructor(
+ private readonly apiKey: ApiKey,
+ private readonly requestSnapshot: RequestSnapshot,
+ private readonly startedAt: number,
+ private readonly backgroundScheduler: BackgroundScheduler,
+ ) {}
+
+ /** What the prologue hands to `run` as `services.dump`. Bound to this
+ * recording, so it travels as a value. */
+ readonly sink = (event: Event): void => {
+ for (const encoded of this.encode(event)) this.events.push(encoded);
+ };
+
+ // --- mid-flight hooks, the same ones the edge record is stamped with ---
+
+ requestedModel(model: string): void {
+ this.attribution.requestedModel(model);
+ }
+
+ error(kind: 'upstream' | 'gateway', upstream?: string): void {
+ this.attribution.error(kind, upstream);
+ }
+
+ failed(reason: unknown): void {
+ this.attribution.failed(reason);
+ }
+
+ /**
+ * A frame the client was sent, as the event the format names for one.
+ *
+ * The edge dump kept a frame log of its own; here a frame is content about a stream, so it
+ * is `stream.frame` and it folds through the same encoder as everything else. A frame pushed
+ * without opening a stream first belongs to the run's first one, which is what a transport
+ * writing single synthesized frames alongside its answer is doing.
+ */
+ frame(frame: ProtocolFrame): void {
+ this.answerStream ??= this.openStream();
+ this.answerStream.frame(frame);
+ }
+
+ /**
+ * Begins recording one stream, under an id of its own.
+ *
+ * The id is what makes the frames resolvable: the fact holding the stream carries
+ * `{"$stream": n}` and every frame event names the same `n`, so a run that opened two — a
+ * sub-request's stream beside the answer's — keeps them apart. `end` says the record of that
+ * stream is complete, and a client that stopped reading never reaches it.
+ */
+ openStream(): StreamRecording {
+ const streamId = ++this.streams;
+ return {
+ frame: frame => { this.sink({ type: 'stream.frame', streamId, frames: [frame] }); },
+ end: () => { this.sink({ type: 'stream.end', streamId }); },
+ fact: streamFact(streamId),
+ };
+ }
+
+ success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void {
+ this.attribution.success(identity, usage);
+ }
+
+ // --- terminal point ---
+
+ // Two input shapes, matching the edge accumulator's seam:
+ //
+ // • `(status, responseBytes)` — the caller already knows what it wrote.
+ // • `(response)` — tees the answer so the client gets bytes flowing while a
+ // background reader measures the other half. Only the byte count is kept:
+ // what the client was sent is already in the run's own record, so
+ // retaining a second copy of a streamed answer would buy nothing.
+ //
+ // The drain → encode → store put → broker publish runs on the runtime's
+ // BackgroundScheduler so a dump write failure cannot turn a served answer
+ // into a 502.
+ /** A transport that writes its own frames counts what it sent, because nothing downstream
+ * of it can. The run's own bytes are its events; this is the answer's. */
+ recordSentPayloadBytes(byteLength: number): void {
+ this.sentPayloadBytes += byteLength;
+ }
+
+ finalize(status: number | null, responseBytes: number): void;
+ finalize(response: Response): Response;
+ finalize(...args: [number | null, number] | [Response]): void | Response {
+ if (args.length === 2) {
+ const [status, responseBytes] = args;
+ // A transport that wrote its own frames counted them as it went; what it passes here
+ // is whatever else it sent alongside them.
+ this.backgroundScheduler(this.write(status, responseBytes + this.sentPayloadBytes, null));
+ return;
+ }
+
+ const [response] = args;
+ if (response.body === null) {
+ this.finalize(response.status, 0);
+ return response;
+ }
+
+ const [forClient, forMeasure] = response.body.tee();
+ this.backgroundScheduler((async () => {
+ const reader = forMeasure.getReader();
+ let payloadBytes = 0;
+ let streamError: string | null = null;
+ try {
+ for (;;) {
+ const { value, done } = await reader.read();
+ if (done) break;
+ payloadBytes += value.byteLength;
+ }
+ } catch (err) {
+ streamError = oneLineError(err);
+ }
+ await this.write(response.status, payloadBytes, streamError);
+ })());
+
+ return new Response(forClient, {
+ status: response.status,
+ statusText: response.statusText,
+ headers: response.headers,
+ });
+ }
+
+ private async write(status: number | null, responseBytes: number, responseStreamError: string | null): Promise {
+ // ULID-from-completedAt keeps ids increasing with row creation time; the
+ // random tail provides the deterministic tie-breaker for one millisecond.
+ const completedAt = Date.now();
+ const recordId = ulid(completedAt);
+ const meta: DumpMetadata = await this.attribution.metadata({
+ id: recordId,
+ startedAt: this.startedAt,
+ completedAt,
+ method: this.requestSnapshot.method,
+ path: this.requestSnapshot.path,
+ status,
+ requestBytes: this.requestSnapshot.bodyByteLength,
+ responseBytes,
+ fallbackError: streamReadError(this.requestSnapshot.streamError, responseStreamError),
+ });
+
+ // Commit the row before publishing so subscribers fetching detail off the meta frame find it.
+ try {
+ await getDumpStore().put(this.apiKey.id, {
+ shape: 'run',
+ meta,
+ events: new TextEncoder().encode(toNdjson(this.events)),
+ });
+ await getDumpBroker().publish(this.apiKey.id, meta);
+ } catch (err) {
+ console.error(`[dump] run write failed for key=${this.apiKey.id} record=${recordId}`, oneLineError(err));
+ }
+ }
+}
+
+/**
+ * Returns null when the api key opts out of dumps, and the absence is the
+ * mechanism: the prologue has nothing to put in `services.dump`, so the run
+ * emits nothing and accumulates nothing.
+ *
+ * `method` and `path` are passed rather than read off a request so a transport
+ * that carries several turns over one connection can name each one as what it
+ * is — the same reason the edge accumulator takes its method explicitly.
+ */
+export const openRunDump = (
+ apiKey: ApiKey,
+ turn: { readonly method: string; readonly path: string; readonly body: RequestBody },
+ backgroundScheduler: BackgroundScheduler,
+): RunDump | null => {
+ if (apiKey.dumpRetentionSeconds === null) return null;
+ return new RunDump(
+ apiKey,
+ {
+ method: turn.method,
+ path: turn.path,
+ bodyByteLength: turn.body.bytes.byteLength,
+ streamError: turn.body.streamError,
+ },
+ Date.now(),
+ backgroundScheduler,
+ );
+};
diff --git a/packages/gateway/src/dump/schemas.ts b/packages/gateway/src/dump/schemas.ts
index d5c820ada8..ac4d3ee3e7 100644
--- a/packages/gateway/src/dump/schemas.ts
+++ b/packages/gateway/src/dump/schemas.ts
@@ -35,9 +35,13 @@ export const persistedDumpMetadataSchema = dumpMetadataSchema.omit({ upstream: t
export const dumpHeadersSchema = z.array(z.tuple([z.string(), z.string()]));
+// `type` says how to read the file the descriptor points at: raw bytes, the
+// JSON array of captured protocol frames, or a run's NDJSON event stream. It is
+// also what tells the reader which shape the record is — a run's stream is one
+// more body file under the same contract, so the row needs nothing else.
export const dumpBodyDescriptorSchema = z.object({
key: z.string(),
- type: z.enum(['bytes', 'events']),
+ type: z.enum(['bytes', 'events', 'run']),
}).strict();
const dumpProtocolFrameSchema = z.discriminatedUnion('type', [
diff --git a/packages/gateway/src/dump/turn-dump.ts b/packages/gateway/src/dump/turn-dump.ts
new file mode 100644
index 0000000000..4037b368d3
--- /dev/null
+++ b/packages/gateway/src/dump/turn-dump.ts
@@ -0,0 +1,99 @@
+// What a turn stamps on its recording, whichever shape that recording is.
+//
+// Two shapes are alive while the migration runs: the edge record, which holds what the client
+// sent and what it got back, and the run record, which holds every stage of the pipeline as
+// an event stream. The shape follows the endpoint — a pipelined one produces the run shape —
+// and the stages in between neither know nor care which they are stamping.
+
+import type { TokenUsage } from '../repo/types.ts';
+import { isStreamFact, type StreamFact } from '@floway-dev/pipeline';
+import type { ProtocolFrame } from '@floway-dev/protocols/common';
+import type { TelemetryModelIdentity } from '@floway-dev/provider';
+
+/**
+ * One stream, as the recording knows it.
+ *
+ * A run record identifies its streams, because their content arrives over time and after the
+ * fact that holds them: the fact carries `{"$stream": n}` and the frames arrive afterwards
+ * naming that id. `end` is what says the record of this stream is complete — a client that
+ * stopped reading leaves it short, and the absence of the terminator is how a reader tells a
+ * stream that ended from one that was cut off.
+ *
+ * The edge record has one frame log and no way to name anything in it, so its `fact` is null
+ * and its terminator has nothing to write. That is the shape's limit rather than an omission,
+ * and it goes away with the shape.
+ */
+export interface StreamRecording {
+ frame(frame: ProtocolFrame): void;
+ end(): void;
+ readonly fact: StreamFact | null;
+}
+
+export interface TurnDump {
+ requestedModel(model: string): void;
+ success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void;
+ error(kind: 'upstream' | 'gateway', upstream?: string): void;
+ failed(reason: unknown): void;
+ frame(frame: ProtocolFrame): void;
+ /** Begins recording one stream. Every call is a new one, which is what lets a turn that
+ * opens two — a sub-request beside the answer — keep them apart. */
+ openStream(): StreamRecording;
+ /** How much of the answer actually went out. A transport that writes its own frames counts
+ * them itself, because nothing downstream of it can. */
+ recordSentPayloadBytes(byteLength: number): void;
+ /** Closes the recording. A transport that owns its own response — the WebSocket turn —
+ * states what it sent; an HTTP one hands over the response so the bytes can be teed. */
+ finalize(status: number | null, responseBytes: number | readonly unknown[]): void;
+ finalize(response: Response): Response;
+}
+
+/**
+ * Records a stream's frames as they are read, and hands them on untouched.
+ *
+ * What the record holds is the stream as the client is served it, so the tee goes outside
+ * whatever shapes the frames and inside whatever frames them for a transport. Reading is what
+ * records: a losing attempt nobody read contributes nothing, because the release path drains
+ * the stream underneath this rather than through it.
+ *
+ * The value handed back *is* the stream reference, so the fact that holds it encodes as
+ * `{"$stream": n}` and the frames that arrive afterwards say which stream they belong to.
+ *
+ * A record holds protocol frames, which is what most families' streams already carry. The
+ * family whose stream is bare protocol events says how one becomes a frame, because the record
+ * cannot guess and a cast would be it guessing.
+ */
+export function recordStream(stream: AsyncIterable>, dump: TurnDump | null): AsyncIterable>;
+export function recordStream(stream: AsyncIterable, dump: TurnDump | null, asFrame: (value: T) => ProtocolFrame): AsyncIterable;
+export function recordStream(
+ stream: AsyncIterable,
+ dump: TurnDump | null,
+ asFrame: (value: T) => ProtocolFrame = value => value as ProtocolFrame,
+): AsyncIterable {
+ // No recording configured hands the same iterable back, so a record shows no step where
+ // nothing happened and the stream is not wrapped for nobody.
+ if (dump === null) return stream;
+
+ const recording = dump.openStream();
+ return {
+ ...recording.fact,
+ [Symbol.asyncIterator]: () => (async function* () {
+ for await (const value of stream) {
+ recording.frame(asFrame(value));
+ yield value;
+ }
+ // Reached only where the source ran out on its own, which is what makes the record of
+ // this stream complete. A reader that stopped early never gets here.
+ recording.end();
+ })(),
+ };
+}
+
+/**
+ * The reference a value carries to the stream the record holds, for a wrapper to carry across.
+ *
+ * A stream is framed again on its way out — protocol frames become SSE — and what the client
+ * is handed is a different object over the same frames. Carrying the reference onto it is what
+ * lets the fact that produced the stream and the fact that framed it point at one record.
+ */
+export const streamReferenceOf = (value: unknown): StreamFact | Record =>
+ isStreamFact(value) ? { ...value } : {};
diff --git a/packages/gateway/src/dump/types.ts b/packages/gateway/src/dump/types.ts
index a6a9af955b..55ab15fddf 100644
--- a/packages/gateway/src/dump/types.ts
+++ b/packages/gateway/src/dump/types.ts
@@ -8,6 +8,17 @@
// view served to the dashboard by `dumpRecordToWire`.
//
// `DumpMetadata` and `DumpStreamEvent` are body-free and shared verbatim.
+//
+// Across all three, a record is one of two shapes, and `shape` is what a reader
+// dispatches on. An endpoint served by the onion records its **edges** — what
+// the client sent and what the client got back. An endpoint served by a
+// pipeline records the **whole run**: every stage, both directions, as the
+// NDJSON event stream `@floway-dev/pipeline` encodes. The edges are still in
+// that stream; they are the first and last things it holds. The shape follows
+// the endpoint, so both are alive for as long as the two mechanisms are.
+//
+// What stays common is `DumpMetadata`: the dashboard lists both kinds together,
+// and one turn's attribution does not depend on which mechanism served it.
import type { z } from 'zod';
@@ -17,6 +28,11 @@ import type {
dumpStreamEventSchema,
dumpUpstreamRefSchema,
} from './schemas.ts';
+import type { DumpEvent } from '@floway-dev/pipeline';
+
+// Re-exported because the dashboard reads a run record's stream through this
+// module and has no other reach into the pipeline package.
+export type { DumpEvent };
export type DumpRecordId = string;
@@ -25,7 +41,7 @@ export type DumpUpstreamRef = z.infer;
// What went wrong on a failed turn. Either a categorized api-error envelope
// (real upstream non-2xx or a gateway-synthesized envelope — `kind` matches
// `ApiErrorResult.source`) or an uncategorized failure (anything the
-// respond layer / passthrough-serve caught or observed mid-flight: thrown
+// respond layer caught or observed mid-flight: thrown
// exceptions, source-emitted error events, downstream cancels, write
// errors) carrying its one-line reason text. The categorized form stores
// no status — `DumpMetadata.status` already does.
@@ -79,18 +95,38 @@ export interface StoredDumpResponse {
body: StoredDumpResponseBody;
}
-export type StoredDumpRecord = {
+export type StoredDumpEdgeRecord = {
+ shape: 'edge';
meta: DumpMetadata;
request: StoredDumpRequest;
response: StoredDumpResponse;
};
-export type DumpWriteRecord = {
+// The run's NDJSON, still as bytes: it is a body file under the same contract
+// as the other two, gzipped into the file store and pointed at by the row's
+// descriptor. One `put` carries it whole — measured on production, P99 of a
+// turn's request and response together is 2.86 MB, well under the 5 MiB below
+// which multipart has nothing to divide.
+export type StoredDumpRunRecord = {
+ shape: 'run';
+ meta: DumpMetadata;
+ events: Uint8Array;
+};
+
+export type StoredDumpRecord = StoredDumpEdgeRecord | StoredDumpRunRecord;
+
+export type DumpWriteEdgeRecord = {
+ shape: 'edge';
meta: DumpMetadata;
request: DumpWriteRequest;
response: StoredDumpResponse;
};
+// The run half is the stored shape unchanged: a run's stream is encoded once the
+// run is over, so there is nothing to compress ahead of the terminal write the
+// way a request body is.
+export type DumpWriteRecord = DumpWriteEdgeRecord | StoredDumpRunRecord;
+
// --- Wire shape (serialized JSON over the dashboard's control plane) ---
// `utf8` is chosen from the upstream content-type, with a UTF-8-fatal
@@ -119,8 +155,20 @@ interface DumpResponse {
body: DumpResponseBody;
}
-export type DumpRecord = {
+export type DumpEdgeRecord = {
+ shape: 'edge';
meta: DumpMetadata;
request: DumpRequest;
response: DumpResponse;
};
+
+// The stored NDJSON verbatim, decoded as UTF-8. One line is one event and one
+// SSE `data:` payload, so what a reader parses here is what a live observer
+// will parse frame by frame once the fan-out exists.
+export type DumpRunRecord = {
+ shape: 'run';
+ meta: DumpMetadata;
+ events: string;
+};
+
+export type DumpRecord = DumpEdgeRecord | DumpRunRecord;
diff --git a/packages/gateway/src/dump/wire.ts b/packages/gateway/src/dump/wire.ts
index 7659ae2914..a7d3b84518 100644
--- a/packages/gateway/src/dump/wire.ts
+++ b/packages/gateway/src/dump/wire.ts
@@ -31,18 +31,30 @@ const responseBodyToWire = (body: StoredDumpResponseBody, contentType: string):
};
// Sole place the storage shape crosses into the wire shape. Called once,
-// at the control-plane HTTP boundary, just before `c.json(...)`.
-export const dumpRecordToWire = (record: StoredDumpRecord): DumpRecord => ({
- meta: record.meta,
- request: {
- method: record.request.method,
- path: record.request.path,
- headers: record.request.headers,
- body: encodeBodyForWire(record.request.body, contentTypeOf(record.request.headers)),
- },
- response: {
- status: record.response.status,
- headers: record.response.headers,
- body: responseBodyToWire(record.response.body, contentTypeOf(record.response.headers)),
- },
-});
+// at the control-plane HTTP boundary, just before `c.json(...)`. A run's stream
+// is decoded UTF-8-fatal rather than sniffed: the gateway wrote those bytes
+// itself, so anything that fails to decode is a corrupted record and says so.
+export const dumpRecordToWire = (record: StoredDumpRecord): DumpRecord => {
+ if (record.shape === 'run') {
+ return {
+ shape: 'run',
+ meta: record.meta,
+ events: new TextDecoder('utf-8', { fatal: true }).decode(record.events),
+ };
+ }
+ return {
+ shape: 'edge',
+ meta: record.meta,
+ request: {
+ method: record.request.method,
+ path: record.request.path,
+ headers: record.request.headers,
+ body: encodeBodyForWire(record.request.body, contentTypeOf(record.request.headers)),
+ },
+ response: {
+ status: record.response.status,
+ headers: record.response.headers,
+ body: responseBodyToWire(record.response.body, contentTypeOf(record.response.headers)),
+ },
+ };
+};
diff --git a/packages/gateway/src/repo/dump-store.ts b/packages/gateway/src/repo/dump-store.ts
index ab29082fa6..4437589425 100644
--- a/packages/gateway/src/repo/dump-store.ts
+++ b/packages/gateway/src/repo/dump-store.ts
@@ -26,7 +26,7 @@ import type {
import { gunzipBytes, gzipBytes } from '../shared/gzip.ts';
import type { FileStore, SqlDatabase } from '@floway-dev/platform';
-// Bodies live at `dumps/v1/{keyId}/{YYYYMMDDHH}/{recordId}-{uniqueSuffix}.{req|resp}.gz`.
+// Bodies live at `dumps/v1/{keyId}/{YYYYMMDDHH}/{recordId}-{uniqueSuffix}.{req|resp|run}.gz`.
// The hour segment remains useful for operator inspection; lifecycle and
// collection are driven by the shared spilled_files registry.
@@ -72,14 +72,14 @@ const hourBucket = (ms: number): string => {
return `${y}${m}${d}${h}`;
};
-const bodyPath = (keyId: string, bucket: string, recordId: string, side: 'req' | 'resp'): string =>
+const bodyPath = (keyId: string, bucket: string, recordId: string, side: 'req' | 'resp' | 'run'): string =>
`${DUMP_FILE_PREFIX}${keyId}/${bucket}/${recordId}-${crypto.randomUUID()}.${side}.gz`;
const putRawBody = async (
files: FileStore,
key: string,
rawBytes: Uint8Array,
- type: 'bytes' | 'events',
+ type: DumpBodyDescriptor['type'],
): Promise => {
const gz = await gzipBytes(rawBytes);
await files.put(key, gz);
@@ -102,6 +102,12 @@ const fetchBody = async (files: FileStore, descriptor: DumpBodyDescriptor): Prom
return await gunzipBytes(gz);
};
+// A run record has no edge halves at row level — its request and response are
+// events inside the stream — and `request_headers_json` is NOT NULL. The empty
+// list is how the row spells "this shape has none"; nothing reads it back,
+// because `get` dispatches on the body descriptor first.
+const NO_EDGE_HEADERS = '[]';
+
export class FileDumpStore implements DumpStore {
constructor(private readonly db: SqlDatabase, private readonly files: FileStore) {}
@@ -113,14 +119,20 @@ export class FileDumpStore implements DumpStore {
};
}
+ // Both shapes take the same route: files are staged in the registry, written,
+ // and only then pointed at by a row. A run's NDJSON is one more body file
+ // under that contract, carried by the response descriptor — which is what
+ // leaves retention, the sweep and the files-before-row ordering untouched by
+ // its arrival.
async put(keyId: string, record: DumpWriteRecord): Promise {
const bucket = hourBucket(record.meta.completedAt);
- const requestFileKey = record.request.body.decodedByteLength === 0
+ const requestFileKey = record.shape === 'run' || record.request.body.decodedByteLength === 0
? null
: bodyPath(keyId, bucket, record.meta.id, 'req');
- const responseFileKey = record.response.body.type === 'bytes' && record.response.body.body.byteLength === 0
- ? null
+ const responseFileKey = record.shape === 'run'
+ ? bodyPath(keyId, bucket, record.meta.id, 'run')
: record.response.body.type === 'none'
+ || (record.response.body.type === 'bytes' && record.response.body.body.byteLength === 0)
? null
: bodyPath(keyId, bucket, record.meta.id, 'resp');
const staged = [
@@ -142,22 +154,27 @@ export class FileDumpStore implements DumpStore {
.bind(keyId, record.meta.id, Date.now() + SPILLED_FILE_STAGE_GRACE_MS, JSON.stringify(staged))
.run();
}
- const requestDescriptor = record.request.body.decodedByteLength === 0
- ? null
- : await putPreparedBody(this.files, requestFileKey!, record.request.body);
+ let requestDescriptor: DumpBodyDescriptor | null = null;
let responseDescriptor: DumpBodyDescriptor | null = null;
- if (record.response.body.type === 'bytes') {
- if (record.response.body.body.byteLength > 0) {
- responseDescriptor = await putRawBody(this.files, responseFileKey!, record.response.body.body, 'bytes');
+ if (record.shape === 'run') {
+ responseDescriptor = await putRawBody(this.files, responseFileKey!, record.events, 'run');
+ } else {
+ if (requestFileKey !== null) {
+ requestDescriptor = await putPreparedBody(this.files, requestFileKey, record.request.body);
+ }
+ if (record.response.body.type === 'bytes') {
+ if (record.response.body.body.byteLength > 0) {
+ responseDescriptor = await putRawBody(this.files, responseFileKey!, record.response.body.body, 'bytes');
+ }
+ } else if (record.response.body.type === 'stream') {
+ responseDescriptor = await putRawBody(
+ this.files,
+ responseFileKey!,
+ new TextEncoder().encode(encodeDumpStreamEvents(record.response.body.events, `dump record ${record.meta.id} response events`)),
+ 'events',
+ );
}
- } else if (record.response.body.type === 'stream') {
- responseDescriptor = await putRawBody(
- this.files,
- responseFileKey!,
- new TextEncoder().encode(encodeDumpStreamEvents(record.response.body.events, `dump record ${record.meta.id} response events`)),
- 'events',
- );
}
// Files before row — a partial failure leaves orphan files the sweep
@@ -172,8 +189,10 @@ export class FileDumpStore implements DumpStore {
record.meta.completedAt,
record.meta.upstream?.id ?? null,
encodePersistedDumpMetadata(record.meta, `dump record ${record.meta.id} metadata`),
- encodeDumpHeaders(record.request.headers, `dump record ${record.meta.id} request headers`),
- record.response.body.type === 'none'
+ record.shape === 'run'
+ ? NO_EDGE_HEADERS
+ : encodeDumpHeaders(record.request.headers, `dump record ${record.meta.id} request headers`),
+ record.shape === 'run' || record.response.body.type === 'none'
? null
: encodeDumpHeaders(record.response.headers, `dump record ${record.meta.id} response headers`),
requestDescriptor === null
@@ -231,17 +250,24 @@ export class FileDumpStore implements DumpStore {
...decodePersistedDumpMetadata(row.meta_json, `dump record ${recordId} metadata`),
upstream: hydrateUpstream(row),
};
- const requestHeaders = decodeDumpHeaders(row.request_headers_json, `dump record ${recordId} request headers`);
const requestDescriptor = row.request_body_descriptor === null
? null
: decodeDumpBodyDescriptor(row.request_body_descriptor, `dump record ${recordId} request body descriptor`);
- const responseHeaders = row.response_headers_json === null
- ? null
- : decodeDumpHeaders(row.response_headers_json, `dump record ${recordId} response headers`);
const responseDescriptor = row.response_body_descriptor === null
? null
: decodeDumpBodyDescriptor(row.response_body_descriptor, `dump record ${recordId} response body descriptor`);
+ // The body kind is the shape: a run was written as one NDJSON stream and
+ // has no edge halves to rebuild.
+ if (responseDescriptor?.type === 'run') {
+ return { shape: 'run', meta, events: await fetchBody(this.files, responseDescriptor) };
+ }
+
+ const requestHeaders = decodeDumpHeaders(row.request_headers_json, `dump record ${recordId} request headers`);
+ const responseHeaders = row.response_headers_json === null
+ ? null
+ : decodeDumpHeaders(row.response_headers_json, `dump record ${recordId} response headers`);
+
const request: StoredDumpRequest = {
method: meta.method,
path: meta.path,
@@ -272,7 +298,7 @@ export class FileDumpStore implements DumpStore {
headers: responseHeaders ?? [],
body: responseBody,
};
- return { meta, request, response };
+ return { shape: 'edge', meta, request, response };
}
async deleteExpiredBatch(keyId: string, now: number, limit: number): Promise {
diff --git a/packages/gateway/src/runtime/log.ts b/packages/gateway/src/runtime/log.ts
new file mode 100644
index 0000000000..7805ccbd64
--- /dev/null
+++ b/packages/gateway/src/runtime/log.ts
@@ -0,0 +1,17 @@
+// The global log sink. A stage's logger writes to the stage's dump record when one is open
+// and to this sink always — so a run nobody is recording still reports what went wrong.
+//
+// The gateway writes to the console and has no level configuration, so the threshold is
+// fixed here rather than read from one. Warnings and errors are what a run produces that an
+// operator has to see; `debug` and `info` describe one request's progress, and at a line per
+// stage per request they would bury the request log this sits alongside. They are still
+// recorded in full whenever a dump is open, which is when anyone is reading them.
+
+import type { Logger } from '@floway-dev/pipeline';
+
+export const consoleLogSink: Logger = {
+ debug: () => {},
+ info: () => {},
+ warn: (message, fields) => { console.warn(message, fields ?? {}); },
+ error: (message, fields) => { console.error(message, fields ?? {}); },
+};
diff --git a/packages/pipeline/src/run.ts b/packages/pipeline/src/run.ts
index cca2116ec5..2b2a5ef0ea 100644
--- a/packages/pipeline/src/run.ts
+++ b/packages/pipeline/src/run.ts
@@ -52,8 +52,12 @@ export const isOwned = (value: unknown): value is Owned =>
const DEFERRED = Symbol('floway.deferred');
/** A value the run has started and has not finished. It is a property of the value, not a
- * capability a stage was handed: what a run must wait for is legible from its own record. */
-export type Deferred = PromiseLike & { readonly [DEFERRED]: true };
+ * capability a stage was handed: what a run must wait for is legible from its own record.
+ *
+ * A `Promise` rather than a `PromiseLike`, because branding one is all `defer` does — what
+ * comes back is the same object, and narrowing it to the smaller interface would make a
+ * caller reach for `Promise.resolve` to get back what it already had. */
+export type Deferred = Promise & { readonly [DEFERRED]: true };
/**
* Marks a promise as this run's to finish.
@@ -66,7 +70,7 @@ export type Deferred = PromiseLike & { readonly [DEFERRED]: true };
* "stages have no fire-and-forget" means in practice: work a stage starts becomes a fact
* with a name, and the runner waits for it where it can see it.
*/
-export const defer = (promise: PromiseLike): Deferred =>
+export const defer = (promise: Promise): Deferred =>
Object.assign(promise, { [DEFERRED]: true as const });
export const isDeferred = (value: unknown): value is Deferred =>
diff --git a/packages/protocols/__tests__/openai-audio/transcription_test.ts b/packages/protocols/__tests__/openai-audio/transcription_test.ts
new file mode 100644
index 0000000000..a7d47725b9
--- /dev/null
+++ b/packages/protocols/__tests__/openai-audio/transcription_test.ts
@@ -0,0 +1,146 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+ parseOpenAIAudioTranscription,
+ parseOpenAIAudioTranscriptionResponseFormat,
+ parseOpenAIAudioTranscriptionUsage,
+ renderOpenAIAudioTranscription,
+ type OpenAIAudioTranscriptionResponseFormat,
+} from '../../src/openai-audio/index.ts';
+
+const bytes = (document: string): Uint8Array => new TextEncoder().encode(document);
+
+/** What a carried rendering says, and a failure where an object rendering came back
+ * instead — the two are never interchangeable and a test that read one as the other would
+ * pass on the wrong thing. */
+const carried = (rendered: Record | Uint8Array): string => {
+ if (!(rendered instanceof Uint8Array)) throw new Error(`expected a carried document, got ${JSON.stringify(rendered)}`);
+ return new TextDecoder().decode(rendered);
+};
+
+// Whisper's own writers, which is what `whisper-1` runs: SubRip numbers its cues and always
+// writes the hour component, WebVTT opens with its signature and drops the hour component
+// below one hour, and both terminate every cue with a blank line.
+// https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L238-L262
+const SRT = '1\n00:00:00,000 --> 00:00:03,320\n The beach was a popular spot.\n\n'
+ + '2\n00:00:03,320 --> 00:00:08,470\n People were swimming in the ocean.\n\n';
+const VTT = 'WEBVTT\n\n00:00.000 --> 00:03.320\n The beach was a popular spot.\n\n'
+ + '00:03.320 --> 00:08.470\n People were swimming in the ocean.\n\n';
+
+const VERBOSE = {
+ task: 'transcribe',
+ language: 'english',
+ duration: 8.47,
+ text: ' The beach was a popular spot. People were swimming in the ocean.',
+ segments: [
+ { id: 0, seek: 0, start: 0, end: 3.32, text: ' The beach was a popular spot.', tokens: [50364], temperature: 0, avg_logprob: -0.28, compression_ratio: 1.23, no_speech_prob: 0.009 },
+ ],
+};
+
+const carries = (format: OpenAIAudioTranscriptionResponseFormat, document: string): string =>
+ carried(renderOpenAIAudioTranscription(format, parseOpenAIAudioTranscription(format, bytes(document))));
+
+describe('the renderings of a transcription', () => {
+ it('is the protocol default when the form left the field out', () => {
+ expect(parseOpenAIAudioTranscriptionResponseFormat(undefined)).toBe('json');
+ expect(() => parseOpenAIAudioTranscriptionResponseFormat('yaml')).toThrow('response_format is invalid');
+ });
+
+ // The claim the family rests on: `response_format` travels to the upstream inside the
+ // form, so the rendering an answer arrives in is the rendering it is written back in, and
+ // reading then writing has to be the identity on every one of the six.
+ it.each([
+ ['json', JSON.stringify({ text: 'hello' })],
+ ['verbose_json', JSON.stringify(VERBOSE)],
+ ['diarized_json', JSON.stringify({ task: 'transcribe', duration: 2, text: 'Agent: hi', segments: [{ type: 'transcript.text.segment', id: 'seg_1', start: 0, end: 2, text: 'hi', speaker: 'agent' }] })],
+ ['text', ' The beach was a popular spot.\n'],
+ ['srt', SRT],
+ ['vtt', VTT],
+ ] as const)('reads and writes %s back unchanged', (format, document) => {
+ const rendered = renderOpenAIAudioTranscription(format, parseOpenAIAudioTranscription(format, bytes(document)));
+ expect(rendered instanceof Uint8Array ? carried(rendered) : JSON.stringify(rendered)).toBe(document);
+ });
+
+ // A document is carried and not reproduced, so what the upstream wrote survives every
+ // convention it did not follow. Each of these is a byte a writer of ours would have
+ // changed: the terminating blank line Whisper always writes, the ordinal it always starts
+ // at 1, the line ending it always writes as `\n`, and the millisecond precision it always
+ // pads to three digits.
+ it.each([
+ ['srt', '1\n00:00:00,000 --> 00:00:03,320\nno terminating blank line'],
+ ['srt', '7\r\n00:00:00,000 --> 00:00:03,320\r\nnumbered from seven, with CRLF\r\n\r\n'],
+ ['vtt', 'WEBVTT\n\n00:00.0 --> 00:03.32\nfewer fraction digits than Whisper writes\n\n'],
+ ['text', 'a transcript with no trailing newline'],
+ ] as const)('carries a %s document byte for byte, however it was written', (format, document) => {
+ expect(carries(format, document)).toBe(document);
+ });
+
+ // What the canonical value holds is what its rendering carried, and the renderings are not
+ // equally rich. This is the whole of why a rendering is never derived from a value read
+ // out of a different one.
+ it('holds a transcript and its cues where the rendering carried both', () => {
+ expect(parseOpenAIAudioTranscription('verbose_json', bytes(JSON.stringify(VERBOSE)))).toMatchObject({
+ text: VERBOSE.text,
+ cues: [{ start: 0, end: 3.32, text: ' The beach was a popular spot.' }],
+ });
+ });
+
+ it('holds no cues where the rendering carried none', () => {
+ expect(parseOpenAIAudioTranscription('json', bytes(JSON.stringify({ text: 'hello' }))).cues).toBeUndefined();
+ });
+
+ // A subtitle document states the transcript only as its cues' texts, so the transcript on
+ // the canonical value is rebuilt the way Whisper's own text writer rebuilds it and is not
+ // claimed to be the string a `text` request would have returned.
+ // https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L109-L116
+ it('rebuilds the transcript of a subtitle document from its cues', () => {
+ expect(parseOpenAIAudioTranscription('srt', bytes(SRT)).text).toBe(' The beach was a popular spot.\n People were swimming in the ocean.');
+ });
+
+ it('refuses a body the rendering cannot be read from', () => {
+ expect(() => parseOpenAIAudioTranscription('json', bytes('{not-json'))).toThrow(SyntaxError);
+ expect(() => parseOpenAIAudioTranscription('json', bytes(JSON.stringify({ transcript: 'hello' })))).toThrow('text must be a string');
+ expect(() => parseOpenAIAudioTranscription('vtt', bytes(SRT))).toThrow('must open with WEBVTT');
+ });
+
+ // The other half of that refusal: a caller that could not read the body still has the body,
+ // and what it writes back is what arrived. A 2xx nobody could parse is still the upstream's
+ // answer to the client's request.
+ it('writes back a body no reading could open', () => {
+ expect(carried(renderOpenAIAudioTranscription('json', { document: bytes('{not-json') }))).toBe('{not-json');
+ });
+});
+
+describe('what a transcription says it will be billed for', () => {
+ it('splits the audio share out of the input tokens the upstream broke down', () => {
+ expect(parseOpenAIAudioTranscriptionUsage({
+ usage: { type: 'tokens', input_tokens: 14, input_token_details: { text_tokens: 10, audio_tokens: 4 }, output_tokens: 101, total_tokens: 115 },
+ })).toEqual({ kind: 'tokens', inputTokens: 14, inputAudioTokens: 4, outputTokens: 101 });
+ });
+
+ it('reads a duration report, and whisper\'s top-level duration where there is no report', () => {
+ expect(parseOpenAIAudioTranscriptionUsage({ usage: { type: 'duration', seconds: 43 } })).toEqual({ kind: 'duration', seconds: 43 });
+ expect(parseOpenAIAudioTranscriptionUsage(VERBOSE)).toEqual({ kind: 'duration', seconds: 8.47 });
+ });
+
+ // Nothing was stated, or what was stated is not something this reading names. A metric
+ // invented after this was written is not a malformed one, and the caller bills neither.
+ it('is no report where nothing this reading names was stated', () => {
+ expect(parseOpenAIAudioTranscriptionUsage({ text: 'hello' })).toBeUndefined();
+ expect(parseOpenAIAudioTranscriptionUsage({ usage: { type: 'credits', spent: 3 } })).toBeUndefined();
+ });
+
+ // An upstream that reported under a name this reading does know, in a shape it cannot read,
+ // is a third situation and it says so — which is what lets the caller warn rather than
+ // record the request as though the upstream had metered nothing.
+ it('reports a block it names and cannot read, rather than reading it as nothing', () => {
+ expect(() => parseOpenAIAudioTranscriptionUsage({ usage: { type: 'duration', seconds: 'invalid' } }))
+ .toThrow('usage.seconds must be a finite non-negative number');
+ expect(() => parseOpenAIAudioTranscriptionUsage({ usage: { type: 'tokens', input_tokens: -1, output_tokens: 2 } }))
+ .toThrow('usage.input_tokens must be a non-negative safe integer');
+ expect(() => parseOpenAIAudioTranscriptionUsage({ usage: { type: 'tokens', input_tokens: 4, input_token_details: { audio_tokens: 9 }, output_tokens: 2 } }))
+ .toThrow('audio_tokens must not exceed usage.input_tokens');
+ expect(() => parseOpenAIAudioTranscriptionUsage({ usage: 'billed' })).toThrow('usage must be an object');
+ expect(() => parseOpenAIAudioTranscriptionUsage({ duration: 'a while' })).toThrow('duration must be a finite non-negative number');
+ });
+});
diff --git a/packages/protocols/__tests__/openai-embeddings/translate_test.ts b/packages/protocols/__tests__/openai-embeddings/translate_test.ts
new file mode 100644
index 0000000000..ac8ca9b273
--- /dev/null
+++ b/packages/protocols/__tests__/openai-embeddings/translate_test.ts
@@ -0,0 +1,169 @@
+import { describe, expect, test } from 'vitest';
+
+import {
+ parseOpenAIEmbeddingsRequest,
+ parseOpenAIEmbeddingsResponse,
+ renderOpenAIEmbeddingsResponse,
+ serializeOpenAIEmbeddingsRequest,
+} from '../../src/openai-embeddings/translate.ts';
+
+// The float32 little-endian packing of [0.0023064255, -0.009327292, 1, -0.0028842222],
+// and the float64 values those bits denote. Produced by Float32Array + Buffer on Node 24,
+// which is the encoding both official OpenAI SDKs read a base64 embedding with.
+const PACKED = 'ZicXO4DRGLwAAIA/OAU9uw==';
+const VECTOR = [0.002306425478309393, -0.009327292442321777, 1, -0.0028842221945524216];
+
+describe('OpenAI Embeddings request ingress', () => {
+ test('a single text stays a single text, so the wire shape the upstream sees is unchanged', () => {
+ const parsed = parseOpenAIEmbeddingsRequest({ model: 'text-embedding-3-small', input: 'a sentence' });
+
+ expect(parsed.model).toBe('text-embedding-3-small');
+ expect(parsed.request).toEqual({ input: 'a sentence' });
+ expect(serializeOpenAIEmbeddingsRequest(parsed.request)).toEqual({ input: 'a sentence' });
+ });
+
+ test('the four input arms each survive', () => {
+ const arms = [
+ 'one',
+ ['one', 'two'],
+ [1212, 318, 257],
+ [[1212, 318], [257, 1332]],
+ ];
+ for (const input of arms) {
+ expect(parseOpenAIEmbeddingsRequest({ model: 'm', input }).request.input).toEqual(input);
+ }
+ });
+
+ test('an omitted encoding_format is resolved for the client and left off the wire', () => {
+ const parsed = parseOpenAIEmbeddingsRequest({ model: 'm', input: 'a' });
+
+ expect(parsed.encodingFormat).toBe('float');
+ expect(parsed.request.encodingFormat).toBeUndefined();
+ expect(serializeOpenAIEmbeddingsRequest(parsed.request)).not.toHaveProperty('encoding_format');
+ });
+
+ test('a chosen encoding_format is both what the client reads and what the upstream is asked for', () => {
+ const parsed = parseOpenAIEmbeddingsRequest({ model: 'm', input: 'a', encoding_format: 'base64', dimensions: 256, user: 'u' });
+
+ expect(parsed.encodingFormat).toBe('base64');
+ expect(serializeOpenAIEmbeddingsRequest(parsed.request)).toEqual({
+ input: 'a',
+ encoding_format: 'base64',
+ dimensions: 256,
+ user: 'u',
+ });
+ });
+
+ test('a field the protocol has no place for is named rather than dropped', () => {
+ expect(() => parseOpenAIEmbeddingsRequest({ model: 'm', input: 'a', truncate: true, task: 'retrieval' }))
+ .toThrow('OpenAI Embeddings does not support truncate, task');
+ });
+
+ // An empty array satisfies three of the specification's four input arms at once, so the
+ // canonical value would be ambiguous even though the JSON is well-formed.
+ test('an empty input is refused, which is what keeps the parsed arms distinguishable', () => {
+ expect(() => parseOpenAIEmbeddingsRequest({ model: 'm', input: [] })).toThrow('input must not be empty');
+ });
+
+ test('a mixed array is refused rather than read as whichever arm came first', () => {
+ expect(() => parseOpenAIEmbeddingsRequest({ model: 'm', input: ['a', 1] })).toThrow('input[1] must be a non-empty string');
+ expect(() => parseOpenAIEmbeddingsRequest({ model: 'm', input: [1, 'a'] })).toThrow('input[1] must be an integer');
+ });
+
+ test('a non-integer dimensions is refused', () => {
+ expect(() => parseOpenAIEmbeddingsRequest({ model: 'm', input: 'a', dimensions: 0 })).toThrow('dimensions must be a positive integer');
+ expect(() => parseOpenAIEmbeddingsRequest({ model: 'm', input: 'a', dimensions: 1.5 })).toThrow('dimensions must be an integer');
+ });
+});
+
+describe('OpenAI Embeddings response egress', () => {
+ test('a base64 vector is read as the numbers it denotes', () => {
+ const parsed = parseOpenAIEmbeddingsResponse({
+ object: 'list',
+ model: 'text-embedding-3-small',
+ data: [{ object: 'embedding', index: 0, embedding: PACKED }],
+ usage: { prompt_tokens: 8, total_tokens: 8 },
+ }, 'm');
+
+ expect(parsed.embeddings).toEqual([{ index: 0, values: VECTOR }]);
+ expect(parsed.usage).toEqual({ promptTokens: 8, totalTokens: 8 });
+ });
+
+ test('a truncated base64 vector is refused rather than read as a shorter one', () => {
+ expect(() => parseOpenAIEmbeddingsResponse({
+ model: 'm',
+ data: [{ index: 0, embedding: PACKED.slice(0, 8) }],
+ }, 'm')).toThrow('data[0].embedding is not a whole number of float32 values');
+ });
+
+ // The schema marks `model` required and most upstreams send it; Copilot's `/embeddings`
+ // does not. An answer the gateway understands is one it can write again, so the record is
+ // completed with the model the request named rather than refused.
+ test('an upstream that names no model is read as having answered for the requested one', () => {
+ const parsed = parseOpenAIEmbeddingsResponse(
+ { object: 'list', data: [{ object: 'embedding', index: 0, embedding: [0.5] }] },
+ 'text-embedding-real',
+ );
+
+ expect(parsed.model).toBe('text-embedding-real');
+ expect(renderOpenAIEmbeddingsResponse('float', parsed)).toMatchObject({ model: 'text-embedding-real' });
+ });
+
+ // A model the upstream *did* name is its own answer, and the request's id never overrides it.
+ test('a model the upstream named is kept', () => {
+ expect(parseOpenAIEmbeddingsResponse({ model: 'upstream-id', data: [] }, 'requested-id').model).toBe('upstream-id');
+ });
+
+ // An upstream that reports no usage is a fact the gateway has to be able to carry: the
+ // billed entity is present with no quantities, which is not the same as reporting zero.
+ test('a missing usage block leaves the answer intact and reports nothing', () => {
+ const parsed = parseOpenAIEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: [0.5] }] }, 'm');
+
+ expect(parsed.usage).toBeUndefined();
+ expect(renderOpenAIEmbeddingsResponse('float', parsed)).not.toHaveProperty('usage');
+ });
+
+ // The case the encoding split exists for. An OpenAI SDK client asked for base64 without
+ // saying so and will decode whatever it is handed; an upstream that ignores the field
+ // and answers with float arrays would otherwise hand it noise.
+ test('the answer is written in the encoding the client asked for, not the one it arrived in', () => {
+ const arrived = parseOpenAIEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: VECTOR }] }, 'm');
+
+ expect(renderOpenAIEmbeddingsResponse('base64', arrived)).toMatchObject({
+ object: 'list',
+ data: [{ object: 'embedding', index: 0, embedding: PACKED }],
+ });
+ expect(renderOpenAIEmbeddingsResponse('float', parseOpenAIEmbeddingsResponse({
+ model: 'm',
+ data: [{ index: 0, embedding: PACKED }],
+ }, 'm'))).toMatchObject({ data: [{ embedding: VECTOR }] });
+ });
+
+ // Every float32 is a float64, so widening loses nothing and narrowing a value that came
+ // from a float32 gives the same bits back. Holding vectors as numbers is therefore exact
+ // and not an approximation, however many times a vector crosses the gateway.
+ test('a vector survives any number of trips through the two encodings', () => {
+ const once = parseOpenAIEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: PACKED }] }, 'm');
+ const twice = parseOpenAIEmbeddingsResponse(renderOpenAIEmbeddingsResponse('base64', once), 'm');
+ const thrice = parseOpenAIEmbeddingsResponse(renderOpenAIEmbeddingsResponse('base64', twice), 'm');
+
+ expect(renderOpenAIEmbeddingsResponse('base64', thrice)).toMatchObject({ data: [{ embedding: PACKED }] });
+ expect(thrice.embeddings).toEqual(once.embeddings);
+ });
+
+ test('object is written by the gateway at both levels rather than repeated back', () => {
+ const rendered = renderOpenAIEmbeddingsResponse('float', parseOpenAIEmbeddingsResponse({
+ object: 'not-a-list',
+ model: 'm',
+ data: [{ object: 'not-an-embedding', index: 0, embedding: [0.5] }],
+ usage: { prompt_tokens: 1, total_tokens: 1 },
+ }, 'm'));
+
+ expect(rendered).toEqual({
+ object: 'list',
+ data: [{ object: 'embedding', index: 0, embedding: [0.5] }],
+ model: 'm',
+ usage: { prompt_tokens: 1, total_tokens: 1 },
+ });
+ });
+});
diff --git a/packages/protocols/__tests__/openai-images/request_test.ts b/packages/protocols/__tests__/openai-images/request_test.ts
new file mode 100644
index 0000000000..eefcc3d2da
--- /dev/null
+++ b/packages/protocols/__tests__/openai-images/request_test.ts
@@ -0,0 +1,129 @@
+import { describe, expect, test } from 'vitest';
+
+import { parseOpenAIImagesEditsRequest, parseOpenAIImagesGenerationsRequest } from '../../src/openai-images/request.ts';
+
+const json = (value: unknown): Uint8Array => new TextEncoder().encode(JSON.stringify(value));
+
+const multipart = async (form: FormData): Promise<{ contentType: string; bytes: Uint8Array }> => {
+ const encoded = new Response(form);
+ return {
+ contentType: encoded.headers.get('content-type')!,
+ bytes: new Uint8Array(await encoded.arrayBuffer()),
+ };
+};
+
+describe('OpenAI Images Generations ingress', () => {
+ test('routing takes the model and everything else stays as the client wrote it', () => {
+ const parsed = parseOpenAIImagesGenerationsRequest(json({
+ model: 'gpt-image-1',
+ prompt: 'a shiba in space',
+ size: '1024x1024',
+ moderation: 'low',
+ }));
+
+ expect(parsed.model).toBe('gpt-image-1');
+ expect(parsed.request).toEqual({
+ operation: 'generations',
+ parameters: { prompt: 'a shiba in space', size: '1024x1024', moderation: 'low' },
+ });
+ });
+
+ test('a body that is not a JSON object, or names no model, is refused with what to fix', () => {
+ expect(() => parseOpenAIImagesGenerationsRequest(new TextEncoder().encode('not json')))
+ .toThrow('OpenAI Images Generations request body must be valid JSON.');
+ expect(() => parseOpenAIImagesGenerationsRequest(json(['gpt-image-1'])))
+ .toThrow('OpenAI Images Generations request body must be an object.');
+ expect(() => parseOpenAIImagesGenerationsRequest(json({ prompt: 'hi' })))
+ .toThrow('OpenAI Images Generations request body must include a model string.');
+ });
+});
+
+describe('OpenAI Images Edits ingress', () => {
+ test('a JSON edit keeps each reference exactly as it arrived', async () => {
+ const parsed = await parseOpenAIImagesEditsRequest('application/json', json({
+ model: 'gpt-image-1',
+ prompt: 'replace the background',
+ images: [{ image_url: 'data:image/png;base64,iVBORw0KGgo=' }, { file_id: 'file-source' }],
+ mask: { file_id: 'file-mask' },
+ quality: 'high',
+ }));
+
+ expect(parsed.model).toBe('gpt-image-1');
+ expect(parsed.request).toEqual({
+ operation: 'edits',
+ images: [
+ { kind: 'reference', reference: { image_url: 'data:image/png;base64,iVBORw0KGgo=' } },
+ { kind: 'reference', reference: { file_id: 'file-source' } },
+ ],
+ mask: { kind: 'reference', reference: { file_id: 'file-mask' } },
+ parameters: { prompt: 'replace the background', quality: 'high' },
+ });
+ });
+
+ test('a reference naming both ways, or neither, is refused by position', async () => {
+ await expect(parseOpenAIImagesEditsRequest('application/json', json({
+ model: 'gpt-image-1',
+ images: [{ file_id: 'file-source' }, { image_url: 'https://example.com/a.png', file_id: 'file-source' }],
+ }))).rejects.toThrow('OpenAI Images Edits images[1] must contain exactly one string field: image_url or file_id.');
+
+ await expect(parseOpenAIImagesEditsRequest('application/json', json({ model: 'gpt-image-1', prompt: 'hi' })))
+ .rejects.toThrow('OpenAI Images Edits request body must include an images array.');
+ });
+
+ test('a multipart edit holds each file as bytes and every other field as text', async () => {
+ const form = new FormData();
+ form.append('model', 'gpt-image-1');
+ form.append('prompt', 'replace the sky');
+ form.append('n', '2');
+ form.append('image[]', new Blob([new Uint8Array([1, 2, 3])], { type: 'image/png' }), 'photo.png');
+ form.append('image[]', new Blob([new Uint8Array([4, 5])], { type: 'image/webp' }), 'second.webp');
+ form.append('mask', new Blob([new Uint8Array([6])], { type: 'image/png' }), 'mask.png');
+ const { contentType, bytes } = await multipart(form);
+
+ const parsed = await parseOpenAIImagesEditsRequest(contentType, bytes);
+
+ expect(parsed.model).toBe('gpt-image-1');
+ expect(parsed.request).toEqual({
+ operation: 'edits',
+ images: [
+ { kind: 'file', file: { fileName: 'photo.png', mediaType: 'image/png', bytes: new Uint8Array([1, 2, 3]) } },
+ { kind: 'file', file: { fileName: 'second.webp', mediaType: 'image/webp', bytes: new Uint8Array([4, 5]) } },
+ ],
+ mask: { kind: 'file', file: { fileName: 'mask.png', mediaType: 'image/png', bytes: new Uint8Array([6]) } },
+ // A form field is text, so `n` stays the string the client sent it as.
+ parameters: { prompt: 'replace the sky', n: '2' },
+ });
+ });
+
+ test('a multipart edit naming no model is refused before its fields are read', async () => {
+ const form = new FormData();
+ form.append('prompt', 'replace the sky');
+ const { contentType, bytes } = await multipart(form);
+
+ await expect(parseOpenAIImagesEditsRequest(contentType, bytes))
+ .rejects.toThrow('OpenAI Images Edits request body must include a model field.');
+ });
+
+ test('an image field carrying text, and a text field carrying a file, are both refused', async () => {
+ const textImage = new FormData();
+ textImage.append('model', 'gpt-image-1');
+ textImage.append('image', 'not a file');
+ const encodedTextImage = await multipart(textImage);
+ await expect(parseOpenAIImagesEditsRequest(encodedTextImage.contentType, encodedTextImage.bytes))
+ .rejects.toThrow('OpenAI Images Edits image fields must be files.');
+
+ const fileParameter = new FormData();
+ fileParameter.append('model', 'gpt-image-1');
+ fileParameter.append('prompt', new Blob([new Uint8Array([1])], { type: 'text/plain' }), 'prompt.txt');
+ const encodedFileParameter = await multipart(fileParameter);
+ await expect(parseOpenAIImagesEditsRequest(encodedFileParameter.contentType, encodedFileParameter.bytes))
+ .rejects.toThrow('OpenAI Images Edits prompt field must be text.');
+ });
+
+ test('a body in neither of the two media types the endpoint takes is refused as such', async () => {
+ await expect(parseOpenAIImagesEditsRequest('text/plain', new TextEncoder().encode('hi')))
+ .rejects.toThrow('OpenAI Images Edits request body must use application/json or multipart/form-data.');
+ await expect(parseOpenAIImagesEditsRequest(undefined, new TextEncoder().encode('hi')))
+ .rejects.toThrow('OpenAI Images Edits request body must use application/json or multipart/form-data.');
+ });
+});
diff --git a/packages/protocols/__tests__/openai-images/response_test.ts b/packages/protocols/__tests__/openai-images/response_test.ts
new file mode 100644
index 0000000000..d1b590b086
--- /dev/null
+++ b/packages/protocols/__tests__/openai-images/response_test.ts
@@ -0,0 +1,76 @@
+import { describe, expect, test } from 'vitest';
+
+import { renderErrorEnvelope, upstreamErrorMessage } from '../../src/common/error-envelope.ts';
+import { parseOpenAIImagesResponse, parseOpenAIImagesUsage, renderOpenAIImagesResponse } from '../../src/openai-images/response.ts';
+
+describe('OpenAI Images response ingress', () => {
+ test('the answer is parsed and rendered back whole, fields the gateway does not model included', () => {
+ const body = {
+ created: 1713833628,
+ data: [{ b64_json: 'aGVsbG8=', revised_prompt: 'a shiba, in space' }, { url: 'https://example.com/a.png' }],
+ background: 'transparent',
+ output_format: 'png',
+ };
+
+ const parsed = parseOpenAIImagesResponse(body);
+
+ expect(parsed.images).toEqual([
+ { base64: 'aGVsbG8=', revisedPrompt: 'a shiba, in space' },
+ { url: 'https://example.com/a.png' },
+ ]);
+ expect(renderOpenAIImagesResponse(parsed)).toBe(body);
+ });
+
+ test('an answer carrying no images at all is read as such, and one that cannot be read is an error', () => {
+ expect(parseOpenAIImagesResponse({ created: 1 }).images).toEqual([]);
+ expect(() => parseOpenAIImagesResponse({ data: 'one image' })).toThrow('OpenAI Images response data must be an array');
+ expect(() => parseOpenAIImagesResponse({ data: [{ b64_json: 42 }] })).toThrow('OpenAI Images response data[0].b64_json must be a string');
+ expect(() => parseOpenAIImagesResponse('an image')).toThrow('OpenAI Images response body must be an object');
+ });
+});
+
+describe('OpenAI Images usage', () => {
+ test('what the upstream attributed to images is taken out of the count beside it', () => {
+ expect(parseOpenAIImagesUsage({
+ usage: {
+ total_tokens: 100,
+ input_tokens: 50,
+ output_tokens: 50,
+ input_tokens_details: { text_tokens: 10, image_tokens: 40 },
+ },
+ })).toEqual({ inputTokens: 10, inputImageTokens: 40, outputTokens: 50 });
+ });
+
+ test('a count with no detail beside it is the whole count, and a detail saying nothing is as good as absent', () => {
+ expect(parseOpenAIImagesUsage({ usage: { input_tokens: 10, output_tokens: 50 } }))
+ .toEqual({ inputTokens: 10, outputTokens: 50 });
+ expect(parseOpenAIImagesUsage({ usage: { input_tokens: 10, input_tokens_details: {} } }))
+ .toEqual({ inputTokens: 10 });
+ expect(parseOpenAIImagesUsage({ usage: { output_tokens: 50, output_tokens_details: { image_tokens: 50 } } }))
+ .toEqual({ outputTokens: 0, outputImageTokens: 50 });
+ });
+
+ // The distinction the return type exists for: an upstream that reported nothing is a
+ // different fact from one that reported zero, and only the second is a reading.
+ test('nothing readable is no reading, and a reported zero is one', () => {
+ expect(parseOpenAIImagesUsage({ data: [] })).toBeUndefined();
+ expect(parseOpenAIImagesUsage({ usage: {} })).toBeUndefined();
+ expect(parseOpenAIImagesUsage({ usage: { input_tokens: '50' } })).toBeUndefined();
+ expect(parseOpenAIImagesUsage({ usage: { input_tokens: 10, input_tokens_details: 'text' } })).toBeUndefined();
+ expect(parseOpenAIImagesUsage({ usage: { input_tokens: 0, output_tokens: 0 } }))
+ .toEqual({ inputTokens: 0, outputTokens: 0 });
+ });
+});
+
+describe('OpenAI Images failures', () => {
+ test('an upstream error object is what the client is sent, and anything else becomes the gateway envelope', () => {
+ const upstream = { error: { message: 'Your request was rejected', type: 'image_generation_user_error', code: 'moderation_blocked' } };
+
+ expect(upstreamErrorMessage(upstream)).toBe('Your request was rejected');
+ expect(renderErrorEnvelope('Your request was rejected', upstream)).toBe(upstream);
+
+ expect(upstreamErrorMessage('upstream is down')).toBeUndefined();
+ expect(renderErrorEnvelope('Model gpt-image-1 is not available on any configured upstream.', undefined))
+ .toEqual({ error: { message: 'Model gpt-image-1 is not available on any configured upstream.', type: 'api_error' } });
+ });
+});
diff --git a/packages/protocols/src/common/error-envelope.ts b/packages/protocols/src/common/error-envelope.ts
new file mode 100644
index 0000000000..c92fbd7408
--- /dev/null
+++ b/packages/protocols/src/common/error-envelope.ts
@@ -0,0 +1,21 @@
+// What a client is sent when a run produced no answer, and how an upstream's own words are
+// read out of what it sent.
+//
+// An upstream that refused in its own words is handed on in them — a client reads the code
+// and type inside, and the gateway has nothing truer to say than what the upstream said. This
+// is why the test is whether an upstream answered at all rather than which shape it answered
+// in: an OpenAI family's error is `{error:{...}}` and a rerank client's is `{message}`, and
+// both are already the shape that client reads.
+//
+// A refusal that never reached an upstream has only the gateway's own words, and those go in
+// the envelope every protocol here writes errors in.
+
+import { isJsonObject } from './json.ts';
+
+export const upstreamErrorMessage = (body: unknown): string | undefined => {
+ if (!isJsonObject(body) || !isJsonObject(body.error)) return undefined;
+ return typeof body.error.message === 'string' ? body.error.message : undefined;
+};
+
+export const renderErrorEnvelope = (message: string, upstreamBody?: unknown): Record =>
+ isJsonObject(upstreamBody) ? upstreamBody : { error: { message, type: 'api_error' } };
diff --git a/packages/protocols/src/common/index.ts b/packages/protocols/src/common/index.ts
index 445cd1220e..2b7a1293d2 100644
--- a/packages/protocols/src/common/index.ts
+++ b/packages/protocols/src/common/index.ts
@@ -13,5 +13,6 @@ export * from './sse.ts';
export * from './parse-sse.ts';
export * from './parse-events.ts';
+export { renderErrorEnvelope, upstreamErrorMessage } from './error-envelope.ts';
export { isJsonObject, type JsonObject } from './json.ts';
export { captureExtras } from './reassemble-extras.ts';
diff --git a/packages/protocols/src/openai-audio/index.ts b/packages/protocols/src/openai-audio/index.ts
index 8be6a3e6a9..d712dd51ab 100644
--- a/packages/protocols/src/openai-audio/index.ts
+++ b/packages/protocols/src/openai-audio/index.ts
@@ -1,20 +1,29 @@
-// OpenAI-compatible audio transcription stream terminal. The wire remains open
-// so provider additions pass through unchanged while the gateway observes only
-// the terminal event it needs.
-// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L61780-L61924
+// OpenAI-compatible audio transcription. One request shape — a multipart form — and
+// several response renderings, which is the family's whole character: `response_format`
+// picks between three JSON objects and three text documents, and `stream` picks a sequence
+// of events instead of any of them.
-export interface OpenAIAudioTranscriptionStreamEvent {
- type: string;
- [key: string]: unknown;
-}
+export type { OpenAIAudioTranscriptionCue, SubtitleDialect } from './subtitles.ts';
+export { parseSubtitleDocument } from './subtitles.ts';
-export interface OpenAIAudioTranscriptionDoneEvent extends OpenAIAudioTranscriptionStreamEvent {
- type: 'transcript.text.done';
- text: string;
-}
+export type {
+ OpenAIAudioTranscriptionObjectFormat,
+ OpenAIAudioTranscriptionResponseFormat,
+ OpenAIAudioTranscriptionUsage,
+ CanonicalOpenAIAudioTranscription,
+} from './transcription.ts';
+export {
+ OPENAI_AUDIO_TRANSCRIPTION_RESPONSE_FORMATS,
+ isOpenAIAudioTranscriptionObjectFormat,
+ parseOpenAIAudioTranscription,
+ parseOpenAIAudioTranscriptionResponseFormat,
+ parseOpenAIAudioTranscriptionUsage,
+ renderOpenAIAudioTranscription,
+} from './transcription.ts';
-export const isOpenAIAudioTranscriptionDoneEvent = (event: unknown): event is OpenAIAudioTranscriptionDoneEvent =>
- typeof event === 'object'
- && event !== null
- && (event as { type?: unknown }).type === 'transcript.text.done'
- && typeof (event as { text?: unknown }).text === 'string';
+export type { OpenAIAudioTranscriptionDoneEvent, OpenAIAudioTranscriptionStreamEvent } from './stream.ts';
+export {
+ isOpenAIAudioTranscriptionDoneEvent,
+ parseOpenAIAudioTranscriptionStreamEvent,
+ parseOpenAIAudioTranscriptionStreamUsage,
+} from './stream.ts';
diff --git a/packages/protocols/src/openai-audio/stream.ts b/packages/protocols/src/openai-audio/stream.ts
new file mode 100644
index 0000000000..94e90ff10f
--- /dev/null
+++ b/packages/protocols/src/openai-audio/stream.ts
@@ -0,0 +1,39 @@
+// The event stream a transcription answers with when the client asked for one. The stream
+// stays open: a provider addition rides through untouched while the gateway names only the
+// events it acts on. `transcript.text.delta` is the increment, `transcript.text.done` is
+// the terminal one and the only place a streamed transcription states its usage.
+// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L61796-L61917
+
+import type { OpenAIAudioTranscriptionUsage } from './transcription.ts';
+import { parseOpenAIAudioTranscriptionUsage } from './transcription.ts';
+
+export interface OpenAIAudioTranscriptionStreamEvent {
+ type: string;
+ [key: string]: unknown;
+}
+
+export interface OpenAIAudioTranscriptionDoneEvent extends OpenAIAudioTranscriptionStreamEvent {
+ type: 'transcript.text.done';
+ text: string;
+}
+
+export const isOpenAIAudioTranscriptionDoneEvent = (event: unknown): event is OpenAIAudioTranscriptionDoneEvent =>
+ typeof event === 'object'
+ && event !== null
+ && (event as { type?: unknown }).type === 'transcript.text.done'
+ && typeof (event as { text?: unknown }).text === 'string';
+
+/** The reading every frame goes through, so the value at the canonical key is a parsed
+ * event and the edge re-serializes it rather than relaying the bytes it arrived in. */
+export const parseOpenAIAudioTranscriptionStreamEvent = (value: unknown): OpenAIAudioTranscriptionStreamEvent => {
+ if (typeof value !== 'object' || value === null || typeof (value as { type?: unknown }).type !== 'string') {
+ throw new Error('OpenAI Audio Transcriptions stream event must be an object carrying a string type');
+ }
+ return value as OpenAIAudioTranscriptionStreamEvent;
+};
+
+/** A streamed transcription states its usage once, in the terminal event, in the same shape
+ * a JSON body states it. */
+export const parseOpenAIAudioTranscriptionStreamUsage = (
+ event: OpenAIAudioTranscriptionDoneEvent,
+): OpenAIAudioTranscriptionUsage | undefined => parseOpenAIAudioTranscriptionUsage(event);
diff --git a/packages/protocols/src/openai-audio/subtitles.ts b/packages/protocols/src/openai-audio/subtitles.ts
new file mode 100644
index 0000000000..1938104150
--- /dev/null
+++ b/packages/protocols/src/openai-audio/subtitles.ts
@@ -0,0 +1,80 @@
+// SubRip and WebVTT, the two renderings of a transcription that are subtitle documents
+// rather than JSON. Both carry exactly one thing — timed cues — so reading one is a total
+// reading of it.
+//
+// There is no writer here, and that is the point: a subtitle rendering is carried to the
+// client as the bytes the upstream sent, so nothing ever has to reproduce one. What the
+// reading is for is the record and the transcript beside it.
+//
+// The shapes are Whisper's own writers, which is what OpenAI's `whisper-1` runs: SubRip
+// numbers its cues from 1, always writes the hour component and separates the milliseconds
+// with a comma; WebVTT opens with a `WEBVTT` line, drops the hour component below one hour
+// and separates the milliseconds with a period. Both terminate every cue with a blank line.
+// Every one of those is a convention rather than a rule, which is why reading is written to
+// accept more than Whisper writes.
+// https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L238-L262
+
+export interface OpenAIAudioTranscriptionCue {
+ /** Seconds from the start of the audio. */
+ readonly start: number;
+ readonly end: number;
+ readonly text: string;
+}
+
+export type SubtitleDialect = 'srt' | 'vtt';
+
+const VTT_HEADER = 'WEBVTT';
+
+const TIMESTAMP = /^(?:(\d+):)?(\d{1,2}):(\d{1,2})[,.](\d{1,3})$/;
+
+const readTimestamp = (value: string, dialect: SubtitleDialect): number => {
+ const parts = TIMESTAMP.exec(value.trim());
+ if (parts === null) throw new Error(`${dialect} timestamp is not readable: ${JSON.stringify(value)}`);
+ const [, hours, minutes, seconds, fraction] = parts;
+ // A fraction shorter than three digits is still a decimal fraction of a second, so it is
+ // padded on the right rather than parsed as a millisecond count.
+ return Number(hours ?? '0') * 3600 + Number(minutes) * 60 + Number(seconds) + Number(fraction.padEnd(3, '0')) / 1000;
+};
+
+const readCueBlock = (block: readonly string[], dialect: SubtitleDialect): OpenAIAudioTranscriptionCue => {
+ // SubRip opens a cue with its ordinal and WebVTT with an optional identifier, so the
+ // timing line is the one holding the arrow and the lines after it are the cue's text.
+ const timingIndex = block.findIndex(line => line.includes('-->'));
+ if (timingIndex < 0) throw new Error(`${dialect} cue has no timing line: ${JSON.stringify(block.join('\n'))}`);
+ const [start, end] = block[timingIndex].split('-->');
+ if (end === undefined) throw new Error(`${dialect} cue timing line has no end: ${JSON.stringify(block[timingIndex])}`);
+ return {
+ start: readTimestamp(start, dialect),
+ end: readTimestamp(end, dialect),
+ text: block.slice(timingIndex + 1).join('\n'),
+ };
+};
+
+export const parseSubtitleDocument = (dialect: SubtitleDialect, document: string): readonly OpenAIAudioTranscriptionCue[] => {
+ const lines = document.replaceAll('\r\n', '\n').split('\n');
+ if (dialect === 'vtt') {
+ // The signature may carry a byte order mark and may be followed by header metadata on
+ // the same line. https://www.w3.org/TR/webvtt1/#webvtt-file-body
+ const signature = lines.shift()?.replace('', '') ?? '';
+ if (!signature.startsWith(VTT_HEADER)) {
+ throw new Error(`WebVTT document must open with ${VTT_HEADER}: ${JSON.stringify(signature)}`);
+ }
+ }
+
+ // A blank line ends a cue in both dialects, which is what makes a cue's own text able to
+ // span several lines.
+ const blocks: string[][] = [];
+ let open: string[] | null = null;
+ for (const line of lines) {
+ if (line.trim().length === 0) {
+ open = null;
+ continue;
+ }
+ if (open === null) {
+ open = [];
+ blocks.push(open);
+ }
+ open.push(line);
+ }
+ return blocks.map(block => readCueBlock(block, dialect));
+};
diff --git a/packages/protocols/src/openai-audio/transcription.ts b/packages/protocols/src/openai-audio/transcription.ts
new file mode 100644
index 0000000000..3b1edf7f96
--- /dev/null
+++ b/packages/protocols/src/openai-audio/transcription.ts
@@ -0,0 +1,207 @@
+// The one transcription, and the several documents that are renderings of it.
+//
+// `response_format` picks a rendering, and it travels to the upstream in the form the client
+// sent, so the rendering the client asked for is the rendering the upstream answers in. That
+// is what makes one canonical value enough: whichever rendering arrived is the one that goes
+// back, and no rendering is ever derived from a value read out of a different one.
+//
+// A document rendering is **carried**, never written again. `text`, `srt` and `vtt` are
+// documents the upstream wrote for the client to read, and re-writing one from what we read
+// out of it changes bytes nobody asked to have changed: a cue writer of our own terminates
+// the last cue with a blank line the upstream may not have written, renumbers a SubRip cue
+// the upstream numbered from something else, and rounds a timestamp to its own precision. So
+// the canonical value holds the bytes that arrived and hands them straight back, and what it
+// reads out of them is what the record shows and what usage is measured from.
+//
+// Which is as well, because the renderings are not equally rich. `verbose_json` carries a
+// whole transcript and its cues; `json` carries the transcript with usage beside it; `srt`
+// and `vtt` carry the cues alone; `text` carries the transcript alone and nothing else at
+// all. Those are properties of the renderings, not gaps in the reading.
+// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L714-L745
+
+import type { OpenAIAudioTranscriptionCue } from './subtitles.ts';
+import { parseSubtitleDocument } from './subtitles.ts';
+
+// `diarized_json` joined the enum with `gpt-4o-transcribe-diarize`; it is a JSON rendering
+// and needs nothing of its own here.
+// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L28527-L28545
+export const OPENAI_AUDIO_TRANSCRIPTION_RESPONSE_FORMATS = ['json', 'text', 'srt', 'verbose_json', 'vtt', 'diarized_json'] as const;
+export type OpenAIAudioTranscriptionResponseFormat = typeof OPENAI_AUDIO_TRANSCRIPTION_RESPONSE_FORMATS[number];
+
+/** The form field is optional and the protocol's own default is `json`. */
+export const parseOpenAIAudioTranscriptionResponseFormat = (value: unknown): OpenAIAudioTranscriptionResponseFormat => {
+ if (value === undefined) return 'json';
+ if (typeof value === 'string' && (OPENAI_AUDIO_TRANSCRIPTION_RESPONSE_FORMATS as readonly string[]).includes(value)) {
+ return value as OpenAIAudioTranscriptionResponseFormat;
+ }
+ throw new Error(`OpenAI Audio Transcriptions response_format is invalid: ${JSON.stringify(value)}`);
+};
+
+/** A JSON rendering answers with an object; `text`, `srt` and `vtt` answer with a document.
+ * The split decides how the body is read and written, and it is the whole of what the
+ * response format decides for a non-streaming answer. */
+export type OpenAIAudioTranscriptionObjectFormat = 'json' | 'verbose_json' | 'diarized_json';
+
+export const isOpenAIAudioTranscriptionObjectFormat = (
+ format: OpenAIAudioTranscriptionResponseFormat,
+): format is OpenAIAudioTranscriptionObjectFormat =>
+ format === 'json' || format === 'verbose_json' || format === 'diarized_json';
+
+/** Token-billed and duration-billed models report usage under one discriminated key. A
+ * transcription answered in `text`, `srt` or `vtt` reports none, because those renderings
+ * cannot express it. */
+export type OpenAIAudioTranscriptionUsage =
+ | {
+ readonly kind: 'tokens';
+ readonly inputTokens: number;
+ /** Split out of `inputTokens` when the upstream broke it down, because audio input is
+ * priced apart from text input. */
+ readonly inputAudioTokens?: number;
+ readonly outputTokens: number;
+ }
+ | { readonly kind: 'duration'; readonly seconds: number };
+
+export interface CanonicalOpenAIAudioTranscription {
+ /** What the upstream sent, byte for byte. Every rendering but the objects is written back
+ * from this, and so is a body the reading below could not open.
+ *
+ * Bytes rather than a string, because a decode is a reading like any other: a document
+ * under a charset nothing here asked about survives being carried and does not survive
+ * being decoded and encoded again. */
+ readonly document: Uint8Array;
+ /** The whole transcript, as the rendering stated it. A subtitle document does not carry
+ * one, so for `srt` and `vtt` this is the cues' texts joined by newlines — which is what
+ * Whisper's own text writer produces from the same result, and is not the string a `text`
+ * request would have returned. Absent on a document the reading could not open.
+ * https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L109-L116 */
+ readonly text?: string;
+ /** The timed cues. This is the whole of `srt` and `vtt`; `verbose_json` and
+ * `diarized_json` carry the same thing as `segments`. */
+ readonly cues?: readonly OpenAIAudioTranscriptionCue[];
+ /** The upstream's own object, kept whole for the renderings that are objects. A JSON
+ * answer is re-serialized from this, so a field this type does not name — a segment's
+ * `avg_logprob`, a `logprobs` array, a `speaker` label — reaches the client unchanged,
+ * and it is also what usage is read out of. */
+ readonly raw?: Record;
+}
+
+const isRecord = (value: unknown): value is Record =>
+ typeof value === 'object' && value !== null && !Array.isArray(value);
+
+const isFiniteNumber = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value);
+
+const seconds = (value: unknown, where: string): number => {
+ if (!isFiniteNumber(value) || value < 0) throw new Error(`OpenAI Audio Transcriptions ${where} must be a finite non-negative number`);
+ return value;
+};
+
+const count = (value: unknown, where: string): number => {
+ if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
+ throw new Error(`OpenAI Audio Transcriptions ${where} must be a non-negative safe integer`);
+ }
+ return value;
+};
+
+/**
+ * What the upstream said it will bill for.
+ *
+ * `undefined` is "nothing this reading recognizes was stated": no report at all, or one under
+ * a discriminator this protocol does not name — a metric invented after this was written is
+ * not a malformed one. A report that *is* one of the two and cannot be read is the other
+ * situation entirely, and it throws, so the caller can say so rather than bill an upstream
+ * that did meter as though it had not.
+ *
+ * Whisper's `verbose_json` predates the `usage` block and states the same duration at the top
+ * level, so that is read where no block is present.
+ * https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L36513-L36545
+ * https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L61968-L62051
+ */
+export const parseOpenAIAudioTranscriptionUsage = (value: unknown): OpenAIAudioTranscriptionUsage | undefined => {
+ if (!isRecord(value)) return undefined;
+ if (value.usage === undefined) {
+ if (value.duration === undefined) return undefined;
+ return { kind: 'duration', seconds: seconds(value.duration, 'duration') };
+ }
+ const usage = value.usage;
+ if (!isRecord(usage)) throw new Error('OpenAI Audio Transcriptions usage must be an object');
+ if (usage.type === 'duration') return { kind: 'duration', seconds: seconds(usage.seconds, 'usage.seconds') };
+ if (usage.type !== 'tokens') return undefined;
+
+ const inputTokens = count(usage.input_tokens, 'usage.input_tokens');
+ const outputTokens = count(usage.output_tokens, 'usage.output_tokens');
+ const details = usage.input_token_details;
+ if (details !== undefined && !isRecord(details)) throw new Error('OpenAI Audio Transcriptions usage.input_token_details must be an object');
+ const audioTokens = details?.audio_tokens === undefined
+ ? undefined
+ : count(details.audio_tokens, 'usage.input_token_details.audio_tokens');
+ if (audioTokens !== undefined && audioTokens > inputTokens) {
+ throw new Error('OpenAI Audio Transcriptions usage.input_token_details.audio_tokens must not exceed usage.input_tokens');
+ }
+ return {
+ kind: 'tokens',
+ inputTokens,
+ ...(audioTokens === undefined ? {} : { inputAudioTokens: audioTokens }),
+ outputTokens,
+ };
+};
+
+const requiredText = (value: unknown, where: string): string => {
+ if (typeof value !== 'string') throw new Error(`OpenAI Audio Transcriptions ${where} must be a string`);
+ return value;
+};
+
+/** `verbose_json` and `diarized_json` both time their segments with seconds and text; the
+ * fields they do not share stay in `raw` and reach the client through it. */
+const objectCues = (value: Record): readonly OpenAIAudioTranscriptionCue[] | undefined => {
+ if (!Array.isArray(value.segments)) return undefined;
+ return value.segments.map((segment: unknown, index) => {
+ if (!isRecord(segment) || !isFiniteNumber(segment.start) || !isFiniteNumber(segment.end)) {
+ throw new Error(`OpenAI Audio Transcriptions segment ${index} must carry numeric start and end`);
+ }
+ return { start: segment.start, end: segment.end, text: requiredText(segment.text, `segment ${index} text`) };
+ });
+};
+
+/**
+ * The one reading, over the bytes the upstream sent.
+ *
+ * It throws on a body it cannot open, and a caller that carries the document rather than
+ * serializing one from what it read answers anyway: what a failed reading costs is the
+ * transcript in the record and the usage measured beside it, never the answer.
+ */
+export const parseOpenAIAudioTranscription = (
+ format: OpenAIAudioTranscriptionResponseFormat,
+ document: Uint8Array,
+): CanonicalOpenAIAudioTranscription => {
+ const decoded = new TextDecoder().decode(document);
+ if (isOpenAIAudioTranscriptionObjectFormat(format)) {
+ const body: unknown = JSON.parse(decoded);
+ if (!isRecord(body)) throw new Error('OpenAI Audio Transcriptions response body must be an object');
+ const cues = objectCues(body);
+ return {
+ document,
+ text: requiredText(body.text, 'text'),
+ ...(cues === undefined ? {} : { cues }),
+ raw: body,
+ };
+ }
+ if (format === 'text') return { document, text: decoded };
+ const cues = parseSubtitleDocument(format, decoded);
+ return { document, text: cues.map(cue => cue.text).join('\n'), cues };
+};
+
+/**
+ * The one writing, back into the rendering the client asked for.
+ *
+ * An object rendering is the upstream's own object re-serialized, which is what keeps a field
+ * this protocol does not name on its way to the client. Everything else is the document that
+ * arrived, handed back untouched — and so is an object rendering whose body the reading could
+ * not open, because bytes nobody could read are still the answer the upstream gave.
+ */
+export const renderOpenAIAudioTranscription = (
+ format: OpenAIAudioTranscriptionResponseFormat,
+ transcription: CanonicalOpenAIAudioTranscription,
+): Record | Uint8Array =>
+ isOpenAIAudioTranscriptionObjectFormat(format) && transcription.raw !== undefined
+ ? transcription.raw
+ : transcription.document;
diff --git a/packages/protocols/src/openai-completions/index.ts b/packages/protocols/src/openai-completions/index.ts
index 85aa6f412c..5051241e5b 100644
--- a/packages/protocols/src/openai-completions/index.ts
+++ b/packages/protocols/src/openai-completions/index.ts
@@ -1,15 +1,53 @@
-// OpenAI text-completion protocol (POST /v1/completions). Floway runs
-// this endpoint as a passthrough, so this module declares the
-// protocol's wire shape rather than a read-time DTO: the gateway's
-// /completions handler parses the inbound body through its own local
-// shape and reads `model` / `stream` / `stream_options` directly. Only
-// `model` is read structurally by downstream code (the provider
-// interface accepts `Omit` and forwards
-// the rest unchanged); every other field flows through via the index
-// signature.
+// OpenAI text completions (`POST /v1/completions`). The gateway carries this protocol end to
+// end: what a client sends is parsed into `OpenAICompletionsPayload`, what an upstream answers
+// is parsed into an `OpenAICompletionsResult` or a frame stream, and what the client receives
+// is serialized back from those. No body is forwarded verbatim, so the shapes here are the
+// protocol's own rather than a convenience view over bytes in flight.
+//
+// Request and response follow OpenAI's `CreateCompletionRequest` and
+// `CreateCompletionResponse`, and the specification's own note on the latter is why there is
+// no separate streaming envelope type below the chunk level — "both the streamed and the
+// non-streamed response objects share the same shape (unlike the chat endpoint)":
+// https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L33793-L34030
+// https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34031-L34149
+// `model` is the one field the gateway itself reads structurally on the way in — routing is
+// what needs it — and `stream` / `stream_options` are what decide the shape of the answer.
+// The rest is named because the protocol names it, and is optional here because the upstream
+// is what accepts or refuses a request: the specification also requires `prompt`, and an
+// opinion about that held here would have to be kept in step with every OpenAI-compatible
+// upstream we route to. The index signature carries a vendor extension through to the
+// upstream that understands it.
export interface OpenAICompletionsPayload {
model: string;
+ prompt?: string | readonly string[] | readonly number[] | readonly (readonly number[])[] | null;
+ best_of?: number | null;
+ echo?: boolean | null;
+ frequency_penalty?: number | null;
+ logit_bias?: Record | null;
+ logprobs?: number | null;
+ max_tokens?: number | null;
+ n?: number | null;
+ presence_penalty?: number | null;
+ seed?: number | null;
+ stop?: string | readonly string[] | null;
+ stream?: boolean | null;
+ stream_options?: OpenAICompletionsStreamOptions | null;
+ suffix?: string | null;
+ temperature?: number | null;
+ top_p?: number | null;
+ user?: string | null;
+ [key: string]: unknown;
+}
+
+// `/v1/completions` shares OpenAI's `ChatCompletionStreamOptions` with `/v1/chat/completions`,
+// so `include_obfuscation` sits beside `include_usage` here as well:
+// https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L31764-L31810
+// The index signature is what lets the gateway turn `include_usage` on without dropping the
+// siblings a client sent.
+export interface OpenAICompletionsStreamOptions {
+ include_usage?: boolean;
+ include_obfuscation?: boolean;
[key: string]: unknown;
}
@@ -18,7 +56,7 @@ export interface OpenAICompletionsPayload {
// final placeholder choice carrying only `index` (no `text`, no
// `finish_reason`) alongside the usage block — so `text` and
// `finish_reason` are optional, matching that shape on the typed surface.
-// `logprobs` is opaque to the gateway — passed through as-is.
+// `logprobs` is opaque to the gateway — carried through as-is.
interface OpenAICompletionsChoiceStreaming {
index: number;
text?: string;
@@ -26,16 +64,18 @@ interface OpenAICompletionsChoiceStreaming {
logprobs?: unknown;
}
+// OpenAI's `CompletionUsage`, which `/v1/completions` reuses verbatim from
+// `/v1/chat/completions`:
+// https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L32210-L32283
+// The optional prompt-cache split is the part upstreams disagree about: OpenAI's own text
+// models do not populate it today, while vLLM, llama.cpp, Fireworks, OpenRouter and xAI Grok
+// all emit `cached_tokens` here on `/v1/completions`, and Azure mirrors the schema. Billing
+// reads the split through `openAICacheTokensFromUsage`, which also knows the field names the
+// wilder forks use, so what is named here is the schema rather than the union of the wild.
export interface OpenAICompletionsUsage {
prompt_tokens: number;
completion_tokens: number;
total_tokens: number;
- // OpenAI's CompletionUsage schema (which /v1/completions reuses verbatim
- // from /v1/chat/completions) carries an optional prompt-cache split.
- // OpenAI's own text models do not populate it today, but vLLM, llama.cpp,
- // Fireworks, OpenRouter, and xAI Grok all emit `cached_tokens` here on
- // /v1/completions, and Azure mirrors the schema. Floway extracts it when
- // present so billing metrics match what the upstream actually reported.
prompt_tokens_details?: { cached_tokens?: number };
}
@@ -61,15 +101,23 @@ export interface OpenAICompletionsChoice {
logprobs?: unknown;
}
+// `service_tier` is not in the specification's response schema — it belongs to the chat
+// endpoint — but a vLLM fork was observed emitting it on the non-streaming
+// `/v1/completions` body (null, on a Zhipu/GLM build) and billing reads it as the pricing
+// tier when it is there, so it is named rather than left to the index signature.
export interface OpenAICompletionsResult {
id: string;
object: 'text_completion';
created: number;
model: string;
choices: OpenAICompletionsChoice[];
+ service_tier?: string | null;
usage?: OpenAICompletionsUsage;
system_fingerprint?: string;
+ [key: string]: unknown;
}
+export { parseOpenAICompletionsPayload, parseOpenAICompletionsResult } from './parse.ts';
export { reassembleOpenAICompletionsEvents } from './reassemble.ts';
+export { parseOpenAICompletionsStream, type ParseOpenAICompletionsStreamOptions } from './stream.ts';
export { openaiCompletionsProtocolFrameToSSEFrame } from './to-sse.ts';
diff --git a/packages/protocols/src/openai-completions/parse.ts b/packages/protocols/src/openai-completions/parse.ts
new file mode 100644
index 0000000000..5837d1f7eb
--- /dev/null
+++ b/packages/protocols/src/openai-completions/parse.ts
@@ -0,0 +1,22 @@
+import type { OpenAICompletionsPayload, OpenAICompletionsResult } from './index.ts';
+import { isJsonObject } from '../common/json.ts';
+
+// The two boundaries where a `/v1/completions` body becomes a value. Both check what the
+// gateway itself depends on and nothing beyond it: the request must name a model, because
+// routing reads it, and either body must be a JSON object, because everything downstream
+// addresses it as one. Which fields an upstream accepts and which it returns is the
+// upstream's to decide, and a gateway opinion about the rest would have to be kept in step
+// with every OpenAI-compatible implementation we route to.
+
+export const parseOpenAICompletionsPayload = (value: unknown): OpenAICompletionsPayload => {
+ if (!isJsonObject(value)) throw new Error('OpenAI Completions request body must be an object.');
+ if (typeof value.model !== 'string' || value.model.length === 0) {
+ throw new Error('OpenAI Completions request body must include a model string.');
+ }
+ return value as OpenAICompletionsPayload;
+};
+
+export const parseOpenAICompletionsResult = (value: unknown): OpenAICompletionsResult => {
+ if (!isJsonObject(value)) throw new Error('OpenAI Completions response body must be an object.');
+ return value as OpenAICompletionsResult;
+};
diff --git a/packages/protocols/src/openai-completions/stream.ts b/packages/protocols/src/openai-completions/stream.ts
new file mode 100644
index 0000000000..179b5f228a
--- /dev/null
+++ b/packages/protocols/src/openai-completions/stream.ts
@@ -0,0 +1,26 @@
+import type { OpenAICompletionsStreamEvent } from './index.ts';
+import { parseTargetStreamFrames } from '../common/parse-events.ts';
+import { parseSSEStream } from '../common/parse-sse.ts';
+import { doneFrame, eventFrame, type ProtocolFrame } from '../common/sse.ts';
+
+export interface ParseOpenAICompletionsStreamOptions {
+ signal?: AbortSignal;
+}
+
+// The upstream's SSE body as protocol frames. Transport framing ends here: what comes out
+// carries `text_completion` chunks and a terminal frame, and nothing that reads it knows how
+// those arrived.
+export const parseOpenAICompletionsStream = (
+ body: ReadableStream,
+ options: ParseOpenAICompletionsStreamOptions = {},
+): AsyncGenerator> => (async function* () {
+ for await (const frame of parseTargetStreamFrames(parseSSEStream(body, options), {
+ protocol: 'OpenAI Completions',
+ })) {
+ if (frame.type === 'done') {
+ yield doneFrame();
+ return;
+ }
+ yield eventFrame(frame.data);
+ }
+})();
diff --git a/packages/protocols/src/openai-embeddings/index.ts b/packages/protocols/src/openai-embeddings/index.ts
index a82596b253..3555398383 100644
--- a/packages/protocols/src/openai-embeddings/index.ts
+++ b/packages/protocols/src/openai-embeddings/index.ts
@@ -1,5 +1,100 @@
+// OpenAI's `/v1/embeddings` contract, and the canonical form the gateway holds it in.
+// Specification: https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34214-L34331
+//
+// There is one OpenAI Embeddings protocol, so the canonical form and the wire differ in one
+// substantive place: `encoding_format` decides how a vector is *written*, and a vector is
+// the same vector either way — so the canonical response holds numbers, and the encoding
+// is a rendering decision the edge makes from what the client asked for.
+//
+// That divergence is load-bearing rather than tidy. Both official OpenAI SDKs send
+// `encoding_format: base64` when the caller did not choose one, so base64 is the common
+// case on the wire and not an exotic one:
+// https://github.com/openai/openai-python/blob/10ee3f0da2ac6f93345c1204bd7bb1a2faa79ff2/src/openai/resources/embeddings.py#L111-L112
+// https://github.com/openai/openai-node/blob/cc7dbfa9b9dd6fe0ff72141e9c0c3d82b18ba9aa/src/resources/embeddings.ts#L36-L42
+// An upstream that ignores the field and answers with float arrays hands such a client a
+// body it will decode as base64 and turn into noise. Parsing whatever arrived and
+// rendering what the client asked for is what closes that.
+
+/** How a vector is written on the wire. Under `base64` an embedding is one string, not an
+ * array of numbers.
+ * https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34272-L34280 */
+export type OpenAIEmbeddingsEncodingFormat = 'float' | 'base64';
+
+/** One text, a batch of texts, one pre-tokenized text, or a batch of those — the
+ * specification's own four-armed `oneOf`:
+ * https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34218-L34254
+ *
+ * The arms are told apart by reading an element, which works only because every array arm
+ * is `minItems: 1` there — an empty array would belong to three of them at once. Parsing
+ * rejects it, so a value of this type is always unambiguous. */
+export type OpenAIEmbeddingsInput = string | readonly string[] | readonly number[] | readonly (readonly number[])[];
+
+export interface CanonicalOpenAIEmbeddingsRequest {
+ input: OpenAIEmbeddingsInput;
+ /** What the upstream is asked for, absent included — a client that did not ask must not
+ * grow a field on the wire it never sent. */
+ encodingFormat?: OpenAIEmbeddingsEncodingFormat;
+ /** Only `text-embedding-3` and later models honour it
+ * (https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34281-L34286),
+ * and which models those are is the upstream's knowledge and not the catalog's — so it
+ * travels to whichever upstream is chosen and that upstream answers for it. */
+ dimensions?: number;
+ /** https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34287-L34293 */
+ user?: string;
+}
+
+export interface ParsedOpenAIEmbeddingsRequest {
+ model: string;
+ /** What the client will be able to read: `encoding_format` resolved against the
+ * protocol's own default of `float`
+ * (https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34276).
+ * Concrete, because the answer has to be written in one encoding or the other — and
+ * distinct from the request's own field, which stays absent when the client omitted it. */
+ encodingFormat: OpenAIEmbeddingsEncodingFormat;
+ request: CanonicalOpenAIEmbeddingsRequest;
+}
+
+export interface CanonicalOpenAIEmbedding {
+ /** Which input this vector answers, carried by the specification as a field rather than
+ * left to array position:
+ * https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L38168-L38170 */
+ index: number;
+ values: readonly number[];
+}
+
+export interface CanonicalOpenAIEmbeddingsUsage {
+ promptTokens: number;
+ totalTokens: number;
+}
+
+export interface CanonicalOpenAIEmbeddingsResponse {
+ /** What the upstream says it generated with, which need not be the id the client
+ * addressed — an upstream is free to answer under a dated or internal name. */
+ model: string;
+ embeddings: readonly CanonicalOpenAIEmbedding[];
+ /** The specification requires it
+ * (https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34327-L34331)
+ * and it is optional here regardless, because an upstream that reports nothing is a
+ * situation the gateway has to be able to state: the billed entity is then present with
+ * no quantities, which is a different fact from reporting zero. Rejecting the body
+ * instead would turn an answered call into a 502 and throw away the vectors it
+ * returned. */
+ usage?: CanonicalOpenAIEmbeddingsUsage;
+}
+
+/** The request as it goes out. `model` is the upstream's own id and the provider
+ * substitutes it, so what the gateway hands a provider is everything but that. */
export interface OpenAIEmbeddingsPayload {
model: string;
- input?: unknown;
- [key: string]: unknown;
+ input: OpenAIEmbeddingsInput;
+ encoding_format?: OpenAIEmbeddingsEncodingFormat;
+ dimensions?: number;
+ user?: string;
}
+
+export {
+ parseOpenAIEmbeddingsRequest,
+ parseOpenAIEmbeddingsResponse,
+ renderOpenAIEmbeddingsResponse,
+ serializeOpenAIEmbeddingsRequest,
+} from './translate.ts';
diff --git a/packages/protocols/src/openai-embeddings/translate.ts b/packages/protocols/src/openai-embeddings/translate.ts
new file mode 100644
index 0000000000..f524d8b8e0
--- /dev/null
+++ b/packages/protocols/src/openai-embeddings/translate.ts
@@ -0,0 +1,199 @@
+import type {
+ CanonicalOpenAIEmbedding,
+ CanonicalOpenAIEmbeddingsRequest,
+ CanonicalOpenAIEmbeddingsResponse,
+ CanonicalOpenAIEmbeddingsUsage,
+ OpenAIEmbeddingsEncodingFormat,
+ OpenAIEmbeddingsInput,
+ OpenAIEmbeddingsPayload,
+ ParsedOpenAIEmbeddingsRequest,
+} from './index.ts';
+import { decodeForgivingBase64, encodeBase64 } from '../common/base-encoding.ts';
+
+const isRecord = (value: unknown): value is Record =>
+ typeof value === 'object' && value !== null && !Array.isArray(value);
+
+const requiredString = (value: unknown, field: string): string => {
+ if (typeof value !== 'string' || value.length === 0) throw new Error(`${field} must be a non-empty string`);
+ return value;
+};
+
+const optionalString = (value: unknown, field: string): string | undefined => {
+ if (value === undefined) return undefined;
+ if (typeof value !== 'string') throw new Error(`${field} must be a string`);
+ return value;
+};
+
+const requiredInteger = (value: unknown, field: string): number => {
+ if (typeof value !== 'number' || !Number.isInteger(value)) throw new Error(`${field} must be an integer`);
+ return value;
+};
+
+const optionalPositiveInteger = (value: unknown, field: string): number | undefined => {
+ if (value === undefined) return undefined;
+ const integer = requiredInteger(value, field);
+ if (integer < 1) throw new Error(`${field} must be a positive integer`);
+ return integer;
+};
+
+const parseEncodingFormat = (value: unknown): OpenAIEmbeddingsEncodingFormat | undefined => {
+ if (value === undefined) return undefined;
+ if (value !== 'float' && value !== 'base64') throw new Error('encoding_format must be float or base64');
+ return value;
+};
+
+const parseTokenIds = (value: readonly unknown[], field: string): readonly number[] => {
+ if (value.length === 0) throw new Error(`${field} must not be empty`);
+ return value.map((token, index) => requiredInteger(token, `${field}[${index}]`));
+};
+
+const parseInput = (value: unknown): OpenAIEmbeddingsInput => {
+ if (typeof value === 'string') return value;
+ if (!Array.isArray(value)) throw new Error('input must be a string or an array');
+ if (value.length === 0) throw new Error('input must not be empty');
+ const head: unknown = value[0];
+ if (typeof head === 'string') return value.map((text, index) => requiredString(text, `input[${index}]`));
+ if (!Array.isArray(head)) return parseTokenIds(value as readonly unknown[], 'input');
+ return value.map((tokens, index) => {
+ if (!Array.isArray(tokens)) throw new Error(`input[${index}] must be an array of integers`);
+ return parseTokenIds(tokens as readonly unknown[], `input[${index}]`);
+ });
+};
+
+// The request schema is `additionalProperties: false`
+// (https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34216),
+// so the protocol has no extension point and a field outside this set is one the gateway
+// cannot carry. Naming it beats dropping it silently and beats guessing what it meant. The
+// two gateways compared against draw the same line: LiteLLM's embeddings surface is
+// `user` / `encoding_format` / `dimensions`, and anything else raises unless the operator
+// opts into dropping —
+// https://github.com/BerriAI/litellm/blob/bc6e7df05b018eefe6c7293790ca3f4de38709ac/litellm/utils.py#L3259-L3273
+// — while copilot-api's request type is narrower still, `input` and `model` alone:
+// https://github.com/ericc-ch/copilot-api/blob/0ea08febdd7e3e055b03dd298bf57e669500b5c1/src/services/copilot/create-embeddings.ts#L19-L22
+const OPENAI_EMBEDDINGS_REQUEST_FIELDS = ['model', 'input', 'encoding_format', 'dimensions', 'user'];
+
+export const parseOpenAIEmbeddingsRequest = (value: unknown): ParsedOpenAIEmbeddingsRequest => {
+ if (!isRecord(value)) throw new Error('OpenAI Embeddings request body must be an object');
+ const unsupported = Object.keys(value).filter(field => !OPENAI_EMBEDDINGS_REQUEST_FIELDS.includes(field));
+ if (unsupported.length > 0) throw new Error(`OpenAI Embeddings does not support ${unsupported.join(', ')}`);
+
+ const encodingFormat = parseEncodingFormat(value.encoding_format);
+ const dimensions = optionalPositiveInteger(value.dimensions, 'dimensions');
+ const user = optionalString(value.user, 'user');
+ return {
+ model: requiredString(value.model, 'model'),
+ encodingFormat: encodingFormat ?? 'float',
+ request: {
+ input: parseInput(value.input),
+ ...(encodingFormat === undefined ? {} : { encodingFormat }),
+ ...(dimensions === undefined ? {} : { dimensions }),
+ ...(user === undefined ? {} : { user }),
+ },
+ };
+};
+
+export const serializeOpenAIEmbeddingsRequest = (request: CanonicalOpenAIEmbeddingsRequest): Omit => ({
+ input: request.input,
+ ...(request.encodingFormat === undefined ? {} : { encoding_format: request.encodingFormat }),
+ ...(request.dimensions === undefined ? {} : { dimensions: request.dimensions }),
+ ...(request.user === undefined ? {} : { user: request.user }),
+});
+
+// A base64 embedding is the vector's float32 elements packed little-endian. Both official
+// SDKs establish that by reading the decoded bytes straight into a native float32 view —
+// `array.array('f', ...)` and `Float32Array`, each host-endian and each correct only
+// because every platform they run on is:
+// https://github.com/openai/openai-python/blob/10ee3f0da2ac6f93345c1204bd7bb1a2faa79ff2/src/openai/resources/embeddings.py#L122-L131
+// https://github.com/openai/openai-node/blob/cc7dbfa9b9dd6fe0ff72141e9c0c3d82b18ba9aa/src/internal/utils/base64.ts#L45-L52
+// `DataView` states the byte order rather than inheriting it.
+//
+// Holding a vector as numbers is exact and not an approximation: every float32 is a
+// float64, so widening loses nothing and narrowing a value that came from a float32 gives
+// the same bits back. A vector survives any number of trips through this pair.
+const FLOAT32_BYTES = 4;
+
+const parseEmbedding = (value: unknown, field: string): readonly number[] => {
+ if (Array.isArray(value)) {
+ return value.map((element, index) => {
+ if (typeof element !== 'number') throw new Error(`${field}[${index}] must be a number`);
+ return element;
+ });
+ }
+ if (typeof value !== 'string') throw new Error(`${field} must be an array of numbers or a base64 string`);
+ const bytes = decodeForgivingBase64(value);
+ if (bytes.length % FLOAT32_BYTES !== 0) throw new Error(`${field} is not a whole number of float32 values`);
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+ return Array.from({ length: bytes.length / FLOAT32_BYTES }, (_, index) => view.getFloat32(index * FLOAT32_BYTES, true));
+};
+
+const renderEmbedding = (values: readonly number[]): string => {
+ const bytes = new Uint8Array(values.length * FLOAT32_BYTES);
+ const view = new DataView(bytes.buffer);
+ values.forEach((value, index) => view.setFloat32(index * FLOAT32_BYTES, value, true));
+ return encodeBase64(bytes);
+};
+
+const parseUsage = (value: unknown): CanonicalOpenAIEmbeddingsUsage | undefined => {
+ if (value === undefined || value === null) return undefined;
+ if (!isRecord(value)) throw new Error('usage must be an object');
+ return {
+ promptTokens: requiredInteger(value.prompt_tokens, 'usage.prompt_tokens'),
+ totalTokens: requiredInteger(value.total_tokens, 'usage.total_tokens'),
+ };
+};
+
+/**
+ * Reads an upstream's answer into the canonical form.
+ *
+ * `requestedModel` completes the record when the upstream did not name one. The schema marks
+ * `model` required —
+ * https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34327-L34331
+ * — and we have observed Copilot's `/embeddings` answering without it. Nothing holds it to
+ * the field: the endpoint's own first-party client reads `data` alone and never looks at it,
+ * where the sibling dotcom route reads its model back and asserts it —
+ * https://github.com/microsoft/vscode-copilot-chat/blob/5863f5a7088958050792b5dccbe8b46c6e13eccc/src/platform/embeddings/common/remoteEmbeddingsComputer.ts#L294-L301
+ *
+ * Naming the model the request asked for is what the client can correlate against, and it is
+ * what another Copilot gateway meters under for the same reason —
+ * https://github.com/caozhiyuan/copilot-api/blob/96854ce1e1e741d621b12d4ca844efd77f6e8d90/src/routes/embeddings/route.ts#L16-L23
+ */
+export const parseOpenAIEmbeddingsResponse = (value: unknown, requestedModel: string): CanonicalOpenAIEmbeddingsResponse => {
+ if (!isRecord(value)) throw new Error('OpenAI Embeddings response body must be an object');
+ const { data } = value;
+ if (!Array.isArray(data)) throw new Error('data must be an array');
+ const usage = parseUsage(value.usage);
+ const embeddings: CanonicalOpenAIEmbedding[] = data.map((entry, position) => {
+ if (!isRecord(entry)) throw new Error(`data[${position}] must be an object`);
+ return {
+ index: requiredInteger(entry.index, `data[${position}].index`),
+ values: parseEmbedding(entry.embedding, `data[${position}].embedding`),
+ };
+ });
+ return {
+ model: value.model === undefined ? requestedModel : requiredString(value.model, 'model'),
+ embeddings,
+ ...(usage === undefined ? {} : { usage }),
+ };
+};
+
+/**
+ * Writes the answer in the encoding the client asked for, whichever one the upstream
+ * answered in. `object` is `x-stainless-const` at both levels, so it is a constant of the
+ * protocol rather than something an upstream varies, and it is written here rather than
+ * repeated back from the body.
+ */
+export const renderOpenAIEmbeddingsResponse = (
+ format: OpenAIEmbeddingsEncodingFormat,
+ response: CanonicalOpenAIEmbeddingsResponse,
+): Record => ({
+ object: 'list',
+ data: response.embeddings.map(embedding => ({
+ object: 'embedding',
+ index: embedding.index,
+ embedding: format === 'base64' ? renderEmbedding(embedding.values) : embedding.values,
+ })),
+ model: response.model,
+ ...(response.usage === undefined ? {} : {
+ usage: { prompt_tokens: response.usage.promptTokens, total_tokens: response.usage.totalTokens },
+ }),
+});
diff --git a/packages/protocols/src/openai-images/index.ts b/packages/protocols/src/openai-images/index.ts
index 7b8462cdb8..e820333e23 100644
--- a/packages/protocols/src/openai-images/index.ts
+++ b/packages/protocols/src/openai-images/index.ts
@@ -1,3 +1,12 @@
+// The OpenAI Images protocol: POST /v1/images/generations and POST /v1/images/edits. One
+// protocol over two endpoints — generations takes JSON, and edits takes either JSON, where
+// every image is a URL, a data URL or a file id, or a multipart form carrying the files
+// themselves.
+// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12858-L12870
+// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620
+
+export type OpenAIImagesOperation = 'generations' | 'edits';
+
// JSON payload accepted by POST /v1/images/generations. Field set follows
// OpenAI's reference for gpt-image-* and legacy dall-e-* (dall-e is
// retired but the union shape is harmless). Declared as an interface with
@@ -29,3 +38,96 @@ export interface OpenAIImagesGenerationsPayload {
export type OpenAIImageEditReference =
| { image_url: string; file_id?: never; [key: string]: unknown }
| { file_id: string; image_url?: never; [key: string]: unknown };
+
+/** A file the client sent in a multipart form, held as bytes: a form is a parsed value like
+ * any other, and bytes are what survives being read once and what a dump can show. */
+export interface OpenAIImagesUploadedFile {
+ fileName: string;
+ mediaType: string;
+ bytes: Uint8Array;
+}
+
+/** One image an edit reads. Multipart carries the file itself; JSON carries a reference for
+ * the upstream to resolve, kept exactly as the client wrote it — whether a data URL can be
+ * turned back into a file is the upstream serializer's question, not this one's. */
+export type OpenAIImagesEditImage =
+ | { kind: 'file'; file: OpenAIImagesUploadedFile }
+ | { kind: 'reference'; reference: OpenAIImageEditReference };
+
+export interface CanonicalOpenAIImagesGenerationsRequest {
+ operation: 'generations';
+ /** Everything the client sent but `model`: routing owns the model id, and what is left is
+ * what the upstream is asked for. */
+ parameters: Record;
+}
+
+export interface CanonicalOpenAIImagesEditsRequest {
+ operation: 'edits';
+ images: OpenAIImagesEditImage[];
+ mask?: OpenAIImagesEditImage;
+ parameters: Record;
+}
+
+export type CanonicalOpenAIImagesRequest = CanonicalOpenAIImagesGenerationsRequest | CanonicalOpenAIImagesEditsRequest;
+
+export interface ParsedOpenAIImagesRequest {
+ model: string;
+ request: CanonicalOpenAIImagesRequest;
+}
+
+/** One image the upstream returned. `response_format` decides which arm carries it on the
+ * dall-e models, while the GPT image models answer base64 and do not serve a URL at all, so
+ * neither arm is the one to expect.
+ * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L51044-L51067 */
+export interface CanonicalOpenAIImage {
+ url?: string;
+ base64?: string;
+ revisedPrompt?: string;
+}
+
+/** Token counts as the images endpoints report them, disjoint: what the upstream attributed to
+ * images is taken out of the count beside it. Only the GPT image models report any of this —
+ * the dall-e models report nothing — which is why the whole reading is optional as well as
+ * each field.
+ * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L78089-L78115 */
+export interface CanonicalOpenAIImagesUsage {
+ inputTokens?: number;
+ inputImageTokens?: number;
+ outputTokens?: number;
+ outputImageTokens?: number;
+}
+
+export interface CanonicalOpenAIImagesResponse {
+ /** The parsed body as it arrived. One protocol in and one out means rendering is
+ * re-serializing this, so the fields the gateway does not model still reach the client. */
+ raw: Record;
+ images: CanonicalOpenAIImage[];
+ usage?: CanonicalOpenAIImagesUsage;
+}
+
+/** One event on a streamed answer. A stream carries partial images and then the completed one,
+ * and the completed one carries the `usage` block the non-streamed body carries — so what a
+ * call is billed by is stated the same way whichever shape it came back in.
+ * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L51337-L51424
+ * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L51158-L51246
+ *
+ * `type` is an open string rather than the four names the specification lists: an event type
+ * is the upstream's own word, and a shape added later has to reach the client whether or not
+ * this gateway has heard of it. The index signature carries everything not named here. */
+export interface OpenAIImagesStreamEvent {
+ type: string;
+ b64_json?: string;
+ created_at?: number;
+ size?: string;
+ quality?: string;
+ background?: string;
+ output_format?: string;
+ partial_image_index?: number;
+ usage?: unknown;
+ [key: string]: unknown;
+}
+
+export { openaiImagesRequestWantsStream, parseOpenAIImagesEditsRequest, parseOpenAIImagesGenerationsRequest } from './request.ts';
+export { parseOpenAIImagesResponse, parseOpenAIImagesUsage, renderOpenAIImagesResponse } from './response.ts';
+export { OPENAI_IMAGES_MISSING_TERMINAL_MESSAGE, isOpenAIImagesTerminalEvent, parseOpenAIImagesStream, type ParseOpenAIImagesStreamOptions } from './stream.ts';
+export { openaiImagesStreamEventToSSEFrame } from './to-sse.ts';
diff --git a/packages/protocols/src/openai-images/request.ts b/packages/protocols/src/openai-images/request.ts
new file mode 100644
index 0000000000..4a485787b7
--- /dev/null
+++ b/packages/protocols/src/openai-images/request.ts
@@ -0,0 +1,124 @@
+// Reading what the client sent into the canonical request. The two endpoints take different
+// bodies — generations is JSON, edits is JSON or a multipart form — and each message here is
+// what the client is told, so they name the endpoint and the field they are about.
+
+import type { CanonicalOpenAIImagesRequest, OpenAIImageEditReference, OpenAIImagesEditImage, OpenAIImagesUploadedFile, ParsedOpenAIImagesRequest } from './index.ts';
+import { isJsonMediaType, isMultipartFormDataMediaType } from '../common/media-type.ts';
+
+const isRecord = (value: unknown): value is Record =>
+ typeof value === 'object' && value !== null && !Array.isArray(value);
+
+/**
+ * Whether the client asked for the answer as a stream.
+ *
+ * The flag rides on to the upstream inside `parameters` exactly as it arrived, and this is the
+ * gateway's own reading of it. It is two values rather than one because an edit may be sent as
+ * a multipart form, where every field arrives as the text the client typed, while generations
+ * and a JSON edit carry the boolean the specification names.
+ * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L47542-L47673
+ */
+export const openaiImagesRequestWantsStream = (request: CanonicalOpenAIImagesRequest): boolean =>
+ request.parameters.stream === true || request.parameters.stream === 'true';
+
+const jsonObject = (body: Uint8Array, endpoint: string): Record