feat(cloud-agent): classify retryable runtime failures

packages/db/src/schema.ts

@@ -4361,6 +4361,7 @@ export type CloudAgentSessionRunFailureCode =
   | 'assistant_error'
   | 'wrapper_error_after_activity'
   | 'missing_assistant_reply'
+  | 'payment_required'
   | 'user_interrupt'
   | 'container_shutdown'
   | 'system_interrupt'

packages/worker-utils/package.json

@@ -23,7 +23,8 @@
     "./security-auto-analysis-policy": "./src/security-auto-analysis-policy.ts",
     "./security-remediation-policy": "./src/security-remediation-policy.ts",
     "./security-notification-policy": "./src/security-notification-policy.ts",
-    "./dependabot-dismissal-target": "./src/dependabot-dismissal-target.ts"
+    "./dependabot-dismissal-target": "./src/dependabot-dismissal-target.ts",
+    "./client-error": "./src/client-error.ts"
   },
   "scripts": {
     "test": "vitest run",

packages/worker-utils/src/client-error.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, it } from 'vitest';
+
+import { ClientErrorSchema, PublicErrorCodeSchema } from './client-error.js';
+
+describe('ClientErrorSchema', () => {
+  it('accepts the public client error wire contract', () => {
+    expect(
+      ClientErrorSchema.parse({
+        code: 'PENDING_QUEUE_FULL',
+        message: 'Queue is full',
+        retryable: true,
+      })
+    ).toEqual({
+      code: 'PENDING_QUEUE_FULL',
+      message: 'Queue is full',
+      retryable: true,
+    });
+  });
+
+  it.each(['', 'lowercase', '_PRIVATE', '9INVALID', 'HAS-DASH'])(
+    'rejects invalid code %j',
+    code => {
+      expect(PublicErrorCodeSchema.safeParse(code).success).toBe(false);
+    }
+  );
+});

packages/worker-utils/src/client-error.ts

@@ -0,0 +1,11 @@
+import { z } from 'zod';
+
+export const PublicErrorCodeSchema = z.string().regex(/^[A-Z][A-Z0-9_]*$/);
+
+export const ClientErrorSchema = z.object({
+  code: PublicErrorCodeSchema,
+  message: z.string(),
+  retryable: z.boolean(),
+});
+
+export type ClientError = z.infer<typeof ClientErrorSchema>;

packages/worker-utils/src/cloud-agent-failure.ts

@@ -27,6 +27,7 @@ export const CLOUD_AGENT_FAILURE_CODES = [
   'assistant_error',
   'wrapper_error_after_activity',
   'missing_assistant_reply',
+  'payment_required',
   'user_interrupt',
   'container_shutdown',
   'system_interrupt',

services/cloud-agent-next/src/callbacks/types.ts

@@ -1,3 +1,5 @@
+import type { CloudAgentFailureStage } from '@kilocode/worker-utils/cloud-agent-failure';
+import type { ClientError } from '@kilocode/worker-utils/client-error';
 import type { SafeFailureProjection } from '../session/safe-failure-projection.js';
 
 export type CallbackTarget = {
@@ -20,6 +22,8 @@ export type ExecutionCallbackPayload = {
   status: 'completed' | 'failed' | 'interrupted';
   errorMessage?: string;
   failure?: SafeFailureProjection;
+  failureStage?: CloudAgentFailureStage;
+  clientError?: ClientError;
   /** Present when errorMessage was shortened to fit the callback queue. */
   errorMessageTruncation?: CallbackTextTruncation;
   lastSeenBranch?: string;

services/cloud-agent-next/src/middleware/auth.test.ts

@@ -0,0 +1,64 @@
+import { Hono } from 'hono';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import type { HonoContext } from '../hono-context.js';
+import type { Env } from '../types.js';
+
+vi.mock('../auth.js', () => ({
+  validateKiloToken: vi.fn(),
+}));
+
+vi.mock('../logger.js', () => {
+  const logger = {
+    setTags: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    withFields: vi.fn(),
+  };
+  logger.withFields.mockReturnValue(logger);
+  return {
+    logger,
+    withLogTags: async (_tags: unknown, fn: () => Promise<unknown>) => fn(),
+    WithLogTags:
+      () =>
+      (
+        _target: unknown,
+        _propertyKey: string,
+        descriptor: PropertyDescriptor
+      ): PropertyDescriptor =>
+        descriptor,
+  };
+});
+
+const { authMiddleware } = await import('./auth.js');
+const { validateKiloToken } = await import('../auth.js');
+
+describe('authMiddleware', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(validateKiloToken).mockResolvedValue({ success: false, error: 'Invalid token' });
+  });
+
+  it('returns a non-retryable unauthorized client error without changing message or path', async () => {
+    const app = new Hono<HonoContext>();
+    app.use('/trpc/*', authMiddleware);
+    app.post('/trpc/:procedure', c => c.json({ ok: true }));
+
+    const response = await app.fetch(
+      new Request('https://worker.test/trpc/send', { method: 'POST' }),
+      { NEXTAUTH_SECRET: 'secret' } as Env
+    );
+    const body: any = await response.json();
+
+    expect(response.status).toBe(401);
+    expect(body.error.message).toBe('Invalid token');
+    expect(body.error.data).toMatchObject({
+      path: 'send',
+      clientError: {
+        code: 'UNAUTHORIZED',
+        message: 'Invalid token',
+        retryable: false,
+      },
+    });
+  });
+});

services/cloud-agent-next/src/middleware/balance.test.ts

@@ -58,6 +58,40 @@ describe('balanceMiddleware', () => {
     );
   }
 
+  it('returns non-retryable clientError for insufficient credits', async () => {
+    vi.mocked(validateBalanceOnly).mockResolvedValue({
+      success: false,
+      status: 402,
+      message: 'Insufficient credits',
+    });
+
+    const response = await postTrpc('start', {});
+    const body: any = await response.json();
+
+    expect(body.error.data.clientError).toEqual({
+      code: 'PAYMENT_REQUIRED',
+      message: 'Insufficient credits',
+      retryable: false,
+    });
+  });
+
+  it('returns retryable clientError for balance infrastructure failures', async () => {
+    vi.mocked(validateBalanceOnly).mockResolvedValue({
+      success: false,
+      status: 500,
+      message: 'Failed to verify balance',
+    });
+
+    const response = await postTrpc('start', {});
+    const body: any = await response.json();
+
+    expect(body.error.data.clientError).toEqual({
+      code: 'INTERNAL_SERVER_ERROR',
+      message: 'Failed to verify balance',
+      retryable: true,
+    });
+  });
+
   it('uses nested start organization context for balance validation', async () => {
     const orgId = '11111111-2222-3333-4444-555555555555';
 

services/cloud-agent-next/src/persistence/CloudAgentSession.ts

@@ -15,6 +15,7 @@ import {
 import { readProfileBundle, type SessionProfileBundle } from '../session-profile.js';
 import { fitCallbackJobToQueueLimit } from '../callbacks/queue-payload.js';
 import type { CallbackJob, CallbackTarget } from '../callbacks/index.js';
+import { projectTerminalClientError } from '../session/terminal-error-projector.js';
 import { drizzle } from 'drizzle-orm/durable-sqlite';
 import { logger } from '../logger.js';
 import { BUILTIN_AGENT_MODES, Limits } from '../schema.js';
@@ -345,6 +346,9 @@ export class CloudAgentSession extends DurableObject<WorkerEnv> {
       executionId: execution.executionId,
       status,
       errorMessage: error,
+      ...(status === 'completed'
+        ? {}
+        : { clientError: projectTerminalClientError({ status, error }) }),
       lastSeenBranch: metadata.repository?.upstreamBranch,
       kiloSessionId: metadata.auth.kiloSessionId,
       gateResult,

services/cloud-agent-next/src/router/auth.test.ts

@@ -0,0 +1,95 @@
+import { describe, expect, it } from 'vitest';
+import { TRPCError } from '@trpc/server';
+import { fetchRequestHandler } from '@trpc/server/adapters/fetch';
+
+import type { TRPCContext } from '../types.js';
+import { t } from './auth.js';
+
+const testRouter = t.router({
+  invalid: t.procedure.query(() => {
+    throw new TRPCError({ code: 'BAD_REQUEST', message: 'Invalid request' });
+  }),
+  unavailable: t.procedure.query(() => {
+    throw new TRPCError({
+      code: 'SERVICE_UNAVAILABLE',
+      message: 'Sandbox unavailable',
+      cause: { error: 'SANDBOX_CONNECT_FAILED', retryable: true },
+    });
+  }),
+  internal: t.procedure.query(() => {
+    throw new TRPCError({ code: 'INTERNAL_SERVER_ERROR', message: 'Internal failure' });
+  }),
+});
+
+async function requestProcedure(procedure: 'invalid' | 'unavailable' | 'internal') {
+  return fetchRequestHandler({
+    endpoint: '/trpc',
+    req: new Request(`http://localhost/trpc/${procedure}`),
+    router: testRouter,
+    createContext: () => ({}) as TRPCContext,
+  });
+}
+
+describe('tRPC client error formatter', () => {
+  it('adds a non-retryable client error to known request failures', async () => {
+    const response = await requestProcedure('invalid');
+
+    await expect(response.json()).resolves.toMatchObject({
+      error: {
+        message: 'Invalid request',
+        data: {
+          code: 'BAD_REQUEST',
+          httpStatus: 400,
+          path: 'invalid',
+          clientError: {
+            code: 'BAD_REQUEST',
+            message: 'Invalid request',
+            retryable: false,
+          },
+        },
+      },
+    });
+  });
+
+  it('preserves explicit legacy retry fields beside the client error', async () => {
+    const response = await requestProcedure('unavailable');
+
+    await expect(response.json()).resolves.toMatchObject({
+      error: {
+        message: 'Sandbox unavailable',
+        data: {
+          code: 'SERVICE_UNAVAILABLE',
+          httpStatus: 503,
+          path: 'unavailable',
+          error: 'SANDBOX_CONNECT_FAILED',
+          retryable: true,
+          clientError: {
+            code: 'SANDBOX_CONNECT_FAILED',
+            message: 'Sandbox unavailable',
+            retryable: true,
+          },
+        },
+      },
+    });
+  });
+
+  it('defaults generic internal failures to retryable', async () => {
+    const response = await requestProcedure('internal');
+
+    await expect(response.json()).resolves.toMatchObject({
+      error: {
+        message: 'Internal failure',
+        data: {
+          code: 'INTERNAL_SERVER_ERROR',
+          httpStatus: 500,
+          path: 'internal',
+          clientError: {
+            code: 'INTERNAL_SERVER_ERROR',
+            message: 'Internal failure',
+            retryable: true,
+          },
+        },
+      },
+    });
+  });
+});

services/cloud-agent-next/src/router/auth.ts

@@ -1,34 +1,15 @@
 import { initTRPC, TRPCError } from '@trpc/server';
 import { timingSafeEqual } from '@kilocode/encryption';
 import type { TRPCContext } from '../types.js';
-
-/**
- * Type for error cause data that should be surfaced in the response.
- * Used for structured 409 Conflict and 503 Retryable errors.
- */
-type ErrorCauseData = {
-  error?: string;
-  message?: string;
-  retryable?: boolean;
-};
+import { projectTrpcErrorData } from '../trpc-error.js';
 
 // Initialize tRPC with context and error formatter
 export const t = initTRPC.context<TRPCContext>().create({
   errorFormatter({ shape, error }) {
-    // Surface cause data in the response for specific error types
-    const causeData = error.cause as ErrorCauseData | undefined;
-    if (causeData && typeof causeData === 'object') {
-      return {
-        ...shape,
-        data: {
-          ...shape.data,
-          // Include structured error info from cause
-          ...(causeData.error && { error: causeData.error }),
-          ...(causeData.retryable !== undefined && { retryable: causeData.retryable }),
-        },
-      };
-    }
-    return shape;
+    return {
+      ...shape,
+      data: projectTrpcErrorData(shape.data, shape.message, error.cause),
+    };
   },
 });
 

services/cloud-agent-next/src/router/schemas.test.ts

@@ -387,17 +387,39 @@ describe('getMessageResult contract', () => {
         acceptedAt: 3,
         terminalAt: 4,
         completionSource: 'wrapper_failure',
-        failure: { stage: 'agent_activity', code: 'assistant_error', attempts: 2 },
+        failure: {
+          stage: 'agent_activity',
+          code: 'assistant_error',
+          attempts: 2,
+          retryable: false,
+        },
+      }).success
+    ).toBe(true);
+  });
+
+  it('requires retryability on failed and interrupted failure details', () => {
+    expect(
+      GetMessageResultOutput.safeParse({
+        ...baseOutput,
+        status: 'interrupted',
+        failure: { retryable: true },
       }).success
     ).toBe(true);
+    expect(
+      GetMessageResultOutput.safeParse({
+        ...baseOutput,
+        status: 'failed',
+        failure: { code: 'assistant_error' },
+      }).success
+    ).toBe(false);
   });
 
   it('fails closed on contradictory lifecycle result fields', () => {
     for (const output of [
       { ...baseOutput, status: 'queued', acceptedAt: 2 },
       { ...baseOutput, status: 'queued', terminalAt: 2 },
       { ...baseOutput, status: 'running', completionSource: 'assistant_message_event' },
-      { ...baseOutput, status: 'queued', failure: { attempts: 1 } },
+      { ...baseOutput, status: 'queued', failure: { attempts: 1, retryable: true } },
       { ...baseOutput, status: 'failed', assistant: { messageId: 'assistant_1', text: 'wrong' } },
       { ...baseOutput, status: 'interrupted', gateResult: 'fail' },
     ]) {

services/cloud-agent-next/src/router/schemas.ts

@@ -870,7 +870,7 @@ export const GetMessageResultOutput = z
     acceptedAt: z.number().optional(),
     terminalAt: z.number().optional(),
     completionSource: SessionMessageCompletionSourceSchema.optional(),
-    failure: SafeFailureProjectionSchema.optional(),
+    failure: SafeFailureProjectionSchema.extend({ retryable: z.boolean() }).optional(),
     gateResult: z.enum(['pass', 'fail']).optional(),
     assistant: z
       .object({