Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/20260831083812-session-metrics-fold.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
'@truefoundry/trueforge-core': patch
'@truefoundry/trueforge': patch
---

Fold session metrics totals on createTurn and terminal writes.
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,8 @@ export interface ISessionStore<
* row or leave `last_turn_id` pointing at a turn that was never created.
* The implementation supplies the mechanism (session lock, row lock/tx, …).
*
* Also bumps `session.last_activity_timestamp_ms` in that same atomic unit.
* Also bumps `session.last_activity_timestamp_ms` and increments
* `session.metrics.total_turns` in that same atomic unit.
*
* Fork semantics for `turn.previous_turn_id`:
* - `null` — new root turn (no parent); always allowed.
Expand All @@ -281,10 +282,8 @@ export interface ISessionStore<
createTurn(input: CreateTurnInput<TTurnCustom>): Promise<void>;

/**
* One tx: (a) conditionally cancel a running turn with `reason`; (b) if it
* did cancel, insert the caller-built `turn_done_event` — already-terminal
* turns skip the event insert; (c) return the now-immutable turn record.
* Missing turn → {@link TurnNotFoundError}.
* Cancel if still running (persist `turn_done` and fold cost/duration into
* `session.metrics`); already-terminal turns are a read. Missing → {@link TurnNotFoundError}.
*/
freezeAndGetTurn(input: FreezeAndGetTurnInput): Promise<TurnRecord<TTurnCustom>>;

Expand All @@ -297,15 +296,9 @@ export interface ISessionStore<
): Promise<{ data: TurnRecordWithoutSnapshot<TTurnCustom>[]; pagination: TokenPagination }>;

/**
* Writes the terminal state and `turn_done_event` atomically. Store contract —
* **first terminal write wins**:
* - Allowed: `running` → `done` | `cancelled` | `error`.
* - Rejected with **409** (or equivalent conflict): any write when status is already
* terminal — including done→cancelled, cancelled→done, error→*, terminal→running.
* - Missing turn → 404 / not-found.
*
* Check under the same concurrency control as other turn mutations (lock/CAS) —
* not a racy read-then-write outside the critical section.
* First terminal write wins (`running` → done/cancelled/error); otherwise 409.
* Winning write also folds cost/duration into `session.metrics`. Missing → 404.
* Must use the same lock/CAS as other turn mutations.
*/
updateTurnState(input: UpdateTurnStateInput): Promise<void>;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,7 @@ export class InMemorySessionStore<
this.events.set(tKey, []);
stored.turnIds.push(input.turn.turn_id);
stored.record.last_turn_id = input.turn.turn_id;
stored.record.metrics.total_turns += 1;
stored.record.last_activity_timestamp_ms = Date.now();
stored.record.updated_at = new Date();
if (input.update_session_title_if_not_exist !== null && stored.record.title === null) {
Expand All @@ -360,6 +361,7 @@ export class InMemorySessionStore<
if (list) {
list.push(deepCopy(input.turn_done_event));
}
this.addTerminalSessionMetrics(input.session_id, turn.created_at, cancelledState);
}

return deepCopy(turn);
Expand Down Expand Up @@ -402,6 +404,7 @@ export class InMemorySessionStore<
if (list) {
list.push(deepCopy(input.turn_done_event));
}
this.addTerminalSessionMetrics(input.session_id, turn.created_at, input.state);
}

async appendToEvents(input: AppendToEventsInput): Promise<void> {
Expand All @@ -415,6 +418,17 @@ export class InMemorySessionStore<
return;
}

/** Cost from turn metrics; duration is completed_at − created_at, floored at 0. */
private addTerminalSessionMetrics(sessionId: string, created_at: Date, state: TerminalTurnState): void {
const stored = this.sessions.get(sessionKey(sessionId));
if (!stored) {
throw new SessionNotFoundError(sessionId);
}
const elapsed_ms = Date.parse(state.completed_at) - created_at.getTime();
stored.record.metrics.total_cost_in_usd += state.metrics?.total_cost_in_usd ?? 0;
Comment thread
sr07asthana marked this conversation as resolved.
stored.record.metrics.total_duration_ms += elapsed_ms > 0 ? Math.trunc(elapsed_ms) : 0;
}

private requireTurn(sessionId: string, turnId: string): TurnRecord<TTurnCustom> {
const turn = this.turns.get(turnKey({ session_id: sessionId, turn_id: turnId }));
if (!turn) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -840,6 +840,24 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
expect(mustGet(after).last_activity_timestamp_ms).toBeGreaterThan(mustGet(before).last_activity_timestamp_ms);
});

it('increments session.metrics.total_turns without cost or duration', async () => {
const store = createStore();
await seedSession(store);
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
const afterFirst = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(afterFirst).metrics).toEqual({
total_cost_in_usd: 0,
total_duration_ms: 0,
total_turns: 1,
});
await finishTurn(store, 'turn-1');
await store.createTurn(
makeCreateTurnInput({ sessionId, turnId: 'turn-2', previousTurnId: 'turn-1', firstTurnId: 'turn-1' }),
);
const afterSecond = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(afterSecond).metrics.total_turns).toBe(2);
});

it('update_session_title_if_not_exist sets once and never overwrites', async () => {
const store = createStore();
await seedSession(store);
Expand Down Expand Up @@ -1322,6 +1340,38 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
});
});

it('folds duration into session.metrics when cancel applies, not on a second freeze', async () => {
Comment thread
sr07asthana marked this conversation as resolved.
const store = createStore();
await seedSession(store);
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
const cancelledState = makeCancelledTurnState(CancellationReason.CancelledForNextTurn);
const record = await store.freezeAndGetTurn({
session_id: sessionId,
turn_id: 'turn-1',
reason: CancellationReason.CancelledForNextTurn,
turn_done_event: makeTurnDoneEvent(cancelledState),
});
if (record.state.status !== 'cancelled') {
throw new Error(`expected cancelled turn, got ${record.state.status}`);
}
const afterCancel = await store.getSession({ tenant_id: tenant, session_id: sessionId });
const elapsed_ms = Date.parse(record.state.completed_at) - record.created_at.getTime();
expect(mustGet(afterCancel).metrics).toEqual({
total_cost_in_usd: 0,
total_duration_ms: elapsed_ms > 0 ? Math.trunc(elapsed_ms) : 0,
total_turns: 1,
});

await store.freezeAndGetTurn({
session_id: sessionId,
turn_id: 'turn-1',
reason: CancellationReason.CancelledForNextTurn,
turn_done_event: makeTurnDoneEvent(cancelledState),
});
const afterSecond = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(afterSecond).metrics).toEqual(mustGet(afterCancel).metrics);
});

it('on an already-terminal turn is a plain read without duplicating turn.done', async () => {
const store = createStore();
await seedSession(store);
Expand Down Expand Up @@ -1460,6 +1510,163 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
expect(doneEvents[0]).toEqual(turnDone);
});

it('adds cost and duration into session.metrics on running → terminal', async () => {
const store = createStore();
await seedSession(store);
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
const turn = await store.getTurn({ session_id: sessionId, turn_id: 'turn-1' });
const createdAt = mustGet(turn).created_at;
const completedAt = new Date(createdAt.getTime() + 1500).toISOString();
const state = {
...makeDoneTurnState(),
completed_at: completedAt,
metrics: { total_cost_in_usd: 1.25 },
};
await store.updateTurnState({
session_id: sessionId,
turn_id: 'turn-1',
state,
turn_done_event: makeTurnDoneEvent(state),
});
const session = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(session).metrics).toEqual({
total_cost_in_usd: 1.25,
total_duration_ms: 1500,
total_turns: 1,
});
});

it('does not add session.metrics again on a losing terminal write', async () => {
const store = createStore();
await seedSession(store);
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
const turn = await store.getTurn({ session_id: sessionId, turn_id: 'turn-1' });
const createdAt = mustGet(turn).created_at;
const doneState = {
...makeDoneTurnState(),
completed_at: new Date(createdAt.getTime() + 1500).toISOString(),
metrics: { total_cost_in_usd: 1.25 },
};
await store.updateTurnState({
session_id: sessionId,
turn_id: 'turn-1',
state: doneState,
turn_done_event: makeTurnDoneEvent(doneState),
});
const afterFirst = mustGet(await store.getSession({ tenant_id: tenant, session_id: sessionId })).metrics;

const losingState = {
...makeCancelledTurnState(CancellationReason.ClientCancelled),
completed_at: new Date(createdAt.getTime() + 8000).toISOString(),
metrics: { total_cost_in_usd: 9.99 },
};
await expect(
store.updateTurnState({
session_id: sessionId,
turn_id: 'turn-1',
state: losingState,
turn_done_event: makeTurnDoneEvent(losingState),
}),
).rejects.toBeInstanceOf(SessionStoreConflictError);

const afterSecond = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(afterSecond).metrics).toEqual(afterFirst);
expect(afterFirst).toEqual({
total_cost_in_usd: 1.25,
total_duration_ms: 1500,
total_turns: 1,
});
});

it('adds cost and duration from a second done turn onto existing session.metrics', async () => {
Comment thread
sr07asthana marked this conversation as resolved.
const store = createStore();
await seedSession(store);
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
const turn1 = await store.getTurn({ session_id: sessionId, turn_id: 'turn-1' });
const turn1Done = {
...makeDoneTurnState(),
completed_at: new Date(mustGet(turn1).created_at.getTime() + 1500).toISOString(),
metrics: { total_cost_in_usd: 1.25 },
};
await store.updateTurnState({
session_id: sessionId,
turn_id: 'turn-1',
state: turn1Done,
turn_done_event: makeTurnDoneEvent(turn1Done),
});

await store.createTurn(
makeCreateTurnInput({ sessionId, turnId: 'turn-2', previousTurnId: 'turn-1', firstTurnId: 'turn-1' }),
);
const turn2 = await store.getTurn({ session_id: sessionId, turn_id: 'turn-2' });
const turn2Done = {
...makeDoneTurnState(),
completed_at: new Date(mustGet(turn2).created_at.getTime() + 800).toISOString(),
metrics: { total_cost_in_usd: 0.5 },
};
await store.updateTurnState({
session_id: sessionId,
turn_id: 'turn-2',
state: turn2Done,
turn_done_event: makeTurnDoneEvent(turn2Done),
});

const session = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(session).metrics).toEqual({
total_cost_in_usd: 1.75,
total_duration_ms: 2300,
total_turns: 2,
});
});

it('accumulates session.metrics across turn1 done, turn2 cancel, and turn3 create', async () => {
const store = createStore();
await seedSession(store);

await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
const turn1 = await store.getTurn({ session_id: sessionId, turn_id: 'turn-1' });
const turn1Done = {
...makeDoneTurnState(),
completed_at: new Date(mustGet(turn1).created_at.getTime() + 1000).toISOString(),
metrics: { total_cost_in_usd: 1.0 },
};
await store.updateTurnState({
session_id: sessionId,
turn_id: 'turn-1',
state: turn1Done,
turn_done_event: makeTurnDoneEvent(turn1Done),
});

await store.createTurn(
makeCreateTurnInput({ sessionId, turnId: 'turn-2', previousTurnId: 'turn-1', firstTurnId: 'turn-1' }),
);
const cancelledState = makeCancelledTurnState(CancellationReason.CancelledForNextTurn);
const turn2 = await store.freezeAndGetTurn({
session_id: sessionId,
turn_id: 'turn-2',
reason: CancellationReason.CancelledForNextTurn,
turn_done_event: makeTurnDoneEvent(cancelledState),
});
if (turn2.state.status !== 'cancelled') {
throw new Error(`expected cancelled turn, got ${turn2.state.status}`);
}
const turn2DurationMs = Math.max(
0,
Math.trunc(Date.parse(turn2.state.completed_at) - turn2.created_at.getTime()),
);

await store.createTurn(
makeCreateTurnInput({ sessionId, turnId: 'turn-3', previousTurnId: 'turn-2', firstTurnId: 'turn-1' }),
);

const session = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(session).metrics).toEqual({
total_cost_in_usd: 1.0,
total_duration_ms: 1000 + turn2DurationMs,
total_turns: 3,
});
});

it('missing turn → not found', async () => {
const store = createStore();
await seedSession(store);
Expand Down
Loading