From 1db1d5ce885178e5f16d2680b41d21ef363b49cb Mon Sep 17 00:00:00 2001 From: Marcus <139352735+sucrammal@users.noreply.github.com> Date: Wed, 10 Jun 2026 16:03:38 -0400 Subject: [PATCH 1/4] Astro doc update Excluded AISettings since we're getting rid of it. --- docs/astro.config.mjs | 1 + .../docs/plugins/llm-scene-builder.mdx | 126 ++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 docs/src/content/docs/plugins/llm-scene-builder.mdx diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index d7d51d1ad..2140896f5 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -47,6 +47,7 @@ export default defineConfig({ { label: '', link: '/plugins/measure-tool/' }, { label: '', link: '/plugins/selection/' }, { label: '', link: '/plugins/skybox/' }, + { label: '', link: '/plugins/llm-scene-builder/' }, ], }, { diff --git a/docs/src/content/docs/plugins/llm-scene-builder.mdx b/docs/src/content/docs/plugins/llm-scene-builder.mdx new file mode 100644 index 000000000..25523bd09 --- /dev/null +++ b/docs/src/content/docs/plugins/llm-scene-builder.mdx @@ -0,0 +1,126 @@ +--- +title: +description: Natural-language frame editing — describe a spatial change in plain text and an LLM proposes the frame deltas. +--- + +import { Aside } from '@astrojs/starlight/components' + +`` adds a **Frame Builder** panel to the visualizer dashboard. You type a natural-language instruction ("Move the arm 200 mm forward along X"), the plugin calls your `onInfer` callback with the current frame state, and presents a diff of every proposed field change before anything is applied. The user confirms or cancels — no frame is mutated until confirmation. + +The plugin is model-agnostic: you wire in whatever LLM backend you prefer via the `onInfer` prop. + +## Usage + +```svelte + + +
+ + + +
+``` + +A robot-outline button appears in the dashboard. Clicking it opens the **Frame Builder** floating panel. Enter a prompt and press **Submit** (or `Enter`) — the panel shows a diff table while the LLM responds, then lets the user confirm or cancel. + +## Props + +| Prop | Type | Default | Description | +| ---------- | ---------------- | ------- | ------------------------------------------------------- | +| `onInfer` | `InferCallback` | — | **Required.** Called with the prompt and current frame state; must return proposed deltas and an explanation. | + +## InferCallback + +```ts +type InferCallback = ( + prompt: string, + components: ComponentFrameInfo[] +) => Promise<{ updates: FrameDelta[]; explanation: string }> +``` + +The plugin passes every component that has a frame defined: + +```ts +interface ComponentFrameInfo { + name: string + frame: { + parent: string | undefined + translation: { x?: number; y?: number; z?: number } | undefined + orientation: { roll: number; pitch: number; yaw: number } // degrees + } +} +``` + +Your callback should return: + +- **`updates`** — an array of `FrameDelta` objects describing changes (see below). Omit any field that should remain unchanged. +- **`explanation`** — a human-readable summary shown above the diff table. + +### FrameDelta + +```ts +interface FrameDelta { + componentName: string + translation?: { x?: number; y?: number; z?: number } // mm, absolute + orientation?: { roll?: number; pitch?: number; yaw?: number } // degrees, delta applied to current + parent?: string + explanation?: string // per-component note shown in the diff +} +``` + + + +The plugin validates every delta before showing the diff: unknown component names, self-referential parent assignments, and non-finite numbers are surfaced as errors without blocking the rest of the update batch. + +## Example: using the Anthropic SDK + +The companion `` component (see below) stores an Anthropic API key in the browser under `settings.anthropicKey`. A minimal server-side route using the Anthropic SDK: + +```ts +// src/routes/api/infer-frames/+server.ts (SvelteKit) +import Anthropic from '@anthropic-ai/sdk' + +export async function POST({ request }) { + const { prompt, components } = await request.json() + + const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY }) + + const message = await client.messages.create({ + model: 'claude-sonnet-4-6', + max_tokens: 1024, + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: `Current robot frame configuration (JSON):\n${JSON.stringify(components, null, 2)}\n\nUser request: ${prompt}\n\nRespond with JSON matching the schema: { updates: FrameDelta[], explanation: string }`, + }, + ], + }, + ], + }) + + const text = message.content.find((b) => b.type === 'text')?.text ?? '{}' + return new Response(text, { headers: { 'Content-Type': 'application/json' } }) +} +``` + From 5ded6936d536281011cff663b32f2ed0a5949f18 Mon Sep 17 00:00:00 2001 From: Marcus <139352735+sucrammal@users.noreply.github.com> Date: Wed, 10 Jun 2026 16:25:16 -0400 Subject: [PATCH 2/4] lint --- docs/src/content/docs/plugins/llm-scene-builder.mdx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/content/docs/plugins/llm-scene-builder.mdx b/docs/src/content/docs/plugins/llm-scene-builder.mdx index 25523bd09..4580e5fee 100644 --- a/docs/src/content/docs/plugins/llm-scene-builder.mdx +++ b/docs/src/content/docs/plugins/llm-scene-builder.mdx @@ -41,9 +41,9 @@ A robot-outline button appears in the dashboard. Clicking it opens the **Frame B ## Props -| Prop | Type | Default | Description | -| ---------- | ---------------- | ------- | ------------------------------------------------------- | -| `onInfer` | `InferCallback` | — | **Required.** Called with the prompt and current frame state; must return proposed deltas and an explanation. | +| Prop | Type | Default | Description | +| --------- | --------------- | ------- | ------------------------------------------------------------------------------------------------------------- | +| `onInfer` | `InferCallback` | — | **Required.** Called with the prompt and current frame state; must return proposed deltas and an explanation. | ## InferCallback @@ -85,7 +85,8 @@ interface FrameDelta { ``` The plugin validates every delta before showing the diff: unknown component names, self-referential parent assignments, and non-finite numbers are surfaced as errors without blocking the rest of the update batch. @@ -123,4 +124,3 @@ export async function POST({ request }) { return new Response(text, { headers: { 'Content-Type': 'application/json' } }) } ``` - From eb45195e36aaae2c38e0a215b10af7e4f6d0a883 Mon Sep 17 00:00:00 2001 From: Marcus <139352735+sucrammal@users.noreply.github.com> Date: Wed, 10 Jun 2026 17:27:39 -0400 Subject: [PATCH 3/4] Get rid of other references to AISettings --- docs/src/content/docs/plugins/llm-scene-builder.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/content/docs/plugins/llm-scene-builder.mdx b/docs/src/content/docs/plugins/llm-scene-builder.mdx index 4580e5fee..13015f16a 100644 --- a/docs/src/content/docs/plugins/llm-scene-builder.mdx +++ b/docs/src/content/docs/plugins/llm-scene-builder.mdx @@ -93,7 +93,7 @@ The plugin validates every delta before showing the diff: unknown component name ## Example: using the Anthropic SDK -The companion `` component (see below) stores an Anthropic API key in the browser under `settings.anthropicKey`. A minimal server-side route using the Anthropic SDK: +Set `ANTHROPIC_API_KEY` in your environment (see `.env.example`). A minimal server-side route using the Anthropic SDK: ```ts // src/routes/api/infer-frames/+server.ts (SvelteKit) From 191c62f5f4901503b94cf0a24c591fb353e280ed Mon Sep 17 00:00:00 2001 From: Marcus <139352735+sucrammal@users.noreply.github.com> Date: Mon, 15 Jun 2026 12:30:11 -0400 Subject: [PATCH 4/4] Move frame builder deps to dev deps --- package.json | 12 +++++------- pnpm-lock.yaml | 30 +++++++++++++++--------------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/package.json b/package.json index 271ed744d..6642a9129 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "@changesets/cli": "2.29.6", "@connectrpc/connect": "1.7.0", "@connectrpc/connect-web": "1.7.0", + "@langchain/anthropic": "^1.4.0", "@dimforge/rapier3d-compat": "0.18.2", "@eslint/compat": "2.0.2", "@eslint/js": "10.0.1", @@ -115,11 +116,13 @@ "type-fest": "^5.0.1", "typescript": "5.9.2", "typescript-eslint": "8.56.1", + "uuid-tool": "^2.0.3", "vite": "7.3.2", "vite-plugin-devtools-json": "1.0.0", "vite-plugin-glsl": "^1.5.5", "vite-plugin-mkcert": "1.17.9", - "vitest": "3.2.6" + "vitest": "3.2.6", + "zod": "^4.4.3" }, "peerDependencies": { "@ag-grid-community/client-side-row-model": ">=32.3.0", @@ -209,16 +212,11 @@ ], "dependencies": { "@bufbuild/protobuf": "1.10.1", - "@connectrpc/connect": "1.7.0", - "@connectrpc/connect-web": "1.7.0", - "@langchain/anthropic": "^1.4.0", "@neodrag/svelte": "^2.3.3", "d3-force": "^3.0.0", "filtrex": "^3.1.0", "koota": "0.6.5", "lodash-es": "4.18.1", - "three-mesh-bvh": "^0.9.8", - "uuid-tool": "^2.0.3", - "zod": "^4.4.3" + "three-mesh-bvh": "^0.9.8" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ea9be5e92..be9b2501d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,15 +15,6 @@ importers: '@bufbuild/protobuf': specifier: 1.10.1 version: 1.10.1 - '@connectrpc/connect': - specifier: 1.7.0 - version: 1.7.0(@bufbuild/protobuf@1.10.1) - '@connectrpc/connect-web': - specifier: 1.7.0 - version: 1.7.0(@bufbuild/protobuf@1.10.1)(@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.1)) - '@langchain/anthropic': - specifier: ^1.4.0 - version: 1.4.0(@langchain/core@1.1.48(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@2.2.0(@opentelemetry/api@1.9.0))(ws@8.21.0)) '@neodrag/svelte': specifier: ^2.3.3 version: 2.3.3(svelte@5.55.7) @@ -45,12 +36,6 @@ importers: three-mesh-bvh: specifier: ^0.9.8 version: 0.9.8(three@0.183.2) - uuid-tool: - specifier: ^2.0.3 - version: 2.0.3 - zod: - specifier: ^4.4.3 - version: 4.4.3 devDependencies: '@ag-grid-community/client-side-row-model': specifier: 32.3.9 @@ -64,6 +49,12 @@ importers: '@changesets/cli': specifier: 2.29.6 version: 2.29.6(@types/node@25.6.0) + '@connectrpc/connect': + specifier: 1.7.0 + version: 1.7.0(@bufbuild/protobuf@1.10.1) + '@connectrpc/connect-web': + specifier: 1.7.0 + version: 1.7.0(@bufbuild/protobuf@1.10.1)(@connectrpc/connect@1.7.0(@bufbuild/protobuf@1.10.1)) '@dimforge/rapier3d-compat': specifier: 0.18.2 version: 0.18.2 @@ -73,6 +64,9 @@ importers: '@eslint/js': specifier: 10.0.1 version: 10.0.1(eslint@10.0.2(jiti@2.6.1)) + '@langchain/anthropic': + specifier: ^1.4.0 + version: 1.4.0(@langchain/core@1.1.48(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@2.2.0(@opentelemetry/api@1.9.0))(ws@8.21.0)) '@playwright/test': specifier: 1.55.1 version: 1.55.1 @@ -274,6 +268,9 @@ importers: typescript-eslint: specifier: 8.56.1 version: 8.56.1(eslint@10.0.2(jiti@2.6.1))(typescript@5.9.2) + uuid-tool: + specifier: ^2.0.3 + version: 2.0.3 vite: specifier: 7.3.2 version: 7.3.2(@types/node@25.6.0)(jiti@2.6.1)(lightningcss@1.30.1)(tsx@4.20.5)(yaml@2.8.1) @@ -289,6 +286,9 @@ importers: vitest: specifier: 3.2.6 version: 3.2.6(@types/node@25.6.0)(@vitest/browser@3.2.4)(jiti@2.6.1)(jsdom@26.1.0)(lightningcss@1.30.1)(tsx@4.20.5)(yaml@2.8.1) + zod: + specifier: ^4.4.3 + version: 4.4.3 packages: