From f5928453887653d0d50daae5813b8b29262ab98c Mon Sep 17 00:00:00 2001 From: George Ng Date: Wed, 8 Jul 2026 14:30:59 -0700 Subject: [PATCH 01/29] Update nfa compiler --- .../actionGrammar/src/grammarOptimizer.ts | 28 ++++ ts/packages/actionGrammar/src/index.ts | 5 +- ts/packages/actionGrammar/src/nfaCompiler.ts | 94 +++++++++++-- .../test/nfaFactoredPrefixValues.spec.ts | 131 ++++++++++++++++++ .../src/commands/compile.ts | 11 +- 5 files changed, 257 insertions(+), 12 deletions(-) create mode 100644 ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index ea2c85c49d..0b09becb9b 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -263,6 +263,34 @@ export const recommendedOptimizations: GrammarOptimizationOptions = { promoteTailRulesParts: true, }; +/** + * NFA-compatible preset: every pass in `recommendedOptimizations` except + * `tailFactoring` and `promoteTailRulesParts`. Those two passes emit + * `RulesPart.tailCall` nodes that only the AST-walking matcher + * (`grammarMatcher.ts`) understands - the NFA compiler + * (`nfaCompiler.ts`) rejects them outright. + * + * `factorCommonPrefixes` (without `tailFactoring`) still factors shared + * leading parts across alternatives, but emits the factored suffix as a + * plain (non-tail) `RulesPart` bound to a synthesized wrapper variable + * (e.g. `__opt_factor_0`), with the parent rule's own `value` either set + * explicitly to reference that variable (when `dispatchifyAlternations` + * finalizes the rule) or left implicit for the compiler's single + * variable-bearing-part forwarding rule to resolve (see + * `isSingleVariableRule` / the multi-part forwarding check in + * `nfaCompiler.ts`). Both shapes compile and match correctly on the NFA + * path. + * + * Use this preset (instead of `recommendedOptimizations`) for any + * grammar that will be compiled to an NFA - which, since `nfa` is the + * default `grammarSystem` for agent-server, is the common case. + */ +export const nfaSafeOptimizations: GrammarOptimizationOptions = { + inlineSingleAlternatives: true, + factorCommonPrefixes: true, + dispatchifyAlternations: true, +}; + /** * Run enabled optimization passes against the compiled grammar AST. * The returned grammar is semantically equivalent to the input - only the diff --git a/ts/packages/actionGrammar/src/index.ts b/ts/packages/actionGrammar/src/index.ts index 87a7a15192..61d3e9013b 100644 --- a/ts/packages/actionGrammar/src/index.ts +++ b/ts/packages/actionGrammar/src/index.ts @@ -17,7 +17,10 @@ export { grammarToJson } from "./grammarSerializer.js"; export { loadGrammarRules, loadGrammarRulesNoThrow } from "./grammarLoader.js"; export type { LoadGrammarRulesOptions } from "./grammarLoader.js"; export type { GrammarOptimizationOptions } from "./grammarOptimizer.js"; -export { recommendedOptimizations } from "./grammarOptimizer.js"; +export { + recommendedOptimizations, + nfaSafeOptimizations, +} from "./grammarOptimizer.js"; export type { SchemaLoader, DebugInfoCollector, diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index c1e0f86525..ec59030abb 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -473,6 +473,35 @@ function isSingleVariableRule(rule: GrammarRule): { variable: string } | false { return false; } +/** + * For a rule with no explicit value expression and more than one part, + * find the single variable-bearing part (if exactly one exists) so its + * value can be forwarded as the rule's effective value. + * + * Mirrors the matcher's implicit-default forwarding rule + * (`getImplicitDefaultValue` in grammarOptimizer.ts), which the grammar + * optimizer's `factorCommonPrefixes` pass relies on: a shared-prefix + * rule factored to `[, RulesPart(variable: "w")]` has no value + * of its own - the matcher (and, with this helper, the NFA compiler) + * forwards whatever value the bound `RulesPart` produces at match time. + * + * Returns `undefined` when zero or more-than-one parts carry a variable + * - the caller treats "more than one" as a genuine ambiguity error and + * "zero" as "no implicit value derivable here". + */ +function findSingleVariableBearingPart( + rule: GrammarRule, +): { variable: string } | "ambiguous" | undefined { + let found: string | undefined; + for (const part of rule.parts) { + const name = part.variable; + if (name === undefined) continue; + if (found !== undefined) return "ambiguous"; + found = name; + } + return found !== undefined ? { variable: found } : undefined; +} + /** * Collect all variable names from a rule's parts * Returns them in order of appearance @@ -614,15 +643,6 @@ export function compileGrammarToNFA(grammar: Grammar, name?: string): NFA { ) { const rule = normalizedGrammar.alternatives[ruleIndex]; - // VALIDATION: Multi-term rules MUST have value expressions - // Single-term rules can omit value expressions (they inherit from the term) - if (rule.parts.length > 1 && !rule.value) { - throw new Error( - `Grammar rule at index ${ruleIndex} has ${rule.parts.length} terms but no value expression. ` + - `Multi-term rules must have an explicit value expression (using ->).`, - ); - } - // Check for single-variable rules like = $(x:wildcard); // These should implicitly produce their variable's value: -> x let effectiveValue = rule.value; @@ -630,6 +650,44 @@ export function compileGrammarToNFA(grammar: Grammar, name?: string): NFA { const singleVar = isSingleVariableRule(rule); if (singleVar) { effectiveValue = { type: "variable", name: singleVar.variable }; + } else if (rule.parts.length > 1) { + // Multi-term rule with no explicit value: mirror the + // matcher's implicit-default forwarding rule (see + // `findSingleVariableBearingPart`) - this is the shape + // produced by the grammar optimizer's + // `factorCommonPrefixes` pass (`nfaSafeOptimizations` + // preset) when it factors a shared optional prefix out + // of alternatives that each carry their own `->` value. + const singleVarPart = findSingleVariableBearingPart(rule); + if (singleVarPart === "ambiguous") { + throw new Error( + `Grammar rule at index ${ruleIndex} has ${rule.parts.length} terms but no value expression, ` + + `and more than one part carries a variable - the implicit value is ambiguous. ` + + `Multi-term rules must have an explicit value expression (using ->).`, + ); + } + if (singleVarPart !== undefined) { + effectiveValue = { + type: "variable", + name: singleVarPart.variable, + }; + } else { + // VALIDATION: Multi-term rules MUST have a value + // expression, either explicit or implicitly derivable + // from exactly one variable-bearing part. + const hasTailCall = rule.parts.some( + (p) => p.type === "rules" && p.tailCall, + ); + throw new Error( + `Grammar rule at index ${ruleIndex} has ${rule.parts.length} terms but no value expression. ` + + `Multi-term rules must have an explicit value expression (using ->).` + + (hasTailCall + ? " This rule contains a tailCall RulesPart, which the NFA compiler " + + "does not support - disable `tailFactoring` / `promoteTailRulesParts` " + + "in the grammar optimizer (use `nfaSafeOptimizations`) for NFA/DFA paths." + : ""), + ); + } } } @@ -1535,6 +1593,24 @@ function compileRulesPartWithSlots( const singleVar = isSingleVariableRule(rule); if (singleVar) { effectiveValue = { type: "variable", name: singleVar.variable }; + } else if (rule.parts.length > 1) { + // Mirror the matcher's implicit-default forwarding rule + // (see `findSingleVariableBearingPart`) for nested + // multi-term rules produced by `factorCommonPrefixes`. + const singleVarPart = findSingleVariableBearingPart(rule); + if (singleVarPart === "ambiguous") { + throw new Error( + `Nested grammar rule has ${rule.parts.length} terms but no value expression, ` + + `and more than one part carries a variable - the implicit value is ambiguous. ` + + `Multi-term rules must have an explicit value expression (using ->).`, + ); + } + if (singleVarPart !== undefined) { + effectiveValue = { + type: "variable", + name: singleVarPart.variable, + }; + } } } diff --git a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts new file mode 100644 index 0000000000..c53cfc438d --- /dev/null +++ b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Regression coverage for the "shared optional prefix breaks NFA +// compilation" bug: when every top-level alternative shares a common +// optional leading rule (e.g. an optional politeness lead-in), the +// grammar optimizer's `factorCommonPrefixes` pass hoists that prefix +// out, leaving a value-less multi-term rule whose remaining value +// lives on a nested `RulesPart`. The NFA compiler used to reject this +// shape outright (`nfaCompiler.ts:619`) even though the AST-walking +// matcher evaluates it correctly. +// +// Fix: (A) the NFA compiler now derives an implicit forwarding value +// for multi-term rules with exactly one variable-bearing part, mirroring +// the matcher's own implicit-default rule; (B) `nfaSafeOptimizations` +// (recommendedOptimizations minus `tailFactoring` / +// `promoteTailRulesParts`) avoids emitting the `tailCall` RulesParts the +// NFA compiler still can't support, so the two changes work together. + +import { loadGrammarRules } from "../src/grammarLoader.js"; +import { compileGrammarToNFA } from "../src/nfaCompiler.js"; +import { matchNFA } from "../src/nfaInterpreter.js"; +import { + recommendedOptimizations, + nfaSafeOptimizations, +} from "../src/grammarOptimizer.js"; +import { Grammar, createStringPart } from "../src/grammarTypes.js"; + +describe("NFA compilation of factored shared-prefix grammars", () => { + // Minimal repro straight from the bug report: every alternative + // shares a common optional prefix rule

. + const agr = ` + = | ; + =

foo -> { actionName: "A" }; + =

bar -> { actionName: "B" }; +

= (please)?; +`; + + it("compiles and matches correctly with nfaSafeOptimizations", () => { + const grammar = loadGrammarRules("factored-prefix.agr", agr, { + optimizations: nfaSafeOptimizations, + }); + + const nfa = compileGrammarToNFA(grammar, "factored-prefix"); + + expect(matchNFA(nfa, ["foo"], true).matched).toBe(true); + expect(matchNFA(nfa, ["foo"], true).actionValue).toEqual({ + actionName: "A", + }); + + expect(matchNFA(nfa, ["please", "foo"], true).matched).toBe(true); + expect(matchNFA(nfa, ["please", "foo"], true).actionValue).toEqual({ + actionName: "A", + }); + + expect(matchNFA(nfa, ["bar"], true).matched).toBe(true); + expect(matchNFA(nfa, ["bar"], true).actionValue).toEqual({ + actionName: "B", + }); + + expect(matchNFA(nfa, ["please", "bar"], true).matched).toBe(true); + expect(matchNFA(nfa, ["please", "bar"], true).actionValue).toEqual({ + actionName: "B", + }); + + // Unrelated input must still fail. + expect(matchNFA(nfa, ["baz"], true).matched).toBe(false); + }); + + it("still refuses tailCall RulesParts from recommendedOptimizations, with a clear message", () => { + // recommendedOptimizations enables tailFactoring / + // promoteTailRulesParts, which the NFA compiler does not (and + // by design cannot cheaply) support. This documents the + // remaining, intentional limitation so a future attempt to + // silently "fix" it here doesn't mask a real incompatibility. + const grammar = loadGrammarRules("factored-prefix.agr", agr, { + optimizations: recommendedOptimizations, + }); + + expect(() => + compileGrammarToNFA(grammar, "factored-prefix-tail"), + ).toThrow(/tailCall RulesPart/); + }); + + it("derives an implicit value for a manually-authored factored rule", () => { + // Same shape as above, but hand-written (bypassing the + // optimizer) to directly exercise the NFA compiler's new + // implicit-value derivation for multi-term, value-less rules. + const agr2 = ` + = | ; + = (please)? $(w:) -> w; + = | ; + = foo -> { actionName: "A" }; + = bar -> { actionName: "B" }; + = never_matches -> { actionName: "unused" }; +`; + const grammar = loadGrammarRules("manual-factored.agr", agr2, {}); + const nfa = compileGrammarToNFA(grammar, "manual-factored"); + + expect(matchNFA(nfa, ["foo"], true).actionValue).toEqual({ + actionName: "A", + }); + expect(matchNFA(nfa, ["please", "bar"], true).actionValue).toEqual({ + actionName: "B", + }); + }); + + it("throws a clear error when the implicit value is genuinely ambiguous", () => { + // Two variable-bearing parts with no top-level value: neither + // the matcher nor the NFA compiler can determine which one's + // value should be forwarded. Built as a raw AST (bypassing the + // parser/loader's own stricter "start rule must produce a + // value" validation) to exercise the NFA compiler's defensive + // check in isolation - this shape should never be reachable + // through the normal parse+optimize pipeline, but the compiler + // must still fail loudly and clearly if it ever is. + const grammar: Grammar = { + alternatives: [ + { + parts: [ + createStringPart(["foo"], "x"), + createStringPart(["bar"], "y"), + ], + }, + ], + }; + expect(() => compileGrammarToNFA(grammar, "ambiguous")).toThrow( + /implicit value is ambiguous/, + ); + }); +}); diff --git a/ts/packages/actionGrammarCompiler/src/commands/compile.ts b/ts/packages/actionGrammarCompiler/src/commands/compile.ts index 79274e2e21..a0229e7814 100644 --- a/ts/packages/actionGrammarCompiler/src/commands/compile.ts +++ b/ts/packages/actionGrammarCompiler/src/commands/compile.ts @@ -7,7 +7,7 @@ import fs from "node:fs"; import { grammarToJson, loadGrammarRulesNoThrow, - recommendedOptimizations, + nfaSafeOptimizations, SchemaLoader, } from "@typeagent/action-grammar"; import { parseSchemaSource } from "@typeagent/action-schema"; @@ -86,7 +86,14 @@ export default class Compile extends Command { : { startValueRequired: true, schemaLoader, - optimizations: recommendedOptimizations, + // agent-server defaults to `grammarSystem: "nfa"`, so + // agc must emit an optimized AST the NFA compiler can + // consume. `nfaSafeOptimizations` keeps every safe + // pass (inlining, prefix-factoring, dispatch) but + // omits `tailFactoring` / `promoteTailRulesParts`, + // whose `RulesPart.tailCall` output only the + // AST-walking matcher understands. + optimizations: nfaSafeOptimizations, }, ); From 332df3aae2461217e9de3ea55bb90925b3ce10f0 Mon Sep 17 00:00:00 2001 From: George Ng Date: Wed, 8 Jul 2026 14:52:03 -0700 Subject: [PATCH 02/29] Cleanup comments --- .../actionGrammar/src/grammarOptimizer.ts | 25 +++-------- ts/packages/actionGrammar/src/nfaCompiler.ts | 36 ++++----------- .../test/nfaFactoredPrefixValues.spec.ts | 44 +++++-------------- .../src/commands/compile.ts | 9 +--- 4 files changed, 26 insertions(+), 88 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index 0b09becb9b..6eb2f04352 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -264,26 +264,11 @@ export const recommendedOptimizations: GrammarOptimizationOptions = { }; /** - * NFA-compatible preset: every pass in `recommendedOptimizations` except - * `tailFactoring` and `promoteTailRulesParts`. Those two passes emit - * `RulesPart.tailCall` nodes that only the AST-walking matcher - * (`grammarMatcher.ts`) understands - the NFA compiler - * (`nfaCompiler.ts`) rejects them outright. - * - * `factorCommonPrefixes` (without `tailFactoring`) still factors shared - * leading parts across alternatives, but emits the factored suffix as a - * plain (non-tail) `RulesPart` bound to a synthesized wrapper variable - * (e.g. `__opt_factor_0`), with the parent rule's own `value` either set - * explicitly to reference that variable (when `dispatchifyAlternations` - * finalizes the rule) or left implicit for the compiler's single - * variable-bearing-part forwarding rule to resolve (see - * `isSingleVariableRule` / the multi-part forwarding check in - * `nfaCompiler.ts`). Both shapes compile and match correctly on the NFA - * path. - * - * Use this preset (instead of `recommendedOptimizations`) for any - * grammar that will be compiled to an NFA - which, since `nfa` is the - * default `grammarSystem` for agent-server, is the common case. + * NFA-compatible preset: all of `recommendedOptimizations` except + * `tailFactoring` and `promoteTailRulesParts`, which emit + * `RulesPart.tailCall` nodes the NFA compiler rejects. Use this preset + * for any grammar that will be compiled to an NFA (the default + * `grammarSystem` for agent-server). */ export const nfaSafeOptimizations: GrammarOptimizationOptions = { inlineSingleAlternatives: true, diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index ec59030abb..3a8370e186 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -474,20 +474,11 @@ function isSingleVariableRule(rule: GrammarRule): { variable: string } | false { } /** - * For a rule with no explicit value expression and more than one part, - * find the single variable-bearing part (if exactly one exists) so its - * value can be forwarded as the rule's effective value. - * - * Mirrors the matcher's implicit-default forwarding rule - * (`getImplicitDefaultValue` in grammarOptimizer.ts), which the grammar - * optimizer's `factorCommonPrefixes` pass relies on: a shared-prefix - * rule factored to `[, RulesPart(variable: "w")]` has no value - * of its own - the matcher (and, with this helper, the NFA compiler) - * forwards whatever value the bound `RulesPart` produces at match time. - * - * Returns `undefined` when zero or more-than-one parts carry a variable - * - the caller treats "more than one" as a genuine ambiguity error and - * "zero" as "no implicit value derivable here". + * For a value-less multi-part rule, find its single variable-bearing + * part so that part's value can be forwarded as the rule's effective + * value (mirrors `getImplicitDefaultValue` in grammarOptimizer.ts). + * Returns `"ambiguous"` if more than one part carries a variable, or + * `undefined` if none do. */ function findSingleVariableBearingPart( rule: GrammarRule, @@ -651,13 +642,9 @@ export function compileGrammarToNFA(grammar: Grammar, name?: string): NFA { if (singleVar) { effectiveValue = { type: "variable", name: singleVar.variable }; } else if (rule.parts.length > 1) { - // Multi-term rule with no explicit value: mirror the - // matcher's implicit-default forwarding rule (see - // `findSingleVariableBearingPart`) - this is the shape - // produced by the grammar optimizer's - // `factorCommonPrefixes` pass (`nfaSafeOptimizations` - // preset) when it factors a shared optional prefix out - // of alternatives that each carry their own `->` value. + // factorCommonPrefixes (nfaSafeOptimizations) can leave + // a multi-part rule with no top-level value; forward + // the single variable-bearing part's value if possible. const singleVarPart = findSingleVariableBearingPart(rule); if (singleVarPart === "ambiguous") { throw new Error( @@ -672,9 +659,6 @@ export function compileGrammarToNFA(grammar: Grammar, name?: string): NFA { name: singleVarPart.variable, }; } else { - // VALIDATION: Multi-term rules MUST have a value - // expression, either explicit or implicitly derivable - // from exactly one variable-bearing part. const hasTailCall = rule.parts.some( (p) => p.type === "rules" && p.tailCall, ); @@ -1594,9 +1578,7 @@ function compileRulesPartWithSlots( if (singleVar) { effectiveValue = { type: "variable", name: singleVar.variable }; } else if (rule.parts.length > 1) { - // Mirror the matcher's implicit-default forwarding rule - // (see `findSingleVariableBearingPart`) for nested - // multi-term rules produced by `factorCommonPrefixes`. + // Same implicit-value forwarding as above, for nested rules. const singleVarPart = findSingleVariableBearingPart(rule); if (singleVarPart === "ambiguous") { throw new Error( diff --git a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts index c53cfc438d..4e230a5f66 100644 --- a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts +++ b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts @@ -1,21 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// Regression coverage for the "shared optional prefix breaks NFA -// compilation" bug: when every top-level alternative shares a common -// optional leading rule (e.g. an optional politeness lead-in), the -// grammar optimizer's `factorCommonPrefixes` pass hoists that prefix -// out, leaving a value-less multi-term rule whose remaining value -// lives on a nested `RulesPart`. The NFA compiler used to reject this -// shape outright (`nfaCompiler.ts:619`) even though the AST-walking -// matcher evaluates it correctly. -// -// Fix: (A) the NFA compiler now derives an implicit forwarding value -// for multi-term rules with exactly one variable-bearing part, mirroring -// the matcher's own implicit-default rule; (B) `nfaSafeOptimizations` -// (recommendedOptimizations minus `tailFactoring` / -// `promoteTailRulesParts`) avoids emitting the `tailCall` RulesParts the -// NFA compiler still can't support, so the two changes work together. +// Regression coverage: grammars where every top-level alternative +// shares a common optional prefix used to fail to compile to an NFA +// once `factorCommonPrefixes` hoisted the shared prefix out, leaving a +// value-less multi-term rule. See nfaCompiler.ts (implicit value +// derivation) and grammarOptimizer.ts (`nfaSafeOptimizations`). import { loadGrammarRules } from "../src/grammarLoader.js"; import { compileGrammarToNFA } from "../src/nfaCompiler.js"; @@ -27,8 +17,7 @@ import { import { Grammar, createStringPart } from "../src/grammarTypes.js"; describe("NFA compilation of factored shared-prefix grammars", () => { - // Minimal repro straight from the bug report: every alternative - // shares a common optional prefix rule

. + // Every alternative shares a common optional prefix rule

. const agr = ` = | ; =

foo -> { actionName: "A" }; @@ -67,12 +56,8 @@ describe("NFA compilation of factored shared-prefix grammars", () => { expect(matchNFA(nfa, ["baz"], true).matched).toBe(false); }); - it("still refuses tailCall RulesParts from recommendedOptimizations, with a clear message", () => { - // recommendedOptimizations enables tailFactoring / - // promoteTailRulesParts, which the NFA compiler does not (and - // by design cannot cheaply) support. This documents the - // remaining, intentional limitation so a future attempt to - // silently "fix" it here doesn't mask a real incompatibility. + it("still refuses tailCall RulesParts from recommendedOptimizations", () => { + // tailFactoring/promoteTailRulesParts still aren't NFA-compatible. const grammar = loadGrammarRules("factored-prefix.agr", agr, { optimizations: recommendedOptimizations, }); @@ -83,9 +68,6 @@ describe("NFA compilation of factored shared-prefix grammars", () => { }); it("derives an implicit value for a manually-authored factored rule", () => { - // Same shape as above, but hand-written (bypassing the - // optimizer) to directly exercise the NFA compiler's new - // implicit-value derivation for multi-term, value-less rules. const agr2 = ` = | ; = (please)? $(w:) -> w; @@ -106,14 +88,8 @@ describe("NFA compilation of factored shared-prefix grammars", () => { }); it("throws a clear error when the implicit value is genuinely ambiguous", () => { - // Two variable-bearing parts with no top-level value: neither - // the matcher nor the NFA compiler can determine which one's - // value should be forwarded. Built as a raw AST (bypassing the - // parser/loader's own stricter "start rule must produce a - // value" validation) to exercise the NFA compiler's defensive - // check in isolation - this shape should never be reachable - // through the normal parse+optimize pipeline, but the compiler - // must still fail loudly and clearly if it ever is. + // Raw AST (bypasses the loader's own validation) with two + // variable-bearing parts and no top-level value. const grammar: Grammar = { alternatives: [ { diff --git a/ts/packages/actionGrammarCompiler/src/commands/compile.ts b/ts/packages/actionGrammarCompiler/src/commands/compile.ts index a0229e7814..d5a21e3990 100644 --- a/ts/packages/actionGrammarCompiler/src/commands/compile.ts +++ b/ts/packages/actionGrammarCompiler/src/commands/compile.ts @@ -86,13 +86,8 @@ export default class Compile extends Command { : { startValueRequired: true, schemaLoader, - // agent-server defaults to `grammarSystem: "nfa"`, so - // agc must emit an optimized AST the NFA compiler can - // consume. `nfaSafeOptimizations` keeps every safe - // pass (inlining, prefix-factoring, dispatch) but - // omits `tailFactoring` / `promoteTailRulesParts`, - // whose `RulesPart.tailCall` output only the - // AST-walking matcher understands. + // NFA is agent-server's default grammar system; + // this preset avoids tailCall RulesParts it can't compile. optimizations: nfaSafeOptimizations, }, ); From 920b970068e930a8b08ef7d4a8e393f647dfac6b Mon Sep 17 00:00:00 2001 From: George Ng Date: Wed, 8 Jul 2026 17:26:29 -0700 Subject: [PATCH 03/29] Refactor to reduce complexity --- ts/packages/actionGrammar/src/nfaCompiler.ts | 127 +++++++++---------- 1 file changed, 62 insertions(+), 65 deletions(-) diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index 3a8370e186..c4ef73db7d 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -11,6 +11,7 @@ import { RulesPart, PhraseSetPart, CompiledSpacingMode, + CompiledValueNode, getCapturedVariableName, createRulesPart, } from "./grammarTypes.js"; @@ -493,6 +494,56 @@ function findSingleVariableBearingPart( return found !== undefined ? { variable: found } : undefined; } +/** + * Derive the value expression for a rule that has no explicit `->` value: + * single-variable rules forward that variable, and factored multi-part + * rules (from `nfaSafeOptimizations`) forward their single variable-bearing + * part's value. Throws if ambiguous (2+ variable-bearing parts); if + * `requireValue` is set, also throws when no value can be derived at all. + */ +function deriveEffectiveValue( + rule: GrammarRule, + describeRule: () => string, + requireValue: boolean, +): CompiledValueNode | undefined { + if (rule.value) { + return rule.value; + } + const singleVar = isSingleVariableRule(rule); + if (singleVar) { + return { type: "variable", name: singleVar.variable }; + } + if (rule.parts.length <= 1) { + return undefined; + } + const singleVarPart = findSingleVariableBearingPart(rule); + if (singleVarPart === "ambiguous") { + throw new Error( + `${describeRule()} has ${rule.parts.length} terms but no value expression, ` + + `and more than one part carries a variable - the implicit value is ambiguous. ` + + `Multi-term rules must have an explicit value expression (using ->).`, + ); + } + if (singleVarPart !== undefined) { + return { type: "variable", name: singleVarPart.variable }; + } + if (!requireValue) { + return undefined; + } + const hasTailCall = rule.parts.some( + (p) => p.type === "rules" && p.tailCall, + ); + throw new Error( + `${describeRule()} has ${rule.parts.length} terms but no value expression. ` + + `Multi-term rules must have an explicit value expression (using ->).` + + (hasTailCall + ? " This rule contains a tailCall RulesPart, which the NFA compiler " + + "does not support - disable `tailFactoring` / `promoteTailRulesParts` " + + "in the grammar optimizer (use `nfaSafeOptimizations`) for NFA/DFA paths." + : ""), + ); +} + /** * Collect all variable names from a rule's parts * Returns them in order of appearance @@ -634,46 +685,12 @@ export function compileGrammarToNFA(grammar: Grammar, name?: string): NFA { ) { const rule = normalizedGrammar.alternatives[ruleIndex]; - // Check for single-variable rules like = $(x:wildcard); - // These should implicitly produce their variable's value: -> x - let effectiveValue = rule.value; - if (!effectiveValue) { - const singleVar = isSingleVariableRule(rule); - if (singleVar) { - effectiveValue = { type: "variable", name: singleVar.variable }; - } else if (rule.parts.length > 1) { - // factorCommonPrefixes (nfaSafeOptimizations) can leave - // a multi-part rule with no top-level value; forward - // the single variable-bearing part's value if possible. - const singleVarPart = findSingleVariableBearingPart(rule); - if (singleVarPart === "ambiguous") { - throw new Error( - `Grammar rule at index ${ruleIndex} has ${rule.parts.length} terms but no value expression, ` + - `and more than one part carries a variable - the implicit value is ambiguous. ` + - `Multi-term rules must have an explicit value expression (using ->).`, - ); - } - if (singleVarPart !== undefined) { - effectiveValue = { - type: "variable", - name: singleVarPart.variable, - }; - } else { - const hasTailCall = rule.parts.some( - (p) => p.type === "rules" && p.tailCall, - ); - throw new Error( - `Grammar rule at index ${ruleIndex} has ${rule.parts.length} terms but no value expression. ` + - `Multi-term rules must have an explicit value expression (using ->).` + - (hasTailCall - ? " This rule contains a tailCall RulesPart, which the NFA compiler " + - "does not support - disable `tailFactoring` / `promoteTailRulesParts` " + - "in the grammar optimizer (use `nfaSafeOptimizations`) for NFA/DFA paths." - : ""), - ); - } - } - } + // Check for single-variable / factored rules with no explicit -> value. + const effectiveValue = deriveEffectiveValue( + rule, + () => `Grammar rule at index ${ruleIndex}`, + true, + ); const ruleEntry = builder.createState(false); @@ -1570,31 +1587,11 @@ function compileRulesPartWithSlots( // Compile and store the action value on the entry state FIRST // We need to know if there's a value before deciding about environments - // Passthrough normalization is done upfront, so rules already have explicit values - // Just check for single-variable rules like = $(x:wildcard); - let effectiveValue = rule.value; - if (!effectiveValue) { - const singleVar = isSingleVariableRule(rule); - if (singleVar) { - effectiveValue = { type: "variable", name: singleVar.variable }; - } else if (rule.parts.length > 1) { - // Same implicit-value forwarding as above, for nested rules. - const singleVarPart = findSingleVariableBearingPart(rule); - if (singleVarPart === "ambiguous") { - throw new Error( - `Nested grammar rule has ${rule.parts.length} terms but no value expression, ` + - `and more than one part carries a variable - the implicit value is ambiguous. ` + - `Multi-term rules must have an explicit value expression (using ->).`, - ); - } - if (singleVarPart !== undefined) { - effectiveValue = { - type: "variable", - name: singleVarPart.variable, - }; - } - } - } + const effectiveValue = deriveEffectiveValue( + rule, + () => "Nested grammar rule", + false, + ); let compiledValue: ValueExpression | undefined; let nestedCompletionActionName: string | undefined; From 1b9cbb3a262f1446a5cbe15443c2913e74af8e03 Mon Sep 17 00:00:00 2001 From: George Ng Date: Wed, 8 Jul 2026 17:38:14 -0700 Subject: [PATCH 04/29] Refresh complexity baseline exceptions after merge Line-number shifts from the NFA fix broke file:line matches for pre-existing baselined offenders (nfaCompiler.ts, grammarOptimizer.ts). Regenerated via 'npm run code-complexity:update-exceptions'. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../code/complexity-baseline-exception.json | 76 +++++++------------ 1 file changed, 29 insertions(+), 47 deletions(-) diff --git a/ts/tools/scripts/code/complexity-baseline-exception.json b/ts/tools/scripts/code/complexity-baseline-exception.json index 55012549ef..9d5b9c0d7d 100644 --- a/ts/tools/scripts/code/complexity-baseline-exception.json +++ b/ts/tools/scripts/code/complexity-baseline-exception.json @@ -1,18 +1,18 @@ { - "generatedAt": "2026-07-08T00:19:36.857Z", + "generatedAt": "2026-07-09T00:34:30.183Z", "sourceReport": "tools/scripts/code/complexity-report/report.json", - "branch": "installsource", - "commit": "87a2da2b552af067c7685847c256d76dbe5974de", + "branch": "dev/georgeng/fix_nfa_with_subrules", + "commit": "ed60c05a48dd275c7a1914a155bca5403b6ae1e0", "thresholds": { "cyclomatic": 25, "cognitive": 30 }, "totals": { - "functionsAnalyzed": 26973, + "functionsAnalyzed": 27001, "filesAnalyzed": 1952, - "overCyclomatic": 204, - "overCognitive": 258, - "totalExceptions": 301 + "overCyclomatic": 203, + "overCognitive": 254, + "totalExceptions": 299 }, "exceptions": [ { @@ -899,7 +899,7 @@ }, { "file": "packages/actionGrammar/src/nfaCompiler.ts", - "line": 118, + "line": 119, "name": "Function 'collectVariablePaths'", "cyclomatic": 35, "cognitive": 48, @@ -1151,7 +1151,7 @@ }, { "file": "packages/actionGrammar/src/grammarOptimizer.ts", - "line": 715, + "line": 728, "name": "Function 'tryInlineRulesPart'", "cyclomatic": 32, "cognitive": 35, @@ -1239,15 +1239,6 @@ "overCyclomatic": true, "overCognitive": false }, - { - "file": "extensions/agr-language/webview-dist/debugPanel.js", - "line": 502, - "name": "Arrow function", - "cyclomatic": 30, - "cognitive": 83, - "overCyclomatic": true, - "overCognitive": true - }, { "file": "packages/dispatcher/dispatcher/src/translation/translateRequest.ts", "line": 257, @@ -1338,6 +1329,15 @@ "overCyclomatic": true, "overCognitive": false }, + { + "file": "extensions/agr-language/webview-dist/debugPanel.js", + "line": 15, + "name": "Arrow function", + "cyclomatic": 30, + "cognitive": 4, + "overCyclomatic": true, + "overCognitive": false + }, { "file": "tools/docsAutogen/src/cli.ts", "line": 453, @@ -1392,15 +1392,6 @@ "overCyclomatic": true, "overCognitive": true }, - { - "file": "extensions/agr-language/webview-dist/debugPanel.js", - "line": 2204, - "name": "Method '_derivedTraceData'", - "cyclomatic": 29, - "cognitive": 33, - "overCyclomatic": true, - "overCognitive": true - }, { "file": "packages/agents/browser/src/agent/knowledge/utils/graphologyLayoutEngine.mts", "line": 523, @@ -1419,6 +1410,15 @@ "overCyclomatic": true, "overCognitive": false }, + { + "file": "extensions/agr-language/webview-dist/debugPanel.js", + "line": 624, + "name": "Method '_derivedTraceData'", + "cyclomatic": 29, + "cognitive": 2, + "overCyclomatic": true, + "overCognitive": false + }, { "file": "packages/commandExecutor/src/config/agentServerConfig.ts", "line": 210, @@ -1626,15 +1626,6 @@ "overCyclomatic": true, "overCognitive": true }, - { - "file": "packages/actionGrammar/src/nfaCompiler.ts", - "line": 1467, - "name": "Function 'compileRulesPartWithSlots'", - "cyclomatic": 27, - "cognitive": 35, - "overCyclomatic": true, - "overCognitive": true - }, { "file": "packages/config/src/runtime/build.ts", "line": 810, @@ -1718,7 +1709,7 @@ }, { "file": "packages/actionGrammar/src/grammarOptimizer.ts", - "line": 2419, + "line": 2432, "name": "Function 'walkPartForKeys'", "cyclomatic": 26, "cognitive": 42, @@ -2328,15 +2319,6 @@ "overCyclomatic": false, "overCognitive": true }, - { - "file": "extensions/agr-language/webview-dist/debugPanel.js", - "line": 562, - "name": "Constructor", - "cyclomatic": 20, - "cognitive": 66, - "overCyclomatic": false, - "overCognitive": true - }, { "file": "examples/workflow/model/src/validate.ts", "line": 669, @@ -2627,7 +2609,7 @@ }, { "file": "packages/actionGrammar/src/grammarOptimizer.ts", - "line": 3503, + "line": 3516, "name": "Function 'walk'", "cyclomatic": 15, "cognitive": 45, From 0f2ae5cbbc2fc405c2399608a4e0c1ce0e2022f1 Mon Sep 17 00:00:00 2001 From: George Ng Date: Thu, 9 Jul 2026 15:45:38 -0700 Subject: [PATCH 05/29] Address comments --- .../actionGrammar/src/grammarOptimizer.ts | 45 ++++++++------- ts/packages/actionGrammar/src/nfaCompiler.ts | 29 ++-------- .../test/nfaFactoredPrefixValues.spec.ts | 55 +++++++++---------- .../src/commands/compile.ts | 12 +++- 4 files changed, 65 insertions(+), 76 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index 6eb2f04352..8cb0a5cd2d 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -3327,31 +3327,34 @@ function getImplicitDefaultValue( rule: GrammarRule, ): CompiledValueNode | undefined { if (rule.value !== undefined) return rule.value; - const parts = rule.parts; - if (parts.length === 0) return undefined; - if (parts.length === 1) { - // Single-part rule: matcher forwards the part's value. For - // var-bearing parts we can express that as a variable - // reference; for unbound `string` / `phraseSet` (whose - // matcher value derives from the matched text) and unbound - // `rules` (whose value derives from the inner match) we - // can't reify the result without changing the AST, so bail. - const name = parts[0].variable; - return name !== undefined ? { type: "variable", name } : undefined; - } - // Multi-part: implicit default requires exactly one - // var-bearing part. Same predicate as the inliner's - // binding-friendly check for multi-part children. - let theVar: string | undefined; + if (rule.parts.length === 0) return undefined; + const result = findSingleValueBearingPart(rule.parts); + return result !== "ambiguous" && result !== undefined + ? { type: "variable", name: result.variable } + : undefined; +} + +/** + * Find the single part (if any) that carries a variable across a rule's + * parts, so its value can stand in for the whole rule's implicit value + * when the rule has no explicit `->` expression. Returns `"ambiguous"` + * when 2+ parts carry a variable, or `undefined` when none do. + * + * Shared by `getImplicitDefaultValue` (above) and the NFA compiler's + * equivalent forwarding logic for factored rules (see + * `deriveEffectiveValue` in nfaCompiler.ts). + */ +export function findSingleValueBearingPart( + parts: GrammarPart[], +): { variable: string } | "ambiguous" | undefined { + let found: string | undefined; for (const p of parts) { const name = p.variable; if (name === undefined) continue; - if (theVar !== undefined) return undefined; - theVar = name; + if (found !== undefined) return "ambiguous"; + found = name; } - return theVar !== undefined - ? { type: "variable", name: theVar } - : undefined; + return found !== undefined ? { variable: found } : undefined; } // ───────────────────────────────────────────────────────────────────────────── diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index c4ef73db7d..32fbe95cae 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -22,6 +22,7 @@ import { ValueExpression, } from "./environment.js"; import { normalizeToken } from "./nfaMatcher.js"; +import { findSingleValueBearingPart } from "./grammarOptimizer.js"; // Scripts that require a word-boundary separator between adjacent tokens. // CJK and other logographic/syllabic scripts are NOT included — no separator needed. @@ -474,32 +475,14 @@ function isSingleVariableRule(rule: GrammarRule): { variable: string } | false { return false; } -/** - * For a value-less multi-part rule, find its single variable-bearing - * part so that part's value can be forwarded as the rule's effective - * value (mirrors `getImplicitDefaultValue` in grammarOptimizer.ts). - * Returns `"ambiguous"` if more than one part carries a variable, or - * `undefined` if none do. - */ -function findSingleVariableBearingPart( - rule: GrammarRule, -): { variable: string } | "ambiguous" | undefined { - let found: string | undefined; - for (const part of rule.parts) { - const name = part.variable; - if (name === undefined) continue; - if (found !== undefined) return "ambiguous"; - found = name; - } - return found !== undefined ? { variable: found } : undefined; -} - /** * Derive the value expression for a rule that has no explicit `->` value: * single-variable rules forward that variable, and factored multi-part * rules (from `nfaSafeOptimizations`) forward their single variable-bearing - * part's value. Throws if ambiguous (2+ variable-bearing parts); if - * `requireValue` is set, also throws when no value can be derived at all. + * part's value (via `findSingleValueBearingPart`, shared with the + * optimizer's `getImplicitDefaultValue`). Throws if ambiguous (2+ + * variable-bearing parts); if `requireValue` is set, also throws when no + * value can be derived at all. */ function deriveEffectiveValue( rule: GrammarRule, @@ -516,7 +499,7 @@ function deriveEffectiveValue( if (rule.parts.length <= 1) { return undefined; } - const singleVarPart = findSingleVariableBearingPart(rule); + const singleVarPart = findSingleValueBearingPart(rule.parts); if (singleVarPart === "ambiguous") { throw new Error( `${describeRule()} has ${rule.parts.length} terms but no value expression, ` + diff --git a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts index 4e230a5f66..d42d07f301 100644 --- a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts +++ b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts @@ -15,6 +15,7 @@ import { nfaSafeOptimizations, } from "../src/grammarOptimizer.js"; import { Grammar, createStringPart } from "../src/grammarTypes.js"; +import { describeForEachMatcher } from "./testUtils.js"; describe("NFA compilation of factored shared-prefix grammars", () => { // Every alternative shares a common optional prefix rule

. @@ -25,36 +26,32 @@ describe("NFA compilation of factored shared-prefix grammars", () => {

= (please)?; `; - it("compiles and matches correctly with nfaSafeOptimizations", () => { - const grammar = loadGrammarRules("factored-prefix.agr", agr, { - optimizations: nfaSafeOptimizations, - }); - - const nfa = compileGrammarToNFA(grammar, "factored-prefix"); + describeForEachMatcher( + "compiles and matches correctly with nfaSafeOptimizations", + (testMatchGrammar) => { + it("matches each branch, with and without the shared prefix", () => { + const grammar = loadGrammarRules("factored-prefix.agr", agr, { + optimizations: nfaSafeOptimizations, + }); - expect(matchNFA(nfa, ["foo"], true).matched).toBe(true); - expect(matchNFA(nfa, ["foo"], true).actionValue).toEqual({ - actionName: "A", - }); - - expect(matchNFA(nfa, ["please", "foo"], true).matched).toBe(true); - expect(matchNFA(nfa, ["please", "foo"], true).actionValue).toEqual({ - actionName: "A", - }); + expect(testMatchGrammar(grammar, "foo")).toStrictEqual([ + { actionName: "A" }, + ]); + expect(testMatchGrammar(grammar, "please foo")).toStrictEqual([ + { actionName: "A" }, + ]); + expect(testMatchGrammar(grammar, "bar")).toStrictEqual([ + { actionName: "B" }, + ]); + expect(testMatchGrammar(grammar, "please bar")).toStrictEqual([ + { actionName: "B" }, + ]); - expect(matchNFA(nfa, ["bar"], true).matched).toBe(true); - expect(matchNFA(nfa, ["bar"], true).actionValue).toEqual({ - actionName: "B", - }); - - expect(matchNFA(nfa, ["please", "bar"], true).matched).toBe(true); - expect(matchNFA(nfa, ["please", "bar"], true).actionValue).toEqual({ - actionName: "B", - }); - - // Unrelated input must still fail. - expect(matchNFA(nfa, ["baz"], true).matched).toBe(false); - }); + // Unrelated input must still fail. + expect(testMatchGrammar(grammar, "baz")).toStrictEqual([]); + }); + }, + ); it("still refuses tailCall RulesParts from recommendedOptimizations", () => { // tailFactoring/promoteTailRulesParts still aren't NFA-compatible. @@ -64,7 +61,7 @@ describe("NFA compilation of factored shared-prefix grammars", () => { expect(() => compileGrammarToNFA(grammar, "factored-prefix-tail"), - ).toThrow(/tailCall RulesPart/); + ).toThrow(/tail/i); }); it("derives an implicit value for a manually-authored factored rule", () => { diff --git a/ts/packages/actionGrammarCompiler/src/commands/compile.ts b/ts/packages/actionGrammarCompiler/src/commands/compile.ts index d5a21e3990..5da398a07d 100644 --- a/ts/packages/actionGrammarCompiler/src/commands/compile.ts +++ b/ts/packages/actionGrammarCompiler/src/commands/compile.ts @@ -8,6 +8,7 @@ import { grammarToJson, loadGrammarRulesNoThrow, nfaSafeOptimizations, + recommendedOptimizations, SchemaLoader, } from "@typeagent/action-grammar"; import { parseSchemaSource } from "@typeagent/action-schema"; @@ -68,6 +69,11 @@ export default class Compile extends Command { "Disable grammar optimizations (produces an unoptimized AST that preserves the 1:1 correspondence between top-level rules and the original source — useful for diagnostics).", default: false, }), + "nfa-safe": Flags.boolean({ + description: + "Use the NFA-compatible optimization preset (excludes tailCall-producing passes) instead of the default recommended optimizations.", + default: false, + }), }; async run(): Promise { @@ -86,9 +92,9 @@ export default class Compile extends Command { : { startValueRequired: true, schemaLoader, - // NFA is agent-server's default grammar system; - // this preset avoids tailCall RulesParts it can't compile. - optimizations: nfaSafeOptimizations, + optimizations: flags["nfa-safe"] + ? nfaSafeOptimizations + : recommendedOptimizations, }, ); From b18a579456cbf45a4d5e678205ad2d83740cc417 Mon Sep 17 00:00:00 2001 From: George Ng Date: Thu, 9 Jul 2026 16:28:22 -0700 Subject: [PATCH 06/29] Add additional test --- .../test/nfaFactoredPrefixValues.spec.ts | 49 ++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts index d42d07f301..fae8339f55 100644 --- a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts +++ b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts @@ -6,6 +6,12 @@ // once `factorCommonPrefixes` hoisted the shared prefix out, leaving a // value-less multi-term rule. See nfaCompiler.ts (implicit value // derivation) and grammarOptimizer.ts (`nfaSafeOptimizations`). +// +// Note: `factorCommonPrefixes`'s own output already stamps an +// explicit `value` on the factored top-level rule, so grammar-source +// tests below don't reach the new implicit-derivation code path. The +// hand-built-AST test further down is what directly exercises +// `findSingleValueBearingPart`'s forwarding logic. import { loadGrammarRules } from "../src/grammarLoader.js"; import { compileGrammarToNFA } from "../src/nfaCompiler.js"; @@ -14,7 +20,11 @@ import { recommendedOptimizations, nfaSafeOptimizations, } from "../src/grammarOptimizer.js"; -import { Grammar, createStringPart } from "../src/grammarTypes.js"; +import { + Grammar, + createStringPart, + createRulesPart, +} from "../src/grammarTypes.js"; import { describeForEachMatcher } from "./testUtils.js"; describe("NFA compilation of factored shared-prefix grammars", () => { @@ -64,7 +74,7 @@ describe("NFA compilation of factored shared-prefix grammars", () => { ).toThrow(/tail/i); }); - it("derives an implicit value for a manually-authored factored rule", () => { + it("evaluates an explicit -> value expression through a manually-factored shape", () => { const agr2 = ` = | ; = (please)? $(w:) -> w; @@ -84,6 +94,41 @@ describe("NFA compilation of factored shared-prefix grammars", () => { }); }); + it("forwards the single variable-bearing part's value for a value-less multi-term rule", () => { + // Hand-built AST mirroring the actual bug shape: a value-less + // 2-part rule (value-less optional prefix + a variable-bearing + // nested RulesPart), with no top-level `value`. The grammar-source + // tests above don't exercise this path because + // `factorCommonPrefixes` and explicit `->` expressions both + // already stamp `rule.value`, so this is the only test that + // directly reaches `findSingleValueBearingPart`'s forwarding case. + const prefixPart = createRulesPart( + [{ parts: [createStringPart(["please"])] }], + { optional: true }, + ); + const nestedPart = createRulesPart( + [ + { + parts: [createStringPart(["foo"])], + value: { type: "literal", value: "A" }, + }, + { + parts: [createStringPart(["bar"])], + value: { type: "literal", value: "B" }, + }, + ], + { variable: "chosen" }, + ); + const grammar: Grammar = { + alternatives: [{ parts: [prefixPart, nestedPart] }], + }; + + const nfa = compileGrammarToNFA(grammar, "hand-built-factored"); + + expect(matchNFA(nfa, ["foo"], true).actionValue).toBe("A"); + expect(matchNFA(nfa, ["please", "bar"], true).actionValue).toBe("B"); + }); + it("throws a clear error when the implicit value is genuinely ambiguous", () => { // Raw AST (bypasses the loader's own validation) with two // variable-bearing parts and no top-level value. From b1bde99a591da732f56bce892f0b9cff63d25821 Mon Sep 17 00:00:00 2001 From: George Ng Date: Thu, 9 Jul 2026 22:53:18 -0700 Subject: [PATCH 07/29] Remove duplicate tests --- .../test/nfaFactoredPrefixValues.spec.ts | 42 ++++--------------- 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts index fae8339f55..af7b0bbef6 100644 --- a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts +++ b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts @@ -12,14 +12,17 @@ // tests below don't reach the new implicit-derivation code path. The // hand-built-AST test further down is what directly exercises // `findSingleValueBearingPart`'s forwarding logic. +// +// NFA's rejection of tailCall RulesParts (still unsupported; see +// `nfaSafeOptimizations`) is already covered by +// grammarOptimizerTailFactoring.spec.ts ("NFA compile of a +// tail-factored grammar throws a descriptive error") - not duplicated +// here. import { loadGrammarRules } from "../src/grammarLoader.js"; import { compileGrammarToNFA } from "../src/nfaCompiler.js"; import { matchNFA } from "../src/nfaInterpreter.js"; -import { - recommendedOptimizations, - nfaSafeOptimizations, -} from "../src/grammarOptimizer.js"; +import { nfaSafeOptimizations } from "../src/grammarOptimizer.js"; import { Grammar, createStringPart, @@ -63,37 +66,6 @@ describe("NFA compilation of factored shared-prefix grammars", () => { }, ); - it("still refuses tailCall RulesParts from recommendedOptimizations", () => { - // tailFactoring/promoteTailRulesParts still aren't NFA-compatible. - const grammar = loadGrammarRules("factored-prefix.agr", agr, { - optimizations: recommendedOptimizations, - }); - - expect(() => - compileGrammarToNFA(grammar, "factored-prefix-tail"), - ).toThrow(/tail/i); - }); - - it("evaluates an explicit -> value expression through a manually-factored shape", () => { - const agr2 = ` - = | ; - = (please)? $(w:) -> w; - = | ; - = foo -> { actionName: "A" }; - = bar -> { actionName: "B" }; - = never_matches -> { actionName: "unused" }; -`; - const grammar = loadGrammarRules("manual-factored.agr", agr2, {}); - const nfa = compileGrammarToNFA(grammar, "manual-factored"); - - expect(matchNFA(nfa, ["foo"], true).actionValue).toEqual({ - actionName: "A", - }); - expect(matchNFA(nfa, ["please", "bar"], true).actionValue).toEqual({ - actionName: "B", - }); - }); - it("forwards the single variable-bearing part's value for a value-less multi-term rule", () => { // Hand-built AST mirroring the actual bug shape: a value-less // 2-part rule (value-less optional prefix + a variable-bearing From 20082d505940a9428616429c7ccec6d05bcf5fd6 Mon Sep 17 00:00:00 2001 From: George Ng Date: Thu, 9 Jul 2026 23:21:06 -0700 Subject: [PATCH 08/29] Update comments and fix behavior regression --- .../actionGrammar/src/grammarOptimizer.ts | 10 ++++++++-- ts/packages/actionGrammar/src/nfaCompiler.ts | 16 +++++++++++++--- .../src/commands/compile.ts | 5 +++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index 8cb0a5cd2d..c73c173f8d 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -267,8 +267,14 @@ export const recommendedOptimizations: GrammarOptimizationOptions = { * NFA-compatible preset: all of `recommendedOptimizations` except * `tailFactoring` and `promoteTailRulesParts`, which emit * `RulesPart.tailCall` nodes the NFA compiler rejects. Use this preset - * for any grammar that will be compiled to an NFA (the default - * `grammarSystem` for agent-server). + * for any grammar that will be compiled to an NFA/DFA path (note: the + * default `grammarSystem` for agent-server is `completionBased`/AST, not + * NFA - this preset only matters when NFA/DFA is explicitly selected). + * + * TODO(#nfa-tail-support): Once the NFA compiler supports tailCall + * RulesParts natively, this preset (and the `agc compile --nfa-safe` + * flag that exposes it) should be removed in favor of always using + * `recommendedOptimizations`. */ export const nfaSafeOptimizations: GrammarOptimizationOptions = { inlineSingleAlternatives: true, diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index 32fbe95cae..cda8281ed0 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -480,9 +480,11 @@ function isSingleVariableRule(rule: GrammarRule): { variable: string } | false { * single-variable rules forward that variable, and factored multi-part * rules (from `nfaSafeOptimizations`) forward their single variable-bearing * part's value (via `findSingleValueBearingPart`, shared with the - * optimizer's `getImplicitDefaultValue`). Throws if ambiguous (2+ - * variable-bearing parts); if `requireValue` is set, also throws when no - * value can be derived at all. + * optimizer's `getImplicitDefaultValue`). Not every rule needs a value - + * only when `requireValue` is set (i.e. the value is actually consumed: + * top-level action rules, or nested rules captured by a parent variable) + * do ambiguous (2+ variable-bearing parts) or missing values throw; + * otherwise both cases just resolve to `undefined`. */ function deriveEffectiveValue( rule: GrammarRule, @@ -501,6 +503,14 @@ function deriveEffectiveValue( } const singleVarPart = findSingleValueBearingPart(rule.parts); if (singleVarPart === "ambiguous") { + // Not every rule needs a value - only rules whose value is actually + // consumed (top-level action rules, or nested rules captured by a + // parent variable) do. If nothing downstream needs this rule's + // value, ambiguity is harmless: just report "no value", matching + // the `singleVarPart === undefined` case below. + if (!requireValue) { + return undefined; + } throw new Error( `${describeRule()} has ${rule.parts.length} terms but no value expression, ` + `and more than one part carries a variable - the implicit value is ambiguous. ` + diff --git a/ts/packages/actionGrammarCompiler/src/commands/compile.ts b/ts/packages/actionGrammarCompiler/src/commands/compile.ts index 5da398a07d..3d9b01ffc1 100644 --- a/ts/packages/actionGrammarCompiler/src/commands/compile.ts +++ b/ts/packages/actionGrammarCompiler/src/commands/compile.ts @@ -69,6 +69,11 @@ export default class Compile extends Command { "Disable grammar optimizations (produces an unoptimized AST that preserves the 1:1 correspondence between top-level rules and the original source — useful for diagnostics).", default: false, }), + // TODO(#nfa-tail-support): Remove this flag once the NFA compiler + // supports tailCall-producing optimizations (tailFactoring / + // promoteTailRulesParts) directly, at which point + // `recommendedOptimizations` alone should be NFA-safe and this + // opt-in preset becomes unnecessary. "nfa-safe": Flags.boolean({ description: "Use the NFA-compatible optimization preset (excludes tailCall-producing passes) instead of the default recommended optimizations.", From ab2bbaeb16b2f994af2eae17527ea97e260b400d Mon Sep 17 00:00:00 2001 From: George Ng Date: Thu, 9 Jul 2026 23:37:28 -0700 Subject: [PATCH 09/29] Clean up logic --- ts/packages/actionGrammar/src/nfaCompiler.ts | 24 ++++++++------------ 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index cda8281ed0..19df5ca2ea 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -502,27 +502,23 @@ function deriveEffectiveValue( return undefined; } const singleVarPart = findSingleValueBearingPart(rule.parts); + if (singleVarPart !== undefined && singleVarPart !== "ambiguous") { + return { type: "variable", name: singleVarPart.variable }; + } + // Not every rule needs a value - only rules whose value is actually + // consumed (top-level action rules, or nested rules captured by a + // parent variable) do. If nothing downstream needs this rule's value, + // neither an ambiguous nor a missing implicit value is an error. + if (!requireValue) { + return undefined; + } if (singleVarPart === "ambiguous") { - // Not every rule needs a value - only rules whose value is actually - // consumed (top-level action rules, or nested rules captured by a - // parent variable) do. If nothing downstream needs this rule's - // value, ambiguity is harmless: just report "no value", matching - // the `singleVarPart === undefined` case below. - if (!requireValue) { - return undefined; - } throw new Error( `${describeRule()} has ${rule.parts.length} terms but no value expression, ` + `and more than one part carries a variable - the implicit value is ambiguous. ` + `Multi-term rules must have an explicit value expression (using ->).`, ); } - if (singleVarPart !== undefined) { - return { type: "variable", name: singleVarPart.variable }; - } - if (!requireValue) { - return undefined; - } const hasTailCall = rule.parts.some( (p) => p.type === "rules" && p.tailCall, ); From 7461207c9b117f3c2faf10b3f938f4b75c9cd061 Mon Sep 17 00:00:00 2001 From: George Ng Date: Fri, 10 Jul 2026 01:31:58 -0700 Subject: [PATCH 10/29] Update documents --- ts/docs/architecture/core/actionGrammar.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/ts/docs/architecture/core/actionGrammar.md b/ts/docs/architecture/core/actionGrammar.md index 55b3e64c1b..a6a5e85315 100644 --- a/ts/docs/architecture/core/actionGrammar.md +++ b/ts/docs/architecture/core/actionGrammar.md @@ -605,7 +605,14 @@ tail contract, so the non-tail wrapper builder is only reached when NFA-interpreter matcher (`grammarMatcher.ts`). The NFA compiler / DFA path (`nfaCompiler.ts`) explicitly throws on encountering one, so consumers that route through the NFA/DFA path must leave -`tailFactoring` off. +`tailFactoring` off (and `promoteTailRulesParts`, which produces the +same tail-call shape). `grammarOptimizer.ts` exports +`nfaSafeOptimizations` as the ready-made preset for this - all of +`recommendedOptimizations` except those two passes - and `agc compile +--nfa-safe` exposes it from the CLI. This is an interim opt-in: once +the NFA compiler supports tailCall RulesParts natively, `agc compile` +should go back to always using `recommendedOptimizations` and +`nfaSafeOptimizations` / `--nfa-safe` should be removed. **No fixed-point loop.** Factoring is applied once per group of alternatives — the trie's grouping converges in a single pass and @@ -1518,6 +1525,16 @@ matched text as a string. For example, `"hello"` produces `"hello"`. explicit value expression produce no value — the compiler warns about this because the output is ambiguous. +The NFA compiler (`nfaCompiler.ts`) mirrors this same "no value unless +actually needed" principle for rules reshaped by `factorCommonPrefixes` +(see `nfaSafeOptimizations` above): `deriveEffectiveValue` forwards a +factored rule's single variable-bearing part, and only treats an +ambiguous or missing implicit value as a hard error when the value is +actually required — i.e. for top-level action rules, or nested rules a +parent captures via `$(name:)`. A nested, uncaptured rule with an +ambiguous shape is harmless (nothing reads its value) and silently +resolves to "no value," exactly like the `none` kind above. + ### Design Principles 1. **Strict Conformance** — the purpose of type checking is to ensure From a108a4a70b5c7b1ce8ca5230c7ed4be9e999bd1a Mon Sep 17 00:00:00 2001 From: George Ng Date: Fri, 10 Jul 2026 16:32:53 -0700 Subject: [PATCH 11/29] Remove unnecessary --nfa-safe flag and extract shared implicitValue extractor --- ts/docs/architecture/core/actionGrammar.md | 13 +- .../src/grammarImplicitExtractor.ts | 78 ++++++++++ .../actionGrammar/src/grammarOptimizer.ts | 78 +++------- ts/packages/actionGrammar/src/index.ts | 5 +- ts/packages/actionGrammar/src/nfaCompiler.ts | 31 ++-- .../test/grammarImplicitExtractor.spec.ts | 137 ++++++++++++++++++ .../test/nfaFactoredPrefixValues.spec.ts | 17 +-- .../src/commands/compile.ts | 15 +- 8 files changed, 263 insertions(+), 111 deletions(-) create mode 100644 ts/packages/actionGrammar/src/grammarImplicitExtractor.ts create mode 100644 ts/packages/actionGrammar/test/grammarImplicitExtractor.spec.ts diff --git a/ts/docs/architecture/core/actionGrammar.md b/ts/docs/architecture/core/actionGrammar.md index a6a5e85315..45f92f9d56 100644 --- a/ts/docs/architecture/core/actionGrammar.md +++ b/ts/docs/architecture/core/actionGrammar.md @@ -606,13 +606,8 @@ NFA-interpreter matcher (`grammarMatcher.ts`). The NFA compiler / DFA path (`nfaCompiler.ts`) explicitly throws on encountering one, so consumers that route through the NFA/DFA path must leave `tailFactoring` off (and `promoteTailRulesParts`, which produces the -same tail-call shape). `grammarOptimizer.ts` exports -`nfaSafeOptimizations` as the ready-made preset for this - all of -`recommendedOptimizations` except those two passes - and `agc compile ---nfa-safe` exposes it from the CLI. This is an interim opt-in: once -the NFA compiler supports tailCall RulesParts natively, `agc compile` -should go back to always using `recommendedOptimizations` and -`nfaSafeOptimizations` / `--nfa-safe` should be removed. +same tail-call shape) when calling `optimizeGrammar` / +`loadGrammarRules` directly. **No fixed-point loop.** Factoring is applied once per group of alternatives — the trie's grouping converges in a single pass and @@ -1526,8 +1521,8 @@ explicit value expression produce no value — the compiler warns about this because the output is ambiguous. The NFA compiler (`nfaCompiler.ts`) mirrors this same "no value unless -actually needed" principle for rules reshaped by `factorCommonPrefixes` -(see `nfaSafeOptimizations` above): `deriveEffectiveValue` forwards a +actually needed" principle for rules reshaped by `factorCommonPrefixes`: +`deriveEffectiveValue` forwards a factored rule's single variable-bearing part, and only treats an ambiguous or missing implicit value as a hard error when the value is actually required — i.e. for top-level action rules, or nested rules a diff --git a/ts/packages/actionGrammar/src/grammarImplicitExtractor.ts b/ts/packages/actionGrammar/src/grammarImplicitExtractor.ts new file mode 100644 index 0000000000..16f77b9027 --- /dev/null +++ b/ts/packages/actionGrammar/src/grammarImplicitExtractor.ts @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Structural derivation of a rule's *implicit* value expression when it + * has no explicit `->` expression. + * + * A rule with exactly one variable-bearing part implicitly forwards that + * part's value - this mirrors the AST matcher's implicit-default rule + * (see grammarMatcher.ts) and is the single source of truth shared by: + * - grammarOptimizer.ts's `getImplicitDefaultValue`, used by the + * value-substitution branch of `tryPromoteTrailing` to fold each + * member's effective value into the parent's value expression. + * - nfaCompiler.ts's `deriveEffectiveValue`, which additionally handles + * single-variable rules and decides whether an ambiguous/missing + * value should throw. + */ + +import type { + CompiledValueNode, + GrammarPart, + GrammarRule, +} from "./grammarTypes.js"; + +/** + * Find the single part (if any) that carries a variable across a rule's + * parts, so its value can stand in for the whole rule's implicit value + * when the rule has no explicit `->` expression. Returns `"ambiguous"` + * when 2+ parts carry a variable, or `undefined` when none do. + */ +export function findSingleValueBearingPart( + parts: GrammarPart[], +): { variable: string } | "ambiguous" | undefined { + let found: string | undefined; + for (const p of parts) { + const name = p.variable; + if (name === undefined) continue; + if (found !== undefined) return "ambiguous"; + found = name; + } + return found !== undefined ? { variable: found } : undefined; +} + +/** + * Result of `deriveImplicitValue`: either a concrete forwarding value + * (the rule's own `->` expression, or a single variable-bearing part's + * value), `"ambiguous"` (2+ parts carry a variable - callers decide + * whether that's an error), or `undefined` (no part carries a variable, + * or the rule has no parts at all). + */ +export type ImplicitValueResult = + | { kind: "value"; value: CompiledValueNode } + | { kind: "ambiguous" } + | { kind: "none" }; + +/** + * Compute the value expression a rule would implicitly produce if it + * has no explicit `->` expression (via `findSingleValueBearingPart`). + */ +export function deriveImplicitValue(rule: GrammarRule): ImplicitValueResult { + if (rule.value !== undefined) { + return { kind: "value", value: rule.value }; + } + if (rule.parts.length === 0) { + return { kind: "none" }; + } + const result = findSingleValueBearingPart(rule.parts); + if (result === "ambiguous") { + return { kind: "ambiguous" }; + } + if (result === undefined) { + return { kind: "none" }; + } + return { + kind: "value", + value: { type: "variable", name: result.variable }, + }; +} diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index c73c173f8d..83a0d07f63 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -22,6 +22,10 @@ import { import { leadingWordBoundaryScriptPrefix } from "./spacingScripts.js"; import { leadingNonSeparatorRun } from "./grammarMatcher.js"; import { getDispatchEffectiveMembers } from "./dispatchHelpers.js"; +import { + deriveImplicitValue, + findSingleValueBearingPart, +} from "./grammarImplicitExtractor.js"; import { globalPhraseSetRegistry, PhraseSetMatcher, @@ -263,25 +267,6 @@ export const recommendedOptimizations: GrammarOptimizationOptions = { promoteTailRulesParts: true, }; -/** - * NFA-compatible preset: all of `recommendedOptimizations` except - * `tailFactoring` and `promoteTailRulesParts`, which emit - * `RulesPart.tailCall` nodes the NFA compiler rejects. Use this preset - * for any grammar that will be compiled to an NFA/DFA path (note: the - * default `grammarSystem` for agent-server is `completionBased`/AST, not - * NFA - this preset only matters when NFA/DFA is explicitly selected). - * - * TODO(#nfa-tail-support): Once the NFA compiler supports tailCall - * RulesParts natively, this preset (and the `agc compile --nfa-safe` - * flag that exposes it) should be removed in favor of always using - * `recommendedOptimizations`. - */ -export const nfaSafeOptimizations: GrammarOptimizationOptions = { - inlineSingleAlternatives: true, - factorCommonPrefixes: true, - dispatchifyAlternations: true, -}; - /** * Run enabled optimization passes against the compiled grammar AST. * The returned grammar is semantically equivalent to the input - only the @@ -3209,15 +3194,18 @@ function checkForwardingPromotable( // Multi-part rule: matcher's implicit-default rule requires // exactly one variable-bearing contributor (wildcard / number // always; rules / string / phraseSet only when bound; every - // `GrammarPart` carries an optional `variable` field, so a - // single `p.variable !== undefined` test covers the union). - // Promoting masks the baseline missing/multiple-default throws - // at finalize time, so bail out unless the trailing RulesPart - // is the sole contributor. - for (let i = 0; i < parts.length - 1; i++) { - if (parts[i].variable !== undefined) return false; - } - return true; + // `GrammarPart` carries an optional `variable` field, so + // `findSingleValueBearingPart` - the shared scan also used by + // `deriveImplicitValue` (grammarValueDeriver.ts) - covers the + // union). Promoting masks the baseline missing/multiple-default + // throws at finalize time, so bail out unless the trailing + // RulesPart is the sole contributor. + const contributor = findSingleValueBearingPart(parts); + return ( + contributor !== undefined && + contributor !== "ambiguous" && + contributor.variable === last.variable + ); } /** @@ -3327,40 +3315,14 @@ function trySubstituteMembers( * * Used by the value-substitution branch of `tryPromoteTrailing` to * fold each member's effective value into the parent's value - * expression. + * expression. See `deriveImplicitValue` (grammarImplicitValue.ts) for + * the shared derivation logic, also used by the NFA compiler. */ function getImplicitDefaultValue( rule: GrammarRule, ): CompiledValueNode | undefined { - if (rule.value !== undefined) return rule.value; - if (rule.parts.length === 0) return undefined; - const result = findSingleValueBearingPart(rule.parts); - return result !== "ambiguous" && result !== undefined - ? { type: "variable", name: result.variable } - : undefined; -} - -/** - * Find the single part (if any) that carries a variable across a rule's - * parts, so its value can stand in for the whole rule's implicit value - * when the rule has no explicit `->` expression. Returns `"ambiguous"` - * when 2+ parts carry a variable, or `undefined` when none do. - * - * Shared by `getImplicitDefaultValue` (above) and the NFA compiler's - * equivalent forwarding logic for factored rules (see - * `deriveEffectiveValue` in nfaCompiler.ts). - */ -export function findSingleValueBearingPart( - parts: GrammarPart[], -): { variable: string } | "ambiguous" | undefined { - let found: string | undefined; - for (const p of parts) { - const name = p.variable; - if (name === undefined) continue; - if (found !== undefined) return "ambiguous"; - found = name; - } - return found !== undefined ? { variable: found } : undefined; + const result = deriveImplicitValue(rule); + return result.kind === "value" ? result.value : undefined; } // ───────────────────────────────────────────────────────────────────────────── diff --git a/ts/packages/actionGrammar/src/index.ts b/ts/packages/actionGrammar/src/index.ts index 61d3e9013b..87a7a15192 100644 --- a/ts/packages/actionGrammar/src/index.ts +++ b/ts/packages/actionGrammar/src/index.ts @@ -17,10 +17,7 @@ export { grammarToJson } from "./grammarSerializer.js"; export { loadGrammarRules, loadGrammarRulesNoThrow } from "./grammarLoader.js"; export type { LoadGrammarRulesOptions } from "./grammarLoader.js"; export type { GrammarOptimizationOptions } from "./grammarOptimizer.js"; -export { - recommendedOptimizations, - nfaSafeOptimizations, -} from "./grammarOptimizer.js"; +export { recommendedOptimizations } from "./grammarOptimizer.js"; export type { SchemaLoader, DebugInfoCollector, diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index 19df5ca2ea..8d4d1e3d72 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -22,7 +22,7 @@ import { ValueExpression, } from "./environment.js"; import { normalizeToken } from "./nfaMatcher.js"; -import { findSingleValueBearingPart } from "./grammarOptimizer.js"; +import { deriveImplicitValue } from "./grammarImplicitExtractor.js"; // Scripts that require a word-boundary separator between adjacent tokens. // CJK and other logographic/syllabic scripts are NOT included — no separator needed. @@ -478,32 +478,29 @@ function isSingleVariableRule(rule: GrammarRule): { variable: string } | false { /** * Derive the value expression for a rule that has no explicit `->` value: * single-variable rules forward that variable, and factored multi-part - * rules (from `nfaSafeOptimizations`) forward their single variable-bearing - * part's value (via `findSingleValueBearingPart`, shared with the - * optimizer's `getImplicitDefaultValue`). Not every rule needs a value - - * only when `requireValue` is set (i.e. the value is actually consumed: - * top-level action rules, or nested rules captured by a parent variable) - * do ambiguous (2+ variable-bearing parts) or missing values throw; - * otherwise both cases just resolve to `undefined`. + * rules (e.g. from `factorCommonPrefixes`) forward their single + * variable-bearing part's value (via the shared `deriveImplicitValue`, + * also used by the optimizer's `getImplicitDefaultValue`). Not every rule + * needs a value - only when `requireValue` is set (i.e. the value is + * actually consumed: top-level action rules, or nested rules captured by + * a parent variable) do ambiguous (2+ variable-bearing parts) or missing + * values throw; otherwise both cases just resolve to `undefined`. */ function deriveEffectiveValue( rule: GrammarRule, describeRule: () => string, requireValue: boolean, ): CompiledValueNode | undefined { - if (rule.value) { - return rule.value; - } const singleVar = isSingleVariableRule(rule); if (singleVar) { return { type: "variable", name: singleVar.variable }; } if (rule.parts.length <= 1) { - return undefined; + return rule.value; } - const singleVarPart = findSingleValueBearingPart(rule.parts); - if (singleVarPart !== undefined && singleVarPart !== "ambiguous") { - return { type: "variable", name: singleVarPart.variable }; + const result = deriveImplicitValue(rule); + if (result.kind === "value") { + return result.value; } // Not every rule needs a value - only rules whose value is actually // consumed (top-level action rules, or nested rules captured by a @@ -512,7 +509,7 @@ function deriveEffectiveValue( if (!requireValue) { return undefined; } - if (singleVarPart === "ambiguous") { + if (result.kind === "ambiguous") { throw new Error( `${describeRule()} has ${rule.parts.length} terms but no value expression, ` + `and more than one part carries a variable - the implicit value is ambiguous. ` + @@ -528,7 +525,7 @@ function deriveEffectiveValue( (hasTailCall ? " This rule contains a tailCall RulesPart, which the NFA compiler " + "does not support - disable `tailFactoring` / `promoteTailRulesParts` " + - "in the grammar optimizer (use `nfaSafeOptimizations`) for NFA/DFA paths." + "in the grammar optimizer for NFA/DFA paths." : ""), ); } diff --git a/ts/packages/actionGrammar/test/grammarImplicitExtractor.spec.ts b/ts/packages/actionGrammar/test/grammarImplicitExtractor.spec.ts new file mode 100644 index 0000000000..767cd16833 --- /dev/null +++ b/ts/packages/actionGrammar/test/grammarImplicitExtractor.spec.ts @@ -0,0 +1,137 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Direct unit tests for the pure helpers in grammarImplicitExtractor.ts, +// shared by grammarOptimizer.ts (`getImplicitDefaultValue`, +// `checkForwardingPromotable`) and nfaCompiler.ts (`deriveEffectiveValue`). +// The consumers' own spec files (grammarOptimizerPromoteTail.spec.ts, +// nfaFactoredPrefixValues.spec.ts) already cover end-to-end wiring and +// throw/promote decisions; these tests isolate the pure derivation logic +// itself. + +import { + findSingleValueBearingPart, + deriveImplicitValue, +} from "../src/grammarImplicitExtractor.js"; +import { + createStringPart, + createWildcardPart, + createRulesPart, + GrammarRule, +} from "../src/grammarTypes.js"; + +describe("findSingleValueBearingPart", () => { + it("returns undefined for an empty parts array", () => { + expect(findSingleValueBearingPart([])).toBeUndefined(); + }); + + it("returns undefined when no part carries a variable", () => { + const parts = [createStringPart(["foo"]), createStringPart(["bar"])]; + expect(findSingleValueBearingPart(parts)).toBeUndefined(); + }); + + it("returns the variable of the single variable-bearing part", () => { + const parts = [ + createStringPart(["foo"]), + createWildcardPart("x", "wildcard"), + ]; + expect(findSingleValueBearingPart(parts)).toEqual({ variable: "x" }); + }); + + it("finds the variable-bearing part regardless of position", () => { + const parts = [ + createWildcardPart("x", "wildcard"), + createStringPart(["foo"]), + ]; + expect(findSingleValueBearingPart(parts)).toEqual({ variable: "x" }); + }); + + it('returns "ambiguous" when 2+ parts carry a variable', () => { + const parts = [ + createStringPart(["foo"], "x"), + createStringPart(["bar"], "y"), + ]; + expect(findSingleValueBearingPart(parts)).toBe("ambiguous"); + }); + + it('returns "ambiguous" even when 3+ parts carry a variable', () => { + const parts = [ + createStringPart(["foo"], "x"), + createStringPart(["bar"], "y"), + createStringPart(["baz"], "z"), + ]; + expect(findSingleValueBearingPart(parts)).toBe("ambiguous"); + }); +}); + +describe("deriveImplicitValue", () => { + it("returns the rule's explicit value when present, ignoring parts", () => { + const rule: GrammarRule = { + parts: [ + createStringPart(["foo"], "x"), + createStringPart(["bar"], "y"), + ], + value: { type: "literal", value: "explicit" }, + }; + expect(deriveImplicitValue(rule)).toEqual({ + kind: "value", + value: { type: "literal", value: "explicit" }, + }); + }); + + it('returns "none" for a rule with no parts and no value', () => { + const rule: GrammarRule = { parts: [] }; + expect(deriveImplicitValue(rule)).toEqual({ kind: "none" }); + }); + + it('returns "none" for a single part with no variable', () => { + const rule: GrammarRule = { parts: [createStringPart(["foo"])] }; + expect(deriveImplicitValue(rule)).toEqual({ kind: "none" }); + }); + + it("forwards a single part's variable as the implicit value", () => { + const rule: GrammarRule = { + parts: [ + createStringPart(["foo"]), + createWildcardPart("x", "wildcard"), + ], + }; + expect(deriveImplicitValue(rule)).toEqual({ + kind: "value", + value: { type: "variable", name: "x" }, + }); + }); + + it("forwards a nested RulesPart's captured variable", () => { + const nested = createRulesPart( + [{ parts: [createStringPart(["foo"])] }], + { + variable: "chosen", + }, + ); + const rule: GrammarRule = { + parts: [createStringPart(["prefix"]), nested], + }; + expect(deriveImplicitValue(rule)).toEqual({ + kind: "value", + value: { type: "variable", name: "chosen" }, + }); + }); + + it('returns "ambiguous" when 2+ parts carry a variable and there is no explicit value', () => { + const rule: GrammarRule = { + parts: [ + createStringPart(["foo"], "x"), + createStringPart(["bar"], "y"), + ], + }; + expect(deriveImplicitValue(rule)).toEqual({ kind: "ambiguous" }); + }); + + it('returns "none" for a multi-part rule where nothing carries a variable', () => { + const rule: GrammarRule = { + parts: [createStringPart(["foo"]), createStringPart(["bar"])], + }; + expect(deriveImplicitValue(rule)).toEqual({ kind: "none" }); + }); +}); diff --git a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts index af7b0bbef6..a0bef50e13 100644 --- a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts +++ b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts @@ -5,7 +5,7 @@ // shares a common optional prefix used to fail to compile to an NFA // once `factorCommonPrefixes` hoisted the shared prefix out, leaving a // value-less multi-term rule. See nfaCompiler.ts (implicit value -// derivation) and grammarOptimizer.ts (`nfaSafeOptimizations`). +// derivation) and grammarOptimizer.ts (`factorCommonPrefixes`). // // Note: `factorCommonPrefixes`'s own output already stamps an // explicit `value` on the factored top-level rule, so grammar-source @@ -13,16 +13,15 @@ // hand-built-AST test further down is what directly exercises // `findSingleValueBearingPart`'s forwarding logic. // -// NFA's rejection of tailCall RulesParts (still unsupported; see -// `nfaSafeOptimizations`) is already covered by -// grammarOptimizerTailFactoring.spec.ts ("NFA compile of a -// tail-factored grammar throws a descriptive error") - not duplicated -// here. +// NFA's rejection of tailCall RulesParts (still unsupported) is +// already covered by grammarOptimizerTailFactoring.spec.ts ("NFA +// compile of a tail-factored grammar throws a descriptive error") - +// not duplicated here. import { loadGrammarRules } from "../src/grammarLoader.js"; import { compileGrammarToNFA } from "../src/nfaCompiler.js"; import { matchNFA } from "../src/nfaInterpreter.js"; -import { nfaSafeOptimizations } from "../src/grammarOptimizer.js"; +import { recommendedOptimizations } from "../src/grammarOptimizer.js"; import { Grammar, createStringPart, @@ -40,11 +39,11 @@ describe("NFA compilation of factored shared-prefix grammars", () => { `; describeForEachMatcher( - "compiles and matches correctly with nfaSafeOptimizations", + "compiles and matches correctly with recommendedOptimizations", (testMatchGrammar) => { it("matches each branch, with and without the shared prefix", () => { const grammar = loadGrammarRules("factored-prefix.agr", agr, { - optimizations: nfaSafeOptimizations, + optimizations: recommendedOptimizations, }); expect(testMatchGrammar(grammar, "foo")).toStrictEqual([ diff --git a/ts/packages/actionGrammarCompiler/src/commands/compile.ts b/ts/packages/actionGrammarCompiler/src/commands/compile.ts index 3d9b01ffc1..79274e2e21 100644 --- a/ts/packages/actionGrammarCompiler/src/commands/compile.ts +++ b/ts/packages/actionGrammarCompiler/src/commands/compile.ts @@ -7,7 +7,6 @@ import fs from "node:fs"; import { grammarToJson, loadGrammarRulesNoThrow, - nfaSafeOptimizations, recommendedOptimizations, SchemaLoader, } from "@typeagent/action-grammar"; @@ -69,16 +68,6 @@ export default class Compile extends Command { "Disable grammar optimizations (produces an unoptimized AST that preserves the 1:1 correspondence between top-level rules and the original source — useful for diagnostics).", default: false, }), - // TODO(#nfa-tail-support): Remove this flag once the NFA compiler - // supports tailCall-producing optimizations (tailFactoring / - // promoteTailRulesParts) directly, at which point - // `recommendedOptimizations` alone should be NFA-safe and this - // opt-in preset becomes unnecessary. - "nfa-safe": Flags.boolean({ - description: - "Use the NFA-compatible optimization preset (excludes tailCall-producing passes) instead of the default recommended optimizations.", - default: false, - }), }; async run(): Promise { @@ -97,9 +86,7 @@ export default class Compile extends Command { : { startValueRequired: true, schemaLoader, - optimizations: flags["nfa-safe"] - ? nfaSafeOptimizations - : recommendedOptimizations, + optimizations: recommendedOptimizations, }, ); From 2003a63151df3d84214805643e0903c19711cd03 Mon Sep 17 00:00:00 2001 From: George Ng Date: Fri, 10 Jul 2026 16:46:14 -0700 Subject: [PATCH 12/29] Rename grammarImplicitExtractor to grammarValueDeriver --- .../actionGrammar/src/grammarOptimizer.ts | 10 ++++---- ...citExtractor.ts => grammarValueDeriver.ts} | 23 +++++++++++-------- ts/packages/actionGrammar/src/nfaCompiler.ts | 6 ++--- ...or.spec.ts => grammarValueDeriver.spec.ts} | 22 +++++++++--------- 4 files changed, 32 insertions(+), 29 deletions(-) rename ts/packages/actionGrammar/src/{grammarImplicitExtractor.ts => grammarValueDeriver.ts} (72%) rename ts/packages/actionGrammar/test/{grammarImplicitExtractor.spec.ts => grammarValueDeriver.spec.ts} (87%) diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index 83a0d07f63..2098db12b7 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -23,9 +23,9 @@ import { leadingWordBoundaryScriptPrefix } from "./spacingScripts.js"; import { leadingNonSeparatorRun } from "./grammarMatcher.js"; import { getDispatchEffectiveMembers } from "./dispatchHelpers.js"; import { - deriveImplicitValue, + deriveValue, findSingleValueBearingPart, -} from "./grammarImplicitExtractor.js"; +} from "./grammarValueDeriver.js"; import { globalPhraseSetRegistry, PhraseSetMatcher, @@ -3196,7 +3196,7 @@ function checkForwardingPromotable( // always; rules / string / phraseSet only when bound; every // `GrammarPart` carries an optional `variable` field, so // `findSingleValueBearingPart` - the shared scan also used by - // `deriveImplicitValue` (grammarValueDeriver.ts) - covers the + // `deriveValue` (grammarValueDeriver.ts) - covers the // union). Promoting masks the baseline missing/multiple-default // throws at finalize time, so bail out unless the trailing // RulesPart is the sole contributor. @@ -3315,13 +3315,13 @@ function trySubstituteMembers( * * Used by the value-substitution branch of `tryPromoteTrailing` to * fold each member's effective value into the parent's value - * expression. See `deriveImplicitValue` (grammarImplicitValue.ts) for + * expression. See `deriveValue` (grammarValueDeriver.ts) for * the shared derivation logic, also used by the NFA compiler. */ function getImplicitDefaultValue( rule: GrammarRule, ): CompiledValueNode | undefined { - const result = deriveImplicitValue(rule); + const result = deriveValue(rule); return result.kind === "value" ? result.value : undefined; } diff --git a/ts/packages/actionGrammar/src/grammarImplicitExtractor.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts similarity index 72% rename from ts/packages/actionGrammar/src/grammarImplicitExtractor.ts rename to ts/packages/actionGrammar/src/grammarValueDeriver.ts index 16f77b9027..3a354d7d94 100644 --- a/ts/packages/actionGrammar/src/grammarImplicitExtractor.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -2,8 +2,9 @@ // Licensed under the MIT License. /** - * Structural derivation of a rule's *implicit* value expression when it - * has no explicit `->` expression. + * Structural derivation of a rule's *effective* value expression: its + * explicit `->` expression when present, otherwise an implicit + * forwarding value derived from the rule's parts. * * A rule with exactly one variable-bearing part implicitly forwards that * part's value - this mirrors the AST matcher's implicit-default rule @@ -42,11 +43,12 @@ export function findSingleValueBearingPart( } /** - * Result of `deriveImplicitValue`: either a concrete forwarding value - * (the rule's own `->` expression, or a single variable-bearing part's - * value), `"ambiguous"` (2+ parts carry a variable - callers decide - * whether that's an error), or `undefined` (no part carries a variable, - * or the rule has no parts at all). + * Result of `deriveValue`: either a concrete value (the rule's own `->` + * expression, or a single variable-bearing part's implicitly forwarded + * value), `"ambiguous"` (2+ parts carry a variable and there is no + * explicit value - callers decide whether that's an error), or `"none"` + * (no explicit value and no part carries a variable, or the rule has no + * parts at all). */ export type ImplicitValueResult = | { kind: "value"; value: CompiledValueNode } @@ -54,10 +56,11 @@ export type ImplicitValueResult = | { kind: "none" }; /** - * Compute the value expression a rule would implicitly produce if it - * has no explicit `->` expression (via `findSingleValueBearingPart`). + * Compute a rule's effective value expression: its explicit `->` value + * when present, otherwise the implicit value it would produce (via + * `findSingleValueBearingPart`). */ -export function deriveImplicitValue(rule: GrammarRule): ImplicitValueResult { +export function deriveValue(rule: GrammarRule): ImplicitValueResult { if (rule.value !== undefined) { return { kind: "value", value: rule.value }; } diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index 8d4d1e3d72..e36cf92ab2 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -22,7 +22,7 @@ import { ValueExpression, } from "./environment.js"; import { normalizeToken } from "./nfaMatcher.js"; -import { deriveImplicitValue } from "./grammarImplicitExtractor.js"; +import { deriveValue } from "./grammarValueDeriver.js"; // Scripts that require a word-boundary separator between adjacent tokens. // CJK and other logographic/syllabic scripts are NOT included — no separator needed. @@ -479,7 +479,7 @@ function isSingleVariableRule(rule: GrammarRule): { variable: string } | false { * Derive the value expression for a rule that has no explicit `->` value: * single-variable rules forward that variable, and factored multi-part * rules (e.g. from `factorCommonPrefixes`) forward their single - * variable-bearing part's value (via the shared `deriveImplicitValue`, + * variable-bearing part's value (via the shared `deriveValue`, * also used by the optimizer's `getImplicitDefaultValue`). Not every rule * needs a value - only when `requireValue` is set (i.e. the value is * actually consumed: top-level action rules, or nested rules captured by @@ -498,7 +498,7 @@ function deriveEffectiveValue( if (rule.parts.length <= 1) { return rule.value; } - const result = deriveImplicitValue(rule); + const result = deriveValue(rule); if (result.kind === "value") { return result.value; } diff --git a/ts/packages/actionGrammar/test/grammarImplicitExtractor.spec.ts b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts similarity index 87% rename from ts/packages/actionGrammar/test/grammarImplicitExtractor.spec.ts rename to ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts index 767cd16833..bc5433bcc8 100644 --- a/ts/packages/actionGrammar/test/grammarImplicitExtractor.spec.ts +++ b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// Direct unit tests for the pure helpers in grammarImplicitExtractor.ts, +// Direct unit tests for the pure helpers in grammarValueDeriver.ts, // shared by grammarOptimizer.ts (`getImplicitDefaultValue`, // `checkForwardingPromotable`) and nfaCompiler.ts (`deriveEffectiveValue`). // The consumers' own spec files (grammarOptimizerPromoteTail.spec.ts, @@ -11,8 +11,8 @@ import { findSingleValueBearingPart, - deriveImplicitValue, -} from "../src/grammarImplicitExtractor.js"; + deriveValue, +} from "../src/grammarValueDeriver.js"; import { createStringPart, createWildcardPart, @@ -64,7 +64,7 @@ describe("findSingleValueBearingPart", () => { }); }); -describe("deriveImplicitValue", () => { +describe("deriveValue", () => { it("returns the rule's explicit value when present, ignoring parts", () => { const rule: GrammarRule = { parts: [ @@ -73,7 +73,7 @@ describe("deriveImplicitValue", () => { ], value: { type: "literal", value: "explicit" }, }; - expect(deriveImplicitValue(rule)).toEqual({ + expect(deriveValue(rule)).toEqual({ kind: "value", value: { type: "literal", value: "explicit" }, }); @@ -81,12 +81,12 @@ describe("deriveImplicitValue", () => { it('returns "none" for a rule with no parts and no value', () => { const rule: GrammarRule = { parts: [] }; - expect(deriveImplicitValue(rule)).toEqual({ kind: "none" }); + expect(deriveValue(rule)).toEqual({ kind: "none" }); }); it('returns "none" for a single part with no variable', () => { const rule: GrammarRule = { parts: [createStringPart(["foo"])] }; - expect(deriveImplicitValue(rule)).toEqual({ kind: "none" }); + expect(deriveValue(rule)).toEqual({ kind: "none" }); }); it("forwards a single part's variable as the implicit value", () => { @@ -96,7 +96,7 @@ describe("deriveImplicitValue", () => { createWildcardPart("x", "wildcard"), ], }; - expect(deriveImplicitValue(rule)).toEqual({ + expect(deriveValue(rule)).toEqual({ kind: "value", value: { type: "variable", name: "x" }, }); @@ -112,7 +112,7 @@ describe("deriveImplicitValue", () => { const rule: GrammarRule = { parts: [createStringPart(["prefix"]), nested], }; - expect(deriveImplicitValue(rule)).toEqual({ + expect(deriveValue(rule)).toEqual({ kind: "value", value: { type: "variable", name: "chosen" }, }); @@ -125,13 +125,13 @@ describe("deriveImplicitValue", () => { createStringPart(["bar"], "y"), ], }; - expect(deriveImplicitValue(rule)).toEqual({ kind: "ambiguous" }); + expect(deriveValue(rule)).toEqual({ kind: "ambiguous" }); }); it('returns "none" for a multi-part rule where nothing carries a variable', () => { const rule: GrammarRule = { parts: [createStringPart(["foo"]), createStringPart(["bar"])], }; - expect(deriveImplicitValue(rule)).toEqual({ kind: "none" }); + expect(deriveValue(rule)).toEqual({ kind: "none" }); }); }); From c41544b737a7bc4ea56b917eeeb4893651355ae1 Mon Sep 17 00:00:00 2001 From: George Ng Date: Fri, 10 Jul 2026 17:49:58 -0700 Subject: [PATCH 13/29] Unify value extraction on derviceValue() --- .../actionGrammar/src/grammarOptimizer.ts | 7 +-- .../actionGrammar/src/grammarValueDeriver.ts | 10 +--- ts/packages/actionGrammar/src/nfaCompiler.ts | 55 +++++-------------- 3 files changed, 20 insertions(+), 52 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index 2098db12b7..91c9495324 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -3195,9 +3195,8 @@ function checkForwardingPromotable( // exactly one variable-bearing contributor (wildcard / number // always; rules / string / phraseSet only when bound; every // `GrammarPart` carries an optional `variable` field, so - // `findSingleValueBearingPart` - the shared scan also used by - // `deriveValue` (grammarValueDeriver.ts) - covers the - // union). Promoting masks the baseline missing/multiple-default + // `findSingleValueBearingPart` - the shared scan used here - + // covers the union). Promoting masks the baseline missing/multiple-default // throws at finalize time, so bail out unless the trailing // RulesPart is the sole contributor. const contributor = findSingleValueBearingPart(parts); @@ -3316,7 +3315,7 @@ function trySubstituteMembers( * Used by the value-substitution branch of `tryPromoteTrailing` to * fold each member's effective value into the parent's value * expression. See `deriveValue` (grammarValueDeriver.ts) for - * the shared derivation logic, also used by the NFA compiler. + * the shared derivation logic. */ function getImplicitDefaultValue( rule: GrammarRule, diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index 3a354d7d94..fe026d1c93 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -7,14 +7,8 @@ * forwarding value derived from the rule's parts. * * A rule with exactly one variable-bearing part implicitly forwards that - * part's value - this mirrors the AST matcher's implicit-default rule - * (see grammarMatcher.ts) and is the single source of truth shared by: - * - grammarOptimizer.ts's `getImplicitDefaultValue`, used by the - * value-substitution branch of `tryPromoteTrailing` to fold each - * member's effective value into the parent's value expression. - * - nfaCompiler.ts's `deriveEffectiveValue`, which additionally handles - * single-variable rules and decides whether an ambiguous/missing - * value should throw. + * part's value. This is the single source of truth for that + * derivation, kept independent of any particular compilation backend. */ import type { diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index e36cf92ab2..c2902fd850 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -452,52 +452,23 @@ function stripDispatch(part: RulesPart): RulesPart { }); } -/** - * Check if a rule is a single-variable rule (e.g., = $(x:wildcard);) - * Such rules should implicitly produce their variable's value: -> $(x) - */ -function isSingleVariableRule(rule: GrammarRule): { variable: string } | false { - // A single-variable rule has: - // 1. No explicit value expression - // 2. Single part that is a wildcard or number with a variable - if (rule.value) { - return false; // Has explicit value - } - if (rule.parts.length !== 1) { - return false; // Multiple parts - } - const part = rule.parts[0]; - if (part.type === "wildcard" || part.type === "number") { - if (part.variable) { - return { variable: part.variable }; - } - } - return false; -} - /** * Derive the value expression for a rule that has no explicit `->` value: - * single-variable rules forward that variable, and factored multi-part - * rules (e.g. from `factorCommonPrefixes`) forward their single - * variable-bearing part's value (via the shared `deriveValue`, - * also used by the optimizer's `getImplicitDefaultValue`). Not every rule - * needs a value - only when `requireValue` is set (i.e. the value is - * actually consumed: top-level action rules, or nested rules captured by - * a parent variable) do ambiguous (2+ variable-bearing parts) or missing - * values throw; otherwise both cases just resolve to `undefined`. + * a rule with exactly one variable-bearing part (whatever its type - + * wildcard, number, or a bound rules/string/phraseSet part) implicitly + * forwards that part's value, via the shared `deriveValue` - the single + * place that decides a rule's effective value, explicit or implicit. + * Not every rule needs a value - only when `requireValue` is set (i.e. + * the value is actually consumed: top-level action rules, or nested + * rules captured by a parent variable) do ambiguous (2+ variable-bearing + * parts) or missing values throw; otherwise both cases just resolve to + * `undefined`. */ function deriveEffectiveValue( rule: GrammarRule, describeRule: () => string, requireValue: boolean, ): CompiledValueNode | undefined { - const singleVar = isSingleVariableRule(rule); - if (singleVar) { - return { type: "variable", name: singleVar.variable }; - } - if (rule.parts.length <= 1) { - return rule.value; - } const result = deriveValue(rule); if (result.kind === "value") { return result.value; @@ -519,9 +490,13 @@ function deriveEffectiveValue( const hasTailCall = rule.parts.some( (p) => p.type === "rules" && p.tailCall, ); + const termsDescription = + rule.parts.length === 1 + ? "has 1 term" + : `has ${rule.parts.length} terms`; throw new Error( - `${describeRule()} has ${rule.parts.length} terms but no value expression. ` + - `Multi-term rules must have an explicit value expression (using ->).` + + `${describeRule()} ${termsDescription} but no value expression, and no part carries a variable. ` + + `Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.` + (hasTailCall ? " This rule contains a tailCall RulesPart, which the NFA compiler " + "does not support - disable `tailFactoring` / `promoteTailRulesParts` " + From cf0b4b11422863b0b8c8bd011f3aeb0960ff3881 Mon Sep 17 00:00:00 2001 From: George Ng Date: Fri, 10 Jul 2026 18:29:57 -0700 Subject: [PATCH 14/29] Use findSingleValueBearingPart in grammarValueValidator, update findSingleValuebearingPart --- ts/docs/architecture/core/actionGrammar.md | 17 +++++----- .../actionGrammar/src/grammarValueDeriver.ts | 8 ++--- .../src/grammarValueTypeValidator.ts | 11 +++--- .../test/grammarValueDeriver.spec.ts | 24 +++++++------ .../test/nfaFactoredPrefixValues.spec.ts | 34 +++++++++++++++++++ 5 files changed, 66 insertions(+), 28 deletions(-) diff --git a/ts/docs/architecture/core/actionGrammar.md b/ts/docs/architecture/core/actionGrammar.md index 45f92f9d56..07c7706fda 100644 --- a/ts/docs/architecture/core/actionGrammar.md +++ b/ts/docs/architecture/core/actionGrammar.md @@ -1521,14 +1521,15 @@ explicit value expression produce no value — the compiler warns about this because the output is ambiguous. The NFA compiler (`nfaCompiler.ts`) mirrors this same "no value unless -actually needed" principle for rules reshaped by `factorCommonPrefixes`: -`deriveEffectiveValue` forwards a -factored rule's single variable-bearing part, and only treats an -ambiguous or missing implicit value as a hard error when the value is -actually required — i.e. for top-level action rules, or nested rules a -parent captures via `$(name:)`. A nested, uncaptured rule with an -ambiguous shape is harmless (nothing reads its value) and silently -resolves to "no value," exactly like the `none` kind above. +actually needed" principle via the shared `deriveValue` +(`grammarValueDeriver.ts`): `deriveEffectiveValue` forwards a rule's +single variable-bearing part (whatever its shape - a single-part rule, +or a multi-part rule reshaped by `factorCommonPrefixes`), and only +treats an ambiguous or missing implicit value as a hard error when the +value is actually required — i.e. for top-level action rules, or nested +rules a parent captures via `$(name:)`. A nested, uncaptured rule +with an ambiguous shape is harmless (nothing reads its value) and +silently resolves to "no value," exactly like the `none` kind above. ### Design Principles diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index fe026d1c93..ed2f0f5c43 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -25,15 +25,15 @@ import type { */ export function findSingleValueBearingPart( parts: GrammarPart[], -): { variable: string } | "ambiguous" | undefined { - let found: string | undefined; +): { variable: string; part: GrammarPart } | "ambiguous" | undefined { + let found: { variable: string; part: GrammarPart } | undefined; for (const p of parts) { const name = p.variable; if (name === undefined) continue; if (found !== undefined) return "ambiguous"; - found = name; + found = { variable: name, part: p }; } - return found !== undefined ? { variable: found } : undefined; + return found; } /** diff --git a/ts/packages/actionGrammar/src/grammarValueTypeValidator.ts b/ts/packages/actionGrammar/src/grammarValueTypeValidator.ts index aa2672d063..9236eacbfd 100644 --- a/ts/packages/actionGrammar/src/grammarValueTypeValidator.ts +++ b/ts/packages/actionGrammar/src/grammarValueTypeValidator.ts @@ -16,6 +16,7 @@ import type { } from "@typeagent/action-schema"; import { SchemaCreator } from "@typeagent/action-schema"; import { getDispatchEffectiveMembers } from "./dispatchHelpers.js"; +import { findSingleValueBearingPart } from "./grammarValueDeriver.js"; // Sentinel for "any" — can't determine type const ANY_TYPE: SchemaType = SchemaCreator.any(); @@ -498,15 +499,15 @@ export function classifyRuleValue(rule: GrammarRule): RuleValueKind { if (rule.value !== undefined) { return { kind: "explicit" }; } - const variableParts = rule.parts.filter((p) => p.variable !== undefined); - if (variableParts.length === 1) { + const contributor = findSingleValueBearingPart(rule.parts); + if (contributor !== undefined && contributor !== "ambiguous") { return { kind: "variable", - variableName: variableParts[0].variable!, - part: variableParts[0], + variableName: contributor.variable, + part: contributor.part, }; } - if (variableParts.length === 0 && rule.parts.length === 1) { + if (contributor === undefined && rule.parts.length === 1) { const part = rule.parts[0]; if (part.type === "rules") { // For dispatched parts, the effective member list spans diff --git a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts index bc5433bcc8..85d59a05cd 100644 --- a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts +++ b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts @@ -30,20 +30,22 @@ describe("findSingleValueBearingPart", () => { expect(findSingleValueBearingPart(parts)).toBeUndefined(); }); - it("returns the variable of the single variable-bearing part", () => { - const parts = [ - createStringPart(["foo"]), - createWildcardPart("x", "wildcard"), - ]; - expect(findSingleValueBearingPart(parts)).toEqual({ variable: "x" }); + it("returns the variable and part of the single variable-bearing part", () => { + const wildcardPart = createWildcardPart("x", "wildcard"); + const parts = [createStringPart(["foo"]), wildcardPart]; + expect(findSingleValueBearingPart(parts)).toEqual({ + variable: "x", + part: wildcardPart, + }); }); it("finds the variable-bearing part regardless of position", () => { - const parts = [ - createWildcardPart("x", "wildcard"), - createStringPart(["foo"]), - ]; - expect(findSingleValueBearingPart(parts)).toEqual({ variable: "x" }); + const wildcardPart = createWildcardPart("x", "wildcard"); + const parts = [wildcardPart, createStringPart(["foo"])]; + expect(findSingleValueBearingPart(parts)).toEqual({ + variable: "x", + part: wildcardPart, + }); }); it('returns "ambiguous" when 2+ parts carry a variable', () => { diff --git a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts index a0bef50e13..ec74ef717a 100644 --- a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts +++ b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts @@ -26,6 +26,7 @@ import { Grammar, createStringPart, createRulesPart, + createPhraseSetPart, } from "../src/grammarTypes.js"; import { describeForEachMatcher } from "./testUtils.js"; @@ -117,4 +118,37 @@ describe("NFA compilation of factored shared-prefix grammars", () => { /implicit value is ambiguous/, ); }); + + it("throws an accurate error (not 'Multi-term') for a single-part rule with no variable and no value", () => { + // Raw AST with exactly one part, that part carrying no variable + // and no top-level `value`. String-literal single-part rules are + // auto-normalized (isSingleLiteralRule) to stamp a matched-text + // value, so this uses an unbound phraseSet part instead - a + // shape normalizeRule does not special-case. Before + // deriveEffectiveValue was consolidated to call `deriveValue` + // uniformly, a rule this shape short-circuited to `rule.value` + // (undefined) without throwing. It now throws like any other + // value-less rule - guard here that the message accurately + // reflects a 1-term rule instead of the multi-term wording. + const grammar: Grammar = { + alternatives: [{ parts: [createPhraseSetPart("Polite")] }], + }; + let message = ""; + try { + compileGrammarToNFA(grammar, "single-part-none"); + } catch (e) { + message = (e as Error).message; + } + expect(message).toMatch(/has 1 term but no value expression/); + expect(message).not.toMatch(/Multi-term/); + }); + + it("throws an accurate error for a zero-part rule with no value", () => { + const grammar: Grammar = { + alternatives: [{ parts: [] }], + }; + expect(() => compileGrammarToNFA(grammar, "empty-rule")).toThrow( + /has 0 terms but no value expression/, + ); + }); }); From 2f4c70665a3cf58f83e300e305d3fd5a1b7057ed Mon Sep 17 00:00:00 2001 From: George Ng Date: Fri, 10 Jul 2026 19:13:21 -0700 Subject: [PATCH 15/29] Refactor for semantic and readaibility improvements --- .../actionGrammar/src/grammarValueDeriver.ts | 11 ++- ts/packages/actionGrammar/src/nfaCompiler.ts | 12 ++-- .../test/nfaCompilerValueErrors.spec.ts | 70 +++++++++++++++++++ .../test/nfaFactoredPrefixValues.spec.ts | 52 -------------- 4 files changed, 85 insertions(+), 60 deletions(-) create mode 100644 ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index ed2f0f5c43..49a2dec562 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -17,6 +17,13 @@ import type { GrammarRule, } from "./grammarTypes.js"; +/** + * A part that solely carries a rule's implicit value: the variable name + * it binds, and the `GrammarPart` itself (so callers that need more than + * the name - e.g. its type - don't have to re-scan `parts`). + */ +export type ValueBearingPart = { variable: string; part: GrammarPart }; + /** * Find the single part (if any) that carries a variable across a rule's * parts, so its value can stand in for the whole rule's implicit value @@ -25,8 +32,8 @@ import type { */ export function findSingleValueBearingPart( parts: GrammarPart[], -): { variable: string; part: GrammarPart } | "ambiguous" | undefined { - let found: { variable: string; part: GrammarPart } | undefined; +): ValueBearingPart | "ambiguous" | undefined { + let found: ValueBearingPart | undefined; for (const p of parts) { const name = p.variable; if (name === undefined) continue; diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index c2902fd850..90425913b7 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -480,20 +480,20 @@ function deriveEffectiveValue( if (!requireValue) { return undefined; } + const termsDescription = + rule.parts.length === 1 + ? "has 1 term" + : `has ${rule.parts.length} terms`; if (result.kind === "ambiguous") { throw new Error( - `${describeRule()} has ${rule.parts.length} terms but no value expression, ` + + `${describeRule()} ${termsDescription} but no value expression, ` + `and more than one part carries a variable - the implicit value is ambiguous. ` + - `Multi-term rules must have an explicit value expression (using ->).`, + `Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.`, ); } const hasTailCall = rule.parts.some( (p) => p.type === "rules" && p.tailCall, ); - const termsDescription = - rule.parts.length === 1 - ? "has 1 term" - : `has ${rule.parts.length} terms`; throw new Error( `${describeRule()} ${termsDescription} but no value expression, and no part carries a variable. ` + `Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.` + diff --git a/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts b/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts new file mode 100644 index 0000000000..79637874ae --- /dev/null +++ b/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Direct, hand-built-AST coverage of `deriveEffectiveValue`'s (nfaCompiler.ts) +// error paths when a top-level rule has no explicit `->` value and no +// single variable-bearing part to forward implicitly. These bypass the +// loader's own validation to exercise `compileGrammarToNFA`'s throw +// behavior directly, independent of any particular grammar-source +// scenario (e.g. shared-prefix factoring - see +// nfaFactoredPrefixValues.spec.ts for that). + +import { compileGrammarToNFA } from "../src/nfaCompiler.js"; +import { + Grammar, + createStringPart, + createPhraseSetPart, +} from "../src/grammarTypes.js"; + +describe("deriveEffectiveValue error messages", () => { + it("throws a clear error when the implicit value is genuinely ambiguous", () => { + // Raw AST (bypasses the loader's own validation) with two + // variable-bearing parts and no top-level value. + const grammar: Grammar = { + alternatives: [ + { + parts: [ + createStringPart(["foo"], "x"), + createStringPart(["bar"], "y"), + ], + }, + ], + }; + expect(() => compileGrammarToNFA(grammar, "ambiguous")).toThrow( + /implicit value is ambiguous/, + ); + }); + + it("throws an accurate error (not 'Multi-term') for a single-part rule with no variable and no value", () => { + // Raw AST with exactly one part, that part carrying no variable + // and no top-level `value`. String-literal single-part rules are + // auto-normalized (isSingleLiteralRule) to stamp a matched-text + // value, so this uses an unbound phraseSet part instead - a + // shape normalizeRule does not special-case. Before + // deriveEffectiveValue was consolidated to call `deriveValue` + // uniformly, a rule this shape short-circuited to `rule.value` + // (undefined) without throwing. It now throws like any other + // value-less rule - guard here that the message accurately + // reflects a 1-term rule instead of the multi-term wording. + const grammar: Grammar = { + alternatives: [{ parts: [createPhraseSetPart("Polite")] }], + }; + let message = ""; + try { + compileGrammarToNFA(grammar, "single-part-none"); + } catch (e) { + message = (e as Error).message; + } + expect(message).toMatch(/has 1 term but no value expression/); + expect(message).not.toMatch(/Multi-term/); + }); + + it("throws an accurate error for a zero-part rule with no value", () => { + const grammar: Grammar = { + alternatives: [{ parts: [] }], + }; + expect(() => compileGrammarToNFA(grammar, "empty-rule")).toThrow( + /has 0 terms but no value expression/, + ); + }); +}); diff --git a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts index ec74ef717a..e5bb513a7c 100644 --- a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts +++ b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts @@ -26,7 +26,6 @@ import { Grammar, createStringPart, createRulesPart, - createPhraseSetPart, } from "../src/grammarTypes.js"; import { describeForEachMatcher } from "./testUtils.js"; @@ -100,55 +99,4 @@ describe("NFA compilation of factored shared-prefix grammars", () => { expect(matchNFA(nfa, ["foo"], true).actionValue).toBe("A"); expect(matchNFA(nfa, ["please", "bar"], true).actionValue).toBe("B"); }); - - it("throws a clear error when the implicit value is genuinely ambiguous", () => { - // Raw AST (bypasses the loader's own validation) with two - // variable-bearing parts and no top-level value. - const grammar: Grammar = { - alternatives: [ - { - parts: [ - createStringPart(["foo"], "x"), - createStringPart(["bar"], "y"), - ], - }, - ], - }; - expect(() => compileGrammarToNFA(grammar, "ambiguous")).toThrow( - /implicit value is ambiguous/, - ); - }); - - it("throws an accurate error (not 'Multi-term') for a single-part rule with no variable and no value", () => { - // Raw AST with exactly one part, that part carrying no variable - // and no top-level `value`. String-literal single-part rules are - // auto-normalized (isSingleLiteralRule) to stamp a matched-text - // value, so this uses an unbound phraseSet part instead - a - // shape normalizeRule does not special-case. Before - // deriveEffectiveValue was consolidated to call `deriveValue` - // uniformly, a rule this shape short-circuited to `rule.value` - // (undefined) without throwing. It now throws like any other - // value-less rule - guard here that the message accurately - // reflects a 1-term rule instead of the multi-term wording. - const grammar: Grammar = { - alternatives: [{ parts: [createPhraseSetPart("Polite")] }], - }; - let message = ""; - try { - compileGrammarToNFA(grammar, "single-part-none"); - } catch (e) { - message = (e as Error).message; - } - expect(message).toMatch(/has 1 term but no value expression/); - expect(message).not.toMatch(/Multi-term/); - }); - - it("throws an accurate error for a zero-part rule with no value", () => { - const grammar: Grammar = { - alternatives: [{ parts: [] }], - }; - expect(() => compileGrammarToNFA(grammar, "empty-rule")).toThrow( - /has 0 terms but no value expression/, - ); - }); }); From bddf6ed27e885089f3606d054b11a6d4853e3c5a Mon Sep 17 00:00:00 2001 From: George Ng Date: Fri, 10 Jul 2026 19:26:46 -0700 Subject: [PATCH 16/29] Cleanup return statement in checkForwardinPromotable --- .../actionGrammar/src/grammarOptimizer.ts | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index 91c9495324..6b042fb27d 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -3191,20 +3191,12 @@ function checkForwardingPromotable( ): boolean { if (parts.length <= 1) return true; if (last.variable === undefined) return false; - // Multi-part rule: matcher's implicit-default rule requires - // exactly one variable-bearing contributor (wildcard / number - // always; rules / string / phraseSet only when bound; every - // `GrammarPart` carries an optional `variable` field, so - // `findSingleValueBearingPart` - the shared scan used here - - // covers the union). Promoting masks the baseline missing/multiple-default - // throws at finalize time, so bail out unless the trailing - // RulesPart is the sole contributor. - const contributor = findSingleValueBearingPart(parts); - return ( - contributor !== undefined && - contributor !== "ambiguous" && - contributor.variable === last.variable - ); + // Multi-part rule: matcher's implicit-default rule requires exactly + // one variable-bearing contributor. `last` already carries a + // variable, so the shared scan can only return "ambiguous" (2+ + // contributors) or `last` itself - bail out on "ambiguous" so + // baseline missing/multiple-default throws stay observable. + return findSingleValueBearingPart(parts) !== "ambiguous"; } /** From baca38e333c40e115b04d826a2d270b17a74434d Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 16:27:00 -0700 Subject: [PATCH 17/29] Unify getIMplicitDefaultValue on deriveEffectiveValue, more tailcall into normalizeRules --- ts/docs/architecture/core/actionGrammar.md | 7 +- .../actionGrammar/src/grammarOptimizer.ts | 39 +++----- .../actionGrammar/src/grammarValueDeriver.ts | 52 +++++++++++ ts/packages/actionGrammar/src/nfaCompiler.ts | 89 +++++++------------ .../grammarOptimizerTailFactoring.spec.ts | 37 +++++++- .../test/grammarValueDeriver.spec.ts | 7 +- .../test/nfaCompilerValueErrors.spec.ts | 7 +- 7 files changed, 147 insertions(+), 91 deletions(-) diff --git a/ts/docs/architecture/core/actionGrammar.md b/ts/docs/architecture/core/actionGrammar.md index 07c7706fda..d8f0e19a71 100644 --- a/ts/docs/architecture/core/actionGrammar.md +++ b/ts/docs/architecture/core/actionGrammar.md @@ -602,9 +602,10 @@ tail contract, so the non-tail wrapper builder is only reached when `tailFactoring` is off. **NFA compatibility.** Tail RulesParts are understood only by the -NFA-interpreter matcher (`grammarMatcher.ts`). The NFA compiler / -DFA path (`nfaCompiler.ts`) explicitly throws on encountering one, -so consumers that route through the NFA/DFA path must leave +AST-walking matcher (`grammarMatcher.ts`). The NFA compiler / DFA path +(`nfaCompiler.ts`) explicitly throws on encountering one - this is a known gap and is not yet implemented. NFAs are naturally well-suited to sharing prefix states, but the equivalent frame-sharing +/ variable-forwarding wiring for that backend hasn't been built yet. +Until it is, consumers that route through the NFA/DFA path must leave `tailFactoring` off (and `promoteTailRulesParts`, which produces the same tail-call shape) when calling `optimizeGrammar` / `loadGrammarRules` directly. diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index 6b042fb27d..8bb0cc83f0 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -23,7 +23,7 @@ import { leadingWordBoundaryScriptPrefix } from "./spacingScripts.js"; import { leadingNonSeparatorRun } from "./grammarMatcher.js"; import { getDispatchEffectiveMembers } from "./dispatchHelpers.js"; import { - deriveValue, + deriveEffectiveValue, findSingleValueBearingPart, } from "./grammarValueDeriver.js"; import { @@ -3236,14 +3236,22 @@ function trySubstituteMembers( let bailed = false; const rewriteOne = (m: GrammarRule): GrammarRule => { const renamed = renameAllChildBindings(m.parts, m.value, renameState); + // requireValue: false - a member without a structurally-expressible + // default (e.g. an unbound single-part `phraseSet` / `string`) + // resolves to `undefined` here instead of throwing, so it can + // signal a local bailout below rather than aborting compilation. const effective = renamed.value !== undefined ? renamed.value - : getImplicitDefaultValue({ - ...m, - parts: renamed.parts, - value: renamed.value, - }); + : deriveEffectiveValue( + { + ...m, + parts: renamed.parts, + value: renamed.value, + }, + () => "", + false, + ); if (effective === undefined) { // Member has no explicit value and we can't synthesize // one to substitute (e.g. unbound single-part @@ -3297,25 +3305,6 @@ function trySubstituteMembers( return { dispatch: newDispatch, alternatives: fallback }; } -/** - * Compute a value expression equivalent to what the matcher's - * implicit-default rule would produce for `rule` if it were matched - * standalone. Returns `undefined` if the rule doesn't have a - * structurally-expressible default (e.g. an unbound single-part - * `phraseSet` / `string` whose value depends on the matched text). - * - * Used by the value-substitution branch of `tryPromoteTrailing` to - * fold each member's effective value into the parent's value - * expression. See `deriveValue` (grammarValueDeriver.ts) for - * the shared derivation logic. - */ -function getImplicitDefaultValue( - rule: GrammarRule, -): CompiledValueNode | undefined { - const result = deriveValue(rule); - return result.kind === "value" ? result.value : undefined; -} - // ───────────────────────────────────────────────────────────────────────────── // Structural validation: RulesPart.tailCall contract // ───────────────────────────────────────────────────────────────────────────── diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index 49a2dec562..55e0e5e384 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -80,3 +80,55 @@ export function deriveValue(rule: GrammarRule): ImplicitValueResult { value: { type: "variable", name: result.variable }, }; } + +/** + * Derive the value expression for a rule that has no explicit `->` value: + * a rule with exactly one variable-bearing part (whatever its type - + * wildcard, number, or a bound rules/string/phraseSet part) implicitly + * forwards that part's value, via `deriveValue` above - the single place + * that decides a rule's effective value, explicit or implicit. + * + * Not every rule needs a value - only when `requireValue` is set (i.e. + * the value is actually consumed: top-level action rules, or nested + * rules captured by a parent variable) do ambiguous (2+ variable-bearing + * parts) or missing values throw; otherwise both cases just resolve to + * `undefined`. `describeRule` is only invoked when an error is actually + * thrown, so it's safe to pass a cheap closure. + * + * This function is unaware of any particular compilation backend's + * structural limitations (e.g. the NFA/DFA backend's tailCall gap) - + * callers with such constraints must check for them separately, before + * calling this. + */ +export function deriveEffectiveValue( + rule: GrammarRule, + describeRule: () => string, + requireValue: boolean, +): CompiledValueNode | undefined { + const result = deriveValue(rule); + if (result.kind === "value") { + return result.value; + } + // Not every rule needs a value - only rules whose value is actually + // consumed (top-level action rules, or nested rules captured by a + // parent variable) do. If nothing downstream needs this rule's value, + // neither an ambiguous nor a missing implicit value is an error. + if (!requireValue) { + return undefined; + } + const termsDescription = + rule.parts.length === 1 + ? "has 1 term" + : `has ${rule.parts.length} terms`; + if (result.kind === "ambiguous") { + throw new Error( + `${describeRule()} ${termsDescription} but no value expression, ` + + `and more than one part carries a variable - the implicit value is ambiguous. ` + + `Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.`, + ); + } + throw new Error( + `${describeRule()} ${termsDescription} but no value expression, and no part carries a variable. ` + + `Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.`, + ); +} diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index 90425913b7..a6953682ed 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -11,7 +11,6 @@ import { RulesPart, PhraseSetPart, CompiledSpacingMode, - CompiledValueNode, getCapturedVariableName, createRulesPart, } from "./grammarTypes.js"; @@ -22,7 +21,7 @@ import { ValueExpression, } from "./environment.js"; import { normalizeToken } from "./nfaMatcher.js"; -import { deriveValue } from "./grammarValueDeriver.js"; +import { deriveEffectiveValue } from "./grammarValueDeriver.js"; // Scripts that require a word-boundary separator between adjacent tokens. // CJK and other logographic/syllabic scripts are NOT included — no separator needed. @@ -355,6 +354,23 @@ function normalizeRule( rule: GrammarRule, cache: Map, ): GrammarRule { + // tailCall is a structural NFA/DFA backend gap (NYI, not fundamental - + // see `tailCallNotYetSupportedMessage`), unrelated to value derivation + // or rule normalization. `normalizeRule` recursively visits every + // rule exactly once (top-level and nested, via the cache-based walk + // in `normalizeRulesArray`) before any compilation begins, making + // this the single earliest point to reject it - fails fast, and + // lets every downstream consumer (including `deriveEffectiveValue`) + // stay unaware tailCall exists at all. + const tailCallPart = rule.parts.find( + (p): p is RulesPart => p.type === "rules" && !!p.tailCall, + ); + if (tailCallPart !== undefined) { + throw new Error( + `normalizeGrammar: ${tailCallNotYetSupportedMessage(tailCallPart.name)}`, + ); + } + // First, normalize all nested RulesParts const normalizedParts = rule.parts.map((part) => normalizePart(part, cache), @@ -453,55 +469,20 @@ function stripDispatch(part: RulesPart): RulesPart { } /** - * Derive the value expression for a rule that has no explicit `->` value: - * a rule with exactly one variable-bearing part (whatever its type - - * wildcard, number, or a bound rules/string/phraseSet part) implicitly - * forwards that part's value, via the shared `deriveValue` - the single - * place that decides a rule's effective value, explicit or implicit. - * Not every rule needs a value - only when `requireValue` is set (i.e. - * the value is actually consumed: top-level action rules, or nested - * rules captured by a parent variable) do ambiguous (2+ variable-bearing - * parts) or missing values throw; otherwise both cases just resolve to - * `undefined`. + * Message for the NFA/DFA backend's tailCall gap, shared by every throw + * site so they report consistent wording. This is a known NYI gap, not + * a fundamental limitation: tail-call RulesParts let the AST-walking + * matcher share the parent's stack frame and forward a member's value + * directly as the parent rule's value (see `RulesPart.tailCall` in + * grammarTypes.ts) - NFAs are naturally well-suited to sharing prefix + * states, but the equivalent frame-sharing/variable-forwarding wiring + * for this state-machine backend hasn't been built yet. */ -function deriveEffectiveValue( - rule: GrammarRule, - describeRule: () => string, - requireValue: boolean, -): CompiledValueNode | undefined { - const result = deriveValue(rule); - if (result.kind === "value") { - return result.value; - } - // Not every rule needs a value - only rules whose value is actually - // consumed (top-level action rules, or nested rules captured by a - // parent variable) do. If nothing downstream needs this rule's value, - // neither an ambiguous nor a missing implicit value is an error. - if (!requireValue) { - return undefined; - } - const termsDescription = - rule.parts.length === 1 - ? "has 1 term" - : `has ${rule.parts.length} terms`; - if (result.kind === "ambiguous") { - throw new Error( - `${describeRule()} ${termsDescription} but no value expression, ` + - `and more than one part carries a variable - the implicit value is ambiguous. ` + - `Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.`, - ); - } - const hasTailCall = rule.parts.some( - (p) => p.type === "rules" && p.tailCall, - ); - throw new Error( - `${describeRule()} ${termsDescription} but no value expression, and no part carries a variable. ` + - `Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.` + - (hasTailCall - ? " This rule contains a tailCall RulesPart, which the NFA compiler " + - "does not support - disable `tailFactoring` / `promoteTailRulesParts` " + - "in the grammar optimizer for NFA/DFA paths." - : ""), +function tailCallNotYetSupportedMessage(partName: string | undefined): string { + return ( + `tail RulesPart (name='${partName ?? ""}') is not yet supported by the ` + + "NFA/DFA backend - disable `tailFactoring` / `promoteTailRulesParts` in the " + + "grammar optimizer for NFA/DFA paths." ); } @@ -1413,9 +1394,7 @@ function compileRulesPart( ): number { if (part.tailCall) { throw new Error( - `compileRulesPart: tail RulesParts are not supported by the NFA compiler ` + - `(part.name='${part.name ?? ""}'). ` + - "Disable `tailFactoring` in the grammar optimizer for NFA/DFA paths.", + `compileRulesPart: ${tailCallNotYetSupportedMessage(part.name)}`, ); } if (part.alternatives.length === 0) { @@ -1494,9 +1473,7 @@ function compileRulesPartWithSlots( ): number { if (part.tailCall) { throw new Error( - `compileRulesPartWithSlots: tail RulesParts are not supported by the NFA compiler ` + - `(part.name='${part.name ?? ""}'). ` + - "Disable `tailFactoring` in the grammar optimizer for NFA/DFA paths.", + `compileRulesPartWithSlots: ${tailCallNotYetSupportedMessage(part.name)}`, ); } if (part.alternatives.length === 0) { diff --git a/ts/packages/actionGrammar/test/grammarOptimizerTailFactoring.spec.ts b/ts/packages/actionGrammar/test/grammarOptimizerTailFactoring.spec.ts index 9ee5c2ef80..430ca26306 100644 --- a/ts/packages/actionGrammar/test/grammarOptimizerTailFactoring.spec.ts +++ b/ts/packages/actionGrammar/test/grammarOptimizerTailFactoring.spec.ts @@ -368,7 +368,42 @@ describe("Grammar Optimizer - tailFactoring + NFA compatibility", () => { }, }); expect(() => compileGrammarToNFA(optimized)).toThrow( - /tail RulesParts are not supported by the NFA compiler/, + /tail RulesPart .* is not yet supported by the NFA\/DFA backend/, + ); + }); + + // Regression test for a shape that previously escaped the + // (now-removed) tailCall diagnostic embedded in + // `deriveEffectiveValue`: a rule whose *entire* `parts` array is + // just the tailCall RulesPart (no preceding prefix parts). Such a + // rule is `isPassthroughRule`-eligible and gets stamped with an + // explicit `value` during normalization *before* value-derivation + // ever runs, so a tailCall check embedded only in the + // "missing value" throw path would never fire for it - it would + // silently fall through to the later, differently-worded throw in + // `compileRulesPart`/`compileRulesPartWithSlots` instead. The + // check now lives in `normalizeRule`, ahead of that transform, so + // it catches this shape too with the same message. + it("NFA compile of a single-part tailCall rule throws the same descriptive error", async () => { + const { compileGrammarToNFA } = await import("../src/nfaCompiler.js"); + const memberA: GrammarRule = { + parts: [createStringPart(["alpha"])], + value: { type: "literal", value: "a" }, + }; + const memberB: GrammarRule = { + parts: [createStringPart(["beta"])], + value: { type: "literal", value: "b" }, + }; + const tailPart: RulesPart = { + type: "rules", + optional: undefined, + variable: undefined, + alternatives: [memberA, memberB], + tailCall: true, + }; + const grammar: Grammar = { alternatives: [{ parts: [tailPart] }] }; + expect(() => compileGrammarToNFA(grammar)).toThrow( + /tail RulesPart .* is not yet supported by the NFA\/DFA backend/, ); }); }); diff --git a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts index 85d59a05cd..2ecbdd6042 100644 --- a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts +++ b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts @@ -1,9 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// Direct unit tests for the pure helpers in grammarValueDeriver.ts, -// shared by grammarOptimizer.ts (`getImplicitDefaultValue`, -// `checkForwardingPromotable`) and nfaCompiler.ts (`deriveEffectiveValue`). +// Direct unit tests for the pure helpers in grammarValueDeriver.ts, shared +// by grammarOptimizer.ts (`checkForwardingPromotable`) and +// nfaCompiler.ts (via `deriveEffectiveValue`, which - along with +// `deriveValue` - now lives in this module). // The consumers' own spec files (grammarOptimizerPromoteTail.spec.ts, // nfaFactoredPrefixValues.spec.ts) already cover end-to-end wiring and // throw/promote decisions; these tests isolate the pure derivation logic diff --git a/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts b/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts index 79637874ae..66f47b7269 100644 --- a/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts +++ b/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts @@ -1,9 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// Direct, hand-built-AST coverage of `deriveEffectiveValue`'s (nfaCompiler.ts) -// error paths when a top-level rule has no explicit `->` value and no -// single variable-bearing part to forward implicitly. These bypass the +// Direct, hand-built-AST coverage of `deriveEffectiveValue`'s +// (grammarValueDeriver.ts, called from nfaCompiler.ts) error paths when a +// top-level rule has no explicit `->` value and no single +// variable-bearing part to forward implicitly. These bypass the // loader's own validation to exercise `compileGrammarToNFA`'s throw // behavior directly, independent of any particular grammar-source // scenario (e.g. shared-prefix factoring - see From c74e366523055856c4fdd9fbeda00d004443f5c6 Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 16:33:55 -0700 Subject: [PATCH 18/29] Clean up comments --- .../actionGrammar/src/grammarOptimizer.ts | 4 --- .../actionGrammar/src/grammarValueDeriver.ts | 27 +++++-------------- ts/packages/actionGrammar/src/nfaCompiler.ts | 23 +++++----------- .../grammarOptimizerTailFactoring.spec.ts | 15 +++-------- 4 files changed, 16 insertions(+), 53 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index 8bb0cc83f0..53933a474e 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -3236,10 +3236,6 @@ function trySubstituteMembers( let bailed = false; const rewriteOne = (m: GrammarRule): GrammarRule => { const renamed = renameAllChildBindings(m.parts, m.value, renameState); - // requireValue: false - a member without a structurally-expressible - // default (e.g. an unbound single-part `phraseSet` / `string`) - // resolves to `undefined` here instead of throwing, so it can - // signal a local bailout below rather than aborting compilation. const effective = renamed.value !== undefined ? renamed.value diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index 55e0e5e384..43515dad82 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -82,23 +82,12 @@ export function deriveValue(rule: GrammarRule): ImplicitValueResult { } /** - * Derive the value expression for a rule that has no explicit `->` value: - * a rule with exactly one variable-bearing part (whatever its type - - * wildcard, number, or a bound rules/string/phraseSet part) implicitly - * forwards that part's value, via `deriveValue` above - the single place - * that decides a rule's effective value, explicit or implicit. - * - * Not every rule needs a value - only when `requireValue` is set (i.e. - * the value is actually consumed: top-level action rules, or nested - * rules captured by a parent variable) do ambiguous (2+ variable-bearing - * parts) or missing values throw; otherwise both cases just resolve to - * `undefined`. `describeRule` is only invoked when an error is actually - * thrown, so it's safe to pass a cheap closure. - * - * This function is unaware of any particular compilation backend's - * structural limitations (e.g. the NFA/DFA backend's tailCall gap) - - * callers with such constraints must check for them separately, before - * calling this. + * Derive a rule's effective value expression via `deriveValue`, throwing + * if it's required and missing/ambiguous. Not every rule needs a value - + * only when `requireValue` is set (top-level action rules, or nested + * rules captured by a parent variable) do ambiguous/missing values + * throw; otherwise both resolve to `undefined`. `describeRule` is only + * invoked when an error is thrown. */ export function deriveEffectiveValue( rule: GrammarRule, @@ -109,10 +98,6 @@ export function deriveEffectiveValue( if (result.kind === "value") { return result.value; } - // Not every rule needs a value - only rules whose value is actually - // consumed (top-level action rules, or nested rules captured by a - // parent variable) do. If nothing downstream needs this rule's value, - // neither an ambiguous nor a missing implicit value is an error. if (!requireValue) { return undefined; } diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index a6953682ed..66dd6a51b0 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -354,14 +354,10 @@ function normalizeRule( rule: GrammarRule, cache: Map, ): GrammarRule { - // tailCall is a structural NFA/DFA backend gap (NYI, not fundamental - - // see `tailCallNotYetSupportedMessage`), unrelated to value derivation - // or rule normalization. `normalizeRule` recursively visits every - // rule exactly once (top-level and nested, via the cache-based walk - // in `normalizeRulesArray`) before any compilation begins, making - // this the single earliest point to reject it - fails fast, and - // lets every downstream consumer (including `deriveEffectiveValue`) - // stay unaware tailCall exists at all. + // tailCall is an NYI NFA/DFA backend gap, unrelated to value + // derivation. Checked here since normalizeRule visits every rule + // exactly once (top-level and nested) before compilation begins - + // the single earliest point to reject it. const tailCallPart = rule.parts.find( (p): p is RulesPart => p.type === "rules" && !!p.tailCall, ); @@ -469,14 +465,9 @@ function stripDispatch(part: RulesPart): RulesPart { } /** - * Message for the NFA/DFA backend's tailCall gap, shared by every throw - * site so they report consistent wording. This is a known NYI gap, not - * a fundamental limitation: tail-call RulesParts let the AST-walking - * matcher share the parent's stack frame and forward a member's value - * directly as the parent rule's value (see `RulesPart.tailCall` in - * grammarTypes.ts) - NFAs are naturally well-suited to sharing prefix - * states, but the equivalent frame-sharing/variable-forwarding wiring - * for this state-machine backend hasn't been built yet. + * Message for the NFA/DFA backend's tailCall gap (NYI, not a fundamental + * limitation - see `RulesPart.tailCall` in grammarTypes.ts), shared by + * every throw site for consistent wording. */ function tailCallNotYetSupportedMessage(partName: string | undefined): string { return ( diff --git a/ts/packages/actionGrammar/test/grammarOptimizerTailFactoring.spec.ts b/ts/packages/actionGrammar/test/grammarOptimizerTailFactoring.spec.ts index 430ca26306..09447022db 100644 --- a/ts/packages/actionGrammar/test/grammarOptimizerTailFactoring.spec.ts +++ b/ts/packages/actionGrammar/test/grammarOptimizerTailFactoring.spec.ts @@ -372,18 +372,9 @@ describe("Grammar Optimizer - tailFactoring + NFA compatibility", () => { ); }); - // Regression test for a shape that previously escaped the - // (now-removed) tailCall diagnostic embedded in - // `deriveEffectiveValue`: a rule whose *entire* `parts` array is - // just the tailCall RulesPart (no preceding prefix parts). Such a - // rule is `isPassthroughRule`-eligible and gets stamped with an - // explicit `value` during normalization *before* value-derivation - // ever runs, so a tailCall check embedded only in the - // "missing value" throw path would never fire for it - it would - // silently fall through to the later, differently-worded throw in - // `compileRulesPart`/`compileRulesPartWithSlots` instead. The - // check now lives in `normalizeRule`, ahead of that transform, so - // it catches this shape too with the same message. + // Regression test: a rule whose entire `parts` array is just the + // tailCall RulesPart (no prefix) is `isPassthroughRule`-eligible, so + // it must be caught before that normalization transform runs. it("NFA compile of a single-part tailCall rule throws the same descriptive error", async () => { const { compileGrammarToNFA } = await import("../src/nfaCompiler.js"); const memberA: GrammarRule = { From df3d9f718c0f039be7fad533757fc0be54f49a77 Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 17:05:06 -0700 Subject: [PATCH 19/29] Remove deriveValue --- .../actionGrammar/src/grammarValueDeriver.ts | 58 +++++-------------- .../test/grammarValueDeriver.spec.ts | 48 +++++++-------- .../test/nfaCompilerValueErrors.spec.ts | 9 +-- 3 files changed, 38 insertions(+), 77 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index 43515dad82..7240f398ac 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -44,46 +44,10 @@ export function findSingleValueBearingPart( } /** - * Result of `deriveValue`: either a concrete value (the rule's own `->` - * expression, or a single variable-bearing part's implicitly forwarded - * value), `"ambiguous"` (2+ parts carry a variable and there is no - * explicit value - callers decide whether that's an error), or `"none"` - * (no explicit value and no part carries a variable, or the rule has no - * parts at all). - */ -export type ImplicitValueResult = - | { kind: "value"; value: CompiledValueNode } - | { kind: "ambiguous" } - | { kind: "none" }; - -/** - * Compute a rule's effective value expression: its explicit `->` value - * when present, otherwise the implicit value it would produce (via - * `findSingleValueBearingPart`). - */ -export function deriveValue(rule: GrammarRule): ImplicitValueResult { - if (rule.value !== undefined) { - return { kind: "value", value: rule.value }; - } - if (rule.parts.length === 0) { - return { kind: "none" }; - } - const result = findSingleValueBearingPart(rule.parts); - if (result === "ambiguous") { - return { kind: "ambiguous" }; - } - if (result === undefined) { - return { kind: "none" }; - } - return { - kind: "value", - value: { type: "variable", name: result.variable }, - }; -} - -/** - * Derive a rule's effective value expression via `deriveValue`, throwing - * if it's required and missing/ambiguous. Not every rule needs a value - + * Derive a rule's effective value expression: its explicit `->` value + * when present, otherwise the implicit value from a single + * variable-bearing part (via `findSingleValueBearingPart`). Throws if a + * value is required and missing/ambiguous. Not every rule needs a value - * only when `requireValue` is set (top-level action rules, or nested * rules captured by a parent variable) do ambiguous/missing values * throw; otherwise both resolve to `undefined`. `describeRule` is only @@ -94,9 +58,15 @@ export function deriveEffectiveValue( describeRule: () => string, requireValue: boolean, ): CompiledValueNode | undefined { - const result = deriveValue(rule); - if (result.kind === "value") { - return result.value; + if (rule.value !== undefined) { + return rule.value; + } + const result = + rule.parts.length === 0 + ? undefined + : findSingleValueBearingPart(rule.parts); + if (result !== undefined && result !== "ambiguous") { + return { type: "variable", name: result.variable }; } if (!requireValue) { return undefined; @@ -105,7 +75,7 @@ export function deriveEffectiveValue( rule.parts.length === 1 ? "has 1 term" : `has ${rule.parts.length} terms`; - if (result.kind === "ambiguous") { + if (result === "ambiguous") { throw new Error( `${describeRule()} ${termsDescription} but no value expression, ` + `and more than one part carries a variable - the implicit value is ambiguous. ` + diff --git a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts index 2ecbdd6042..153451f3cf 100644 --- a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts +++ b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts @@ -2,17 +2,17 @@ // Licensed under the MIT License. // Direct unit tests for the pure helpers in grammarValueDeriver.ts, shared -// by grammarOptimizer.ts (`checkForwardingPromotable`) and -// nfaCompiler.ts (via `deriveEffectiveValue`, which - along with -// `deriveValue` - now lives in this module). +// by grammarOptimizer.ts (`checkForwardingPromotable`) and nfaCompiler.ts +// (via `deriveEffectiveValue`, which lives in this module). // The consumers' own spec files (grammarOptimizerPromoteTail.spec.ts, // nfaFactoredPrefixValues.spec.ts) already cover end-to-end wiring and -// throw/promote decisions; these tests isolate the pure derivation logic -// itself. +// promote decisions; nfaCompilerValueErrors.spec.ts covers +// `deriveEffectiveValue`'s throwing (requireValue: true) paths. These +// tests isolate its non-throwing (requireValue: false) derivation logic. import { findSingleValueBearingPart, - deriveValue, + deriveEffectiveValue, } from "../src/grammarValueDeriver.js"; import { createStringPart, @@ -67,7 +67,10 @@ describe("findSingleValueBearingPart", () => { }); }); -describe("deriveValue", () => { +describe("deriveEffectiveValue (requireValue: false)", () => { + const derive = (rule: GrammarRule) => + deriveEffectiveValue(rule, () => "", false); + it("returns the rule's explicit value when present, ignoring parts", () => { const rule: GrammarRule = { parts: [ @@ -76,20 +79,17 @@ describe("deriveValue", () => { ], value: { type: "literal", value: "explicit" }, }; - expect(deriveValue(rule)).toEqual({ - kind: "value", - value: { type: "literal", value: "explicit" }, - }); + expect(derive(rule)).toEqual({ type: "literal", value: "explicit" }); }); - it('returns "none" for a rule with no parts and no value', () => { + it("returns undefined for a rule with no parts and no value", () => { const rule: GrammarRule = { parts: [] }; - expect(deriveValue(rule)).toEqual({ kind: "none" }); + expect(derive(rule)).toBeUndefined(); }); - it('returns "none" for a single part with no variable', () => { + it("returns undefined for a single part with no variable", () => { const rule: GrammarRule = { parts: [createStringPart(["foo"])] }; - expect(deriveValue(rule)).toEqual({ kind: "none" }); + expect(derive(rule)).toBeUndefined(); }); it("forwards a single part's variable as the implicit value", () => { @@ -99,10 +99,7 @@ describe("deriveValue", () => { createWildcardPart("x", "wildcard"), ], }; - expect(deriveValue(rule)).toEqual({ - kind: "value", - value: { type: "variable", name: "x" }, - }); + expect(derive(rule)).toEqual({ type: "variable", name: "x" }); }); it("forwards a nested RulesPart's captured variable", () => { @@ -115,26 +112,23 @@ describe("deriveValue", () => { const rule: GrammarRule = { parts: [createStringPart(["prefix"]), nested], }; - expect(deriveValue(rule)).toEqual({ - kind: "value", - value: { type: "variable", name: "chosen" }, - }); + expect(derive(rule)).toEqual({ type: "variable", name: "chosen" }); }); - it('returns "ambiguous" when 2+ parts carry a variable and there is no explicit value', () => { + it("returns undefined when 2+ parts carry a variable and there is no explicit value", () => { const rule: GrammarRule = { parts: [ createStringPart(["foo"], "x"), createStringPart(["bar"], "y"), ], }; - expect(deriveValue(rule)).toEqual({ kind: "ambiguous" }); + expect(derive(rule)).toBeUndefined(); }); - it('returns "none" for a multi-part rule where nothing carries a variable', () => { + it("returns undefined for a multi-part rule where nothing carries a variable", () => { const rule: GrammarRule = { parts: [createStringPart(["foo"]), createStringPart(["bar"])], }; - expect(deriveValue(rule)).toEqual({ kind: "none" }); + expect(derive(rule)).toBeUndefined(); }); }); diff --git a/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts b/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts index 66f47b7269..cd3aef2740 100644 --- a/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts +++ b/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts @@ -41,12 +41,9 @@ describe("deriveEffectiveValue error messages", () => { // and no top-level `value`. String-literal single-part rules are // auto-normalized (isSingleLiteralRule) to stamp a matched-text // value, so this uses an unbound phraseSet part instead - a - // shape normalizeRule does not special-case. Before - // deriveEffectiveValue was consolidated to call `deriveValue` - // uniformly, a rule this shape short-circuited to `rule.value` - // (undefined) without throwing. It now throws like any other - // value-less rule - guard here that the message accurately - // reflects a 1-term rule instead of the multi-term wording. + // shape normalizeRule does not special-case. Guard here that the + // message accurately reflects a 1-term rule instead of the + // multi-term wording. const grammar: Grammar = { alternatives: [{ parts: [createPhraseSetPart("Polite")] }], }; From ee8ae8a100b5e0811f8461ca9546940b96c04899 Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 17:11:36 -0700 Subject: [PATCH 20/29] Simplify further --- .../actionGrammar/src/grammarOptimizer.ts | 17 +++++------------ .../actionGrammar/src/grammarValueDeriver.ts | 5 +---- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index 53933a474e..a1b51c81b5 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -3236,18 +3236,11 @@ function trySubstituteMembers( let bailed = false; const rewriteOne = (m: GrammarRule): GrammarRule => { const renamed = renameAllChildBindings(m.parts, m.value, renameState); - const effective = - renamed.value !== undefined - ? renamed.value - : deriveEffectiveValue( - { - ...m, - parts: renamed.parts, - value: renamed.value, - }, - () => "", - false, - ); + const effective = deriveEffectiveValue( + { ...m, parts: renamed.parts, value: renamed.value }, + () => "", + false, + ); if (effective === undefined) { // Member has no explicit value and we can't synthesize // one to substitute (e.g. unbound single-part diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index 7240f398ac..6a25175a06 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -61,10 +61,7 @@ export function deriveEffectiveValue( if (rule.value !== undefined) { return rule.value; } - const result = - rule.parts.length === 0 - ? undefined - : findSingleValueBearingPart(rule.parts); + const result = findSingleValueBearingPart(rule.parts); if (result !== undefined && result !== "ambiguous") { return { type: "variable", name: result.variable }; } From d33185f448f1904f30fe18bbba478fa8d2f101fd Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 17:21:35 -0700 Subject: [PATCH 21/29] Additional cleanup of deadcode paths, add default parameters, further simplification --- .../actionGrammar/src/grammarOptimizer.ts | 10 +++---- .../actionGrammar/src/grammarValueDeriver.ts | 9 ++++--- ts/packages/actionGrammar/src/nfaCompiler.ts | 27 +++---------------- .../test/grammarValueDeriver.spec.ts | 3 +-- 4 files changed, 14 insertions(+), 35 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index a1b51c81b5..debcddee88 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -3236,11 +3236,11 @@ function trySubstituteMembers( let bailed = false; const rewriteOne = (m: GrammarRule): GrammarRule => { const renamed = renameAllChildBindings(m.parts, m.value, renameState); - const effective = deriveEffectiveValue( - { ...m, parts: renamed.parts, value: renamed.value }, - () => "", - false, - ); + const effective = deriveEffectiveValue({ + ...m, + parts: renamed.parts, + value: renamed.value, + }); if (effective === undefined) { // Member has no explicit value and we can't synthesize // one to substitute (e.g. unbound single-part diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index 6a25175a06..86a63bdf0f 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -55,8 +55,8 @@ export function findSingleValueBearingPart( */ export function deriveEffectiveValue( rule: GrammarRule, - describeRule: () => string, - requireValue: boolean, + describeRule?: () => string, + requireValue = false, ): CompiledValueNode | undefined { if (rule.value !== undefined) { return rule.value; @@ -72,15 +72,16 @@ export function deriveEffectiveValue( rule.parts.length === 1 ? "has 1 term" : `has ${rule.parts.length} terms`; + const description = describeRule?.() ?? ""; if (result === "ambiguous") { throw new Error( - `${describeRule()} ${termsDescription} but no value expression, ` + + `${description} ${termsDescription} but no value expression, ` + `and more than one part carries a variable - the implicit value is ambiguous. ` + `Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.`, ); } throw new Error( - `${describeRule()} ${termsDescription} but no value expression, and no part carries a variable. ` + + `${description} ${termsDescription} but no value expression, and no part carries a variable. ` + `Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.`, ); } diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index 66dd6a51b0..758f3142da 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -363,7 +363,9 @@ function normalizeRule( ); if (tailCallPart !== undefined) { throw new Error( - `normalizeGrammar: ${tailCallNotYetSupportedMessage(tailCallPart.name)}`, + `normalizeGrammar: tail RulesPart (name='${tailCallPart.name ?? ""}') is not yet supported by the ` + + "NFA/DFA backend - disable `tailFactoring` / `promoteTailRulesParts` in the " + + "grammar optimizer for NFA/DFA paths.", ); } @@ -464,19 +466,6 @@ function stripDispatch(part: RulesPart): RulesPart { }); } -/** - * Message for the NFA/DFA backend's tailCall gap (NYI, not a fundamental - * limitation - see `RulesPart.tailCall` in grammarTypes.ts), shared by - * every throw site for consistent wording. - */ -function tailCallNotYetSupportedMessage(partName: string | undefined): string { - return ( - `tail RulesPart (name='${partName ?? ""}') is not yet supported by the ` + - "NFA/DFA backend - disable `tailFactoring` / `promoteTailRulesParts` in the " + - "grammar optimizer for NFA/DFA paths." - ); -} - /** * Collect all variable names from a rule's parts * Returns them in order of appearance @@ -1383,11 +1372,6 @@ function compileRulesPart( checkedVariables?: Set, overrideVariableName?: string, ): number { - if (part.tailCall) { - throw new Error( - `compileRulesPart: ${tailCallNotYetSupportedMessage(part.name)}`, - ); - } if (part.alternatives.length === 0) { // Empty rules - epsilon transition builder.addEpsilonTransition(fromState, toState); @@ -1462,11 +1446,6 @@ function compileRulesPartWithSlots( toState: number, context: RuleCompilationContext, ): number { - if (part.tailCall) { - throw new Error( - `compileRulesPartWithSlots: ${tailCallNotYetSupportedMessage(part.name)}`, - ); - } if (part.alternatives.length === 0) { // Empty rules - epsilon transition builder.addEpsilonTransition(fromState, toState); diff --git a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts index 153451f3cf..e7f24ac5dd 100644 --- a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts +++ b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts @@ -68,8 +68,7 @@ describe("findSingleValueBearingPart", () => { }); describe("deriveEffectiveValue (requireValue: false)", () => { - const derive = (rule: GrammarRule) => - deriveEffectiveValue(rule, () => "", false); + const derive = (rule: GrammarRule) => deriveEffectiveValue(rule); it("returns the rule's explicit value when present, ignoring parts", () => { const rule: GrammarRule = { From 82dedd2692a986e5605bdc367469f919d5f3c32a Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 17:27:16 -0700 Subject: [PATCH 22/29] Also swap order of describeRule and requireValue in deriveEffectiveValue --- ts/packages/actionGrammar/src/grammarValueDeriver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index 86a63bdf0f..199e971138 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -55,8 +55,8 @@ export function findSingleValueBearingPart( */ export function deriveEffectiveValue( rule: GrammarRule, - describeRule?: () => string, requireValue = false, + describeRule?: () => string, ): CompiledValueNode | undefined { if (rule.value !== undefined) { return rule.value; From a5b7f34020a829fd2ea721cec2bf6939dd02963d Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 17:28:42 -0700 Subject: [PATCH 23/29] Add change accidentally left out --- ts/packages/actionGrammar/src/nfaCompiler.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ts/packages/actionGrammar/src/nfaCompiler.ts b/ts/packages/actionGrammar/src/nfaCompiler.ts index 758f3142da..b37ab0ab6b 100644 --- a/ts/packages/actionGrammar/src/nfaCompiler.ts +++ b/ts/packages/actionGrammar/src/nfaCompiler.ts @@ -610,8 +610,8 @@ export function compileGrammarToNFA(grammar: Grammar, name?: string): NFA { // Check for single-variable / factored rules with no explicit -> value. const effectiveValue = deriveEffectiveValue( rule, - () => `Grammar rule at index ${ruleIndex}`, true, + () => `Grammar rule at index ${ruleIndex}`, ); const ruleEntry = builder.createState(false); @@ -1495,11 +1495,7 @@ function compileRulesPartWithSlots( // Compile and store the action value on the entry state FIRST // We need to know if there's a value before deciding about environments - const effectiveValue = deriveEffectiveValue( - rule, - () => "Nested grammar rule", - false, - ); + const effectiveValue = deriveEffectiveValue(rule); let compiledValue: ValueExpression | undefined; let nestedCompletionActionName: string | undefined; From 5cd35b5b5db9e96727a8b6ad8f1ef4431aa64b8f Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 17:31:45 -0700 Subject: [PATCH 24/29] Simplify further --- .../actionGrammar/src/grammarValueDeriver.ts | 24 +++++++----------- .../test/grammarValueDeriver.spec.ts | 25 ++++++++++++------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index 199e971138..bede42aa04 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -2,13 +2,10 @@ // Licensed under the MIT License. /** - * Structural derivation of a rule's *effective* value expression: its - * explicit `->` expression when present, otherwise an implicit - * forwarding value derived from the rule's parts. - * - * A rule with exactly one variable-bearing part implicitly forwards that - * part's value. This is the single source of truth for that - * derivation, kept independent of any particular compilation backend. + * Derives a rule's *effective* value expression: its explicit `->` + * expression when present, otherwise the implicit value forwarded from + * a single variable-bearing part. Single source of truth for this + * derivation, shared by the grammar optimizer and the NFA compiler. */ import type { @@ -73,15 +70,12 @@ export function deriveEffectiveValue( ? "has 1 term" : `has ${rule.parts.length} terms`; const description = describeRule?.() ?? ""; - if (result === "ambiguous") { - throw new Error( - `${description} ${termsDescription} but no value expression, ` + - `and more than one part carries a variable - the implicit value is ambiguous. ` + - `Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.`, - ); - } + const reason = + result === "ambiguous" + ? "and more than one part carries a variable - the implicit value is ambiguous" + : "and no part carries a variable"; throw new Error( - `${description} ${termsDescription} but no value expression, and no part carries a variable. ` + + `${description} ${termsDescription} but no value expression, ${reason}. ` + `Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.`, ); } diff --git a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts index e7f24ac5dd..2ecf750c4e 100644 --- a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts +++ b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts @@ -68,8 +68,6 @@ describe("findSingleValueBearingPart", () => { }); describe("deriveEffectiveValue (requireValue: false)", () => { - const derive = (rule: GrammarRule) => deriveEffectiveValue(rule); - it("returns the rule's explicit value when present, ignoring parts", () => { const rule: GrammarRule = { parts: [ @@ -78,17 +76,20 @@ describe("deriveEffectiveValue (requireValue: false)", () => { ], value: { type: "literal", value: "explicit" }, }; - expect(derive(rule)).toEqual({ type: "literal", value: "explicit" }); + expect(deriveEffectiveValue(rule)).toEqual({ + type: "literal", + value: "explicit", + }); }); it("returns undefined for a rule with no parts and no value", () => { const rule: GrammarRule = { parts: [] }; - expect(derive(rule)).toBeUndefined(); + expect(deriveEffectiveValue(rule)).toBeUndefined(); }); it("returns undefined for a single part with no variable", () => { const rule: GrammarRule = { parts: [createStringPart(["foo"])] }; - expect(derive(rule)).toBeUndefined(); + expect(deriveEffectiveValue(rule)).toBeUndefined(); }); it("forwards a single part's variable as the implicit value", () => { @@ -98,7 +99,10 @@ describe("deriveEffectiveValue (requireValue: false)", () => { createWildcardPart("x", "wildcard"), ], }; - expect(derive(rule)).toEqual({ type: "variable", name: "x" }); + expect(deriveEffectiveValue(rule)).toEqual({ + type: "variable", + name: "x", + }); }); it("forwards a nested RulesPart's captured variable", () => { @@ -111,7 +115,10 @@ describe("deriveEffectiveValue (requireValue: false)", () => { const rule: GrammarRule = { parts: [createStringPart(["prefix"]), nested], }; - expect(derive(rule)).toEqual({ type: "variable", name: "chosen" }); + expect(deriveEffectiveValue(rule)).toEqual({ + type: "variable", + name: "chosen", + }); }); it("returns undefined when 2+ parts carry a variable and there is no explicit value", () => { @@ -121,13 +128,13 @@ describe("deriveEffectiveValue (requireValue: false)", () => { createStringPart(["bar"], "y"), ], }; - expect(derive(rule)).toBeUndefined(); + expect(deriveEffectiveValue(rule)).toBeUndefined(); }); it("returns undefined for a multi-part rule where nothing carries a variable", () => { const rule: GrammarRule = { parts: [createStringPart(["foo"]), createStringPart(["bar"])], }; - expect(derive(rule)).toBeUndefined(); + expect(deriveEffectiveValue(rule)).toBeUndefined(); }); }); From f6c5076f5a53fac2c8184eceb659ee26c357b6d3 Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 17:33:24 -0700 Subject: [PATCH 25/29] Update test --- .../actionGrammar/test/nfaCompilerValueErrors.spec.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts b/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts index cd3aef2740..eae9a511bc 100644 --- a/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts +++ b/ts/packages/actionGrammar/test/nfaCompilerValueErrors.spec.ts @@ -36,14 +36,12 @@ describe("deriveEffectiveValue error messages", () => { ); }); - it("throws an accurate error (not 'Multi-term') for a single-part rule with no variable and no value", () => { + it("throws an accurate error for a single-part rule with no variable and no value", () => { // Raw AST with exactly one part, that part carrying no variable // and no top-level `value`. String-literal single-part rules are // auto-normalized (isSingleLiteralRule) to stamp a matched-text // value, so this uses an unbound phraseSet part instead - a - // shape normalizeRule does not special-case. Guard here that the - // message accurately reflects a 1-term rule instead of the - // multi-term wording. + // shape normalizeRule does not special-case. const grammar: Grammar = { alternatives: [{ parts: [createPhraseSetPart("Polite")] }], }; @@ -54,7 +52,6 @@ describe("deriveEffectiveValue error messages", () => { message = (e as Error).message; } expect(message).toMatch(/has 1 term but no value expression/); - expect(message).not.toMatch(/Multi-term/); }); it("throws an accurate error for a zero-part rule with no value", () => { From 25cfbf1b453644757140ce0abda0adcb276c3ef4 Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 17:36:56 -0700 Subject: [PATCH 26/29] Clean up unnecessary commenting --- .../actionGrammar/src/grammarValueDeriver.ts | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index bede42aa04..3462929296 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -2,10 +2,8 @@ // Licensed under the MIT License. /** - * Derives a rule's *effective* value expression: its explicit `->` - * expression when present, otherwise the implicit value forwarded from - * a single variable-bearing part. Single source of truth for this - * derivation, shared by the grammar optimizer and the NFA compiler. + * Value-derivation logic shared by grammarOptimizer.ts and + * nfaCompiler.ts: computing a rule's effective value expression. */ import type { @@ -41,13 +39,13 @@ export function findSingleValueBearingPart( } /** - * Derive a rule's effective value expression: its explicit `->` value - * when present, otherwise the implicit value from a single - * variable-bearing part (via `findSingleValueBearingPart`). Throws if a - * value is required and missing/ambiguous. Not every rule needs a value - - * only when `requireValue` is set (top-level action rules, or nested - * rules captured by a parent variable) do ambiguous/missing values - * throw; otherwise both resolve to `undefined`. `describeRule` is only + * A rule's explicit `->` value when present, otherwise the implicit + * value from a single variable-bearing part (via + * `findSingleValueBearingPart`). Throws if a value is required and + * missing/ambiguous. Not every rule needs a value - only when + * `requireValue` is set (top-level action rules, or nested rules + * captured by a parent variable) do ambiguous/missing values throw; + * otherwise both resolve to `undefined`. `describeRule` is only * invoked when an error is thrown. */ export function deriveEffectiveValue( From f67df0d5f598fdf1e6813303f7ac9768aab04e67 Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 18:05:11 -0700 Subject: [PATCH 27/29] Inline findSingleValueBearingPart in grammarValueTypeValidator and update comments --- .../actionGrammar/src/grammarValueDeriver.ts | 13 +++++-------- .../actionGrammar/src/grammarValueTypeValidator.ts | 14 +++++--------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index 3462929296..99ae8e6c3c 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -63,17 +63,14 @@ export function deriveEffectiveValue( if (!requireValue) { return undefined; } - const termsDescription = - rule.parts.length === 1 - ? "has 1 term" - : `has ${rule.parts.length} terms`; const description = describeRule?.() ?? ""; + const term = rule.parts.length === 1 ? "term" : "terms"; const reason = result === "ambiguous" - ? "and more than one part carries a variable - the implicit value is ambiguous" - : "and no part carries a variable"; + ? "more than one part carries a variable - the implicit value is ambiguous" + : "no part carries a variable"; throw new Error( - `${description} ${termsDescription} but no value expression, ${reason}. ` + - `Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.`, + `Internal Error : ${description} has ${rule.parts.length} ${term} but no value expression, and ${reason}. ` + + "Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.", ); } diff --git a/ts/packages/actionGrammar/src/grammarValueTypeValidator.ts b/ts/packages/actionGrammar/src/grammarValueTypeValidator.ts index 9236eacbfd..bd7eb84071 100644 --- a/ts/packages/actionGrammar/src/grammarValueTypeValidator.ts +++ b/ts/packages/actionGrammar/src/grammarValueTypeValidator.ts @@ -16,7 +16,6 @@ import type { } from "@typeagent/action-schema"; import { SchemaCreator } from "@typeagent/action-schema"; import { getDispatchEffectiveMembers } from "./dispatchHelpers.js"; -import { findSingleValueBearingPart } from "./grammarValueDeriver.js"; // Sentinel for "any" — can't determine type const ANY_TYPE: SchemaType = SchemaCreator.any(); @@ -499,15 +498,12 @@ export function classifyRuleValue(rule: GrammarRule): RuleValueKind { if (rule.value !== undefined) { return { kind: "explicit" }; } - const contributor = findSingleValueBearingPart(rule.parts); - if (contributor !== undefined && contributor !== "ambiguous") { - return { - kind: "variable", - variableName: contributor.variable, - part: contributor.part, - }; + const variableParts = rule.parts.filter((p) => p.variable !== undefined); + if (variableParts.length === 1) { + const part = variableParts[0]; + return { kind: "variable", variableName: part.variable!, part }; } - if (contributor === undefined && rule.parts.length === 1) { + if (variableParts.length === 0 && rule.parts.length === 1) { const part = rule.parts[0]; if (part.type === "rules") { // For dispatched parts, the effective member list spans From d396278d3af3e36e7db7c91a2942f8e09b117dd4 Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 18:34:54 -0700 Subject: [PATCH 28/29] Remove findSingleValueBearingPart and inline the logic --- .../actionGrammar/src/grammarOptimizer.ts | 23 ++++---- .../actionGrammar/src/grammarValueDeriver.ts | 55 ++++-------------- .../test/grammarValueDeriver.spec.ts | 56 +------------------ .../test/nfaFactoredPrefixValues.spec.ts | 4 +- 4 files changed, 30 insertions(+), 108 deletions(-) diff --git a/ts/packages/actionGrammar/src/grammarOptimizer.ts b/ts/packages/actionGrammar/src/grammarOptimizer.ts index debcddee88..ea539c3ec3 100644 --- a/ts/packages/actionGrammar/src/grammarOptimizer.ts +++ b/ts/packages/actionGrammar/src/grammarOptimizer.ts @@ -22,10 +22,7 @@ import { import { leadingWordBoundaryScriptPrefix } from "./spacingScripts.js"; import { leadingNonSeparatorRun } from "./grammarMatcher.js"; import { getDispatchEffectiveMembers } from "./dispatchHelpers.js"; -import { - deriveEffectiveValue, - findSingleValueBearingPart, -} from "./grammarValueDeriver.js"; +import { deriveEffectiveValue } from "./grammarValueDeriver.js"; import { globalPhraseSetRegistry, PhraseSetMatcher, @@ -3191,12 +3188,18 @@ function checkForwardingPromotable( ): boolean { if (parts.length <= 1) return true; if (last.variable === undefined) return false; - // Multi-part rule: matcher's implicit-default rule requires exactly - // one variable-bearing contributor. `last` already carries a - // variable, so the shared scan can only return "ambiguous" (2+ - // contributors) or `last` itself - bail out on "ambiguous" so - // baseline missing/multiple-default throws stay observable. - return findSingleValueBearingPart(parts) !== "ambiguous"; + // Multi-part rule: matcher's implicit-default rule requires + // exactly one variable-bearing contributor (wildcard / number + // always; rules / string / phraseSet only when bound; every + // `GrammarPart` carries an optional `variable` field, so a + // single `p.variable !== undefined` test covers the union). + // Promoting masks the baseline missing/multiple-default throws + // at finalize time, so bail out unless the trailing RulesPart + // is the sole contributor. + for (let i = 0; i < parts.length - 1; i++) { + if (parts[i].variable !== undefined) return false; + } + return true; } /** diff --git a/ts/packages/actionGrammar/src/grammarValueDeriver.ts b/ts/packages/actionGrammar/src/grammarValueDeriver.ts index 99ae8e6c3c..7e7d336b30 100644 --- a/ts/packages/actionGrammar/src/grammarValueDeriver.ts +++ b/ts/packages/actionGrammar/src/grammarValueDeriver.ts @@ -6,47 +6,16 @@ * nfaCompiler.ts: computing a rule's effective value expression. */ -import type { - CompiledValueNode, - GrammarPart, - GrammarRule, -} from "./grammarTypes.js"; - -/** - * A part that solely carries a rule's implicit value: the variable name - * it binds, and the `GrammarPart` itself (so callers that need more than - * the name - e.g. its type - don't have to re-scan `parts`). - */ -export type ValueBearingPart = { variable: string; part: GrammarPart }; - -/** - * Find the single part (if any) that carries a variable across a rule's - * parts, so its value can stand in for the whole rule's implicit value - * when the rule has no explicit `->` expression. Returns `"ambiguous"` - * when 2+ parts carry a variable, or `undefined` when none do. - */ -export function findSingleValueBearingPart( - parts: GrammarPart[], -): ValueBearingPart | "ambiguous" | undefined { - let found: ValueBearingPart | undefined; - for (const p of parts) { - const name = p.variable; - if (name === undefined) continue; - if (found !== undefined) return "ambiguous"; - found = { variable: name, part: p }; - } - return found; -} +import type { CompiledValueNode, GrammarRule } from "./grammarTypes.js"; /** * A rule's explicit `->` value when present, otherwise the implicit - * value from a single variable-bearing part (via - * `findSingleValueBearingPart`). Throws if a value is required and - * missing/ambiguous. Not every rule needs a value - only when - * `requireValue` is set (top-level action rules, or nested rules - * captured by a parent variable) do ambiguous/missing values throw; - * otherwise both resolve to `undefined`. `describeRule` is only - * invoked when an error is thrown. + * value from a single variable-bearing part across the rule's `parts`. + * Throws if a value is required and missing/ambiguous. Not every rule + * needs a value - only when `requireValue` is set (top-level action + * rules, or nested rules captured by a parent variable) do + * ambiguous/missing values throw; otherwise both resolve to + * `undefined`. `describeRule` is only invoked when an error is thrown. */ export function deriveEffectiveValue( rule: GrammarRule, @@ -56,9 +25,9 @@ export function deriveEffectiveValue( if (rule.value !== undefined) { return rule.value; } - const result = findSingleValueBearingPart(rule.parts); - if (result !== undefined && result !== "ambiguous") { - return { type: "variable", name: result.variable }; + const variableParts = rule.parts.filter((p) => p.variable !== undefined); + if (variableParts.length === 1) { + return { type: "variable", name: variableParts[0].variable! }; } if (!requireValue) { return undefined; @@ -66,11 +35,11 @@ export function deriveEffectiveValue( const description = describeRule?.() ?? ""; const term = rule.parts.length === 1 ? "term" : "terms"; const reason = - result === "ambiguous" + variableParts.length > 1 ? "more than one part carries a variable - the implicit value is ambiguous" : "no part carries a variable"; throw new Error( - `Internal Error : ${description} has ${rule.parts.length} ${term} but no value expression, and ${reason}. ` + + `Internal error: ${description} has ${rule.parts.length} ${term} but no value expression, and ${reason}. ` + "Rules must have an explicit value expression (using ->) unless exactly one part carries a variable.", ); } diff --git a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts index 2ecf750c4e..ea037ec9b6 100644 --- a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts +++ b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts @@ -1,19 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// Direct unit tests for the pure helpers in grammarValueDeriver.ts, shared -// by grammarOptimizer.ts (`checkForwardingPromotable`) and nfaCompiler.ts -// (via `deriveEffectiveValue`, which lives in this module). +// Direct unit tests for the pure `deriveEffectiveValue` helper in +// grammarValueDeriver.ts, used by grammarOptimizer.ts and nfaCompiler.ts. // The consumers' own spec files (grammarOptimizerPromoteTail.spec.ts, // nfaFactoredPrefixValues.spec.ts) already cover end-to-end wiring and // promote decisions; nfaCompilerValueErrors.spec.ts covers // `deriveEffectiveValue`'s throwing (requireValue: true) paths. These // tests isolate its non-throwing (requireValue: false) derivation logic. -import { - findSingleValueBearingPart, - deriveEffectiveValue, -} from "../src/grammarValueDeriver.js"; +import { deriveEffectiveValue } from "../src/grammarValueDeriver.js"; import { createStringPart, createWildcardPart, @@ -21,52 +17,6 @@ import { GrammarRule, } from "../src/grammarTypes.js"; -describe("findSingleValueBearingPart", () => { - it("returns undefined for an empty parts array", () => { - expect(findSingleValueBearingPart([])).toBeUndefined(); - }); - - it("returns undefined when no part carries a variable", () => { - const parts = [createStringPart(["foo"]), createStringPart(["bar"])]; - expect(findSingleValueBearingPart(parts)).toBeUndefined(); - }); - - it("returns the variable and part of the single variable-bearing part", () => { - const wildcardPart = createWildcardPart("x", "wildcard"); - const parts = [createStringPart(["foo"]), wildcardPart]; - expect(findSingleValueBearingPart(parts)).toEqual({ - variable: "x", - part: wildcardPart, - }); - }); - - it("finds the variable-bearing part regardless of position", () => { - const wildcardPart = createWildcardPart("x", "wildcard"); - const parts = [wildcardPart, createStringPart(["foo"])]; - expect(findSingleValueBearingPart(parts)).toEqual({ - variable: "x", - part: wildcardPart, - }); - }); - - it('returns "ambiguous" when 2+ parts carry a variable', () => { - const parts = [ - createStringPart(["foo"], "x"), - createStringPart(["bar"], "y"), - ]; - expect(findSingleValueBearingPart(parts)).toBe("ambiguous"); - }); - - it('returns "ambiguous" even when 3+ parts carry a variable', () => { - const parts = [ - createStringPart(["foo"], "x"), - createStringPart(["bar"], "y"), - createStringPart(["baz"], "z"), - ]; - expect(findSingleValueBearingPart(parts)).toBe("ambiguous"); - }); -}); - describe("deriveEffectiveValue (requireValue: false)", () => { it("returns the rule's explicit value when present, ignoring parts", () => { const rule: GrammarRule = { diff --git a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts index e5bb513a7c..52892410d7 100644 --- a/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts +++ b/ts/packages/actionGrammar/test/nfaFactoredPrefixValues.spec.ts @@ -11,7 +11,7 @@ // explicit `value` on the factored top-level rule, so grammar-source // tests below don't reach the new implicit-derivation code path. The // hand-built-AST test further down is what directly exercises -// `findSingleValueBearingPart`'s forwarding logic. +// `deriveEffectiveValue`'s forwarding logic. // // NFA's rejection of tailCall RulesParts (still unsupported) is // already covered by grammarOptimizerTailFactoring.spec.ts ("NFA @@ -72,7 +72,7 @@ describe("NFA compilation of factored shared-prefix grammars", () => { // tests above don't exercise this path because // `factorCommonPrefixes` and explicit `->` expressions both // already stamp `rule.value`, so this is the only test that - // directly reaches `findSingleValueBearingPart`'s forwarding case. + // directly reaches `deriveEffectiveValue`'s forwarding case. const prefixPart = createRulesPart( [{ parts: [createStringPart(["please"])] }], { optional: true }, From 861cf817c0c09cee7f3cbd3f0a6d648e7b2862f0 Mon Sep 17 00:00:00 2001 From: George Ng Date: Mon, 13 Jul 2026 18:41:14 -0700 Subject: [PATCH 29/29] Update test coverage and documentation --- ts/docs/architecture/core/actionGrammar.md | 4 ++-- .../test/grammarValueDeriver.spec.ts | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/ts/docs/architecture/core/actionGrammar.md b/ts/docs/architecture/core/actionGrammar.md index d8f0e19a71..84cb783ab3 100644 --- a/ts/docs/architecture/core/actionGrammar.md +++ b/ts/docs/architecture/core/actionGrammar.md @@ -1522,8 +1522,8 @@ explicit value expression produce no value — the compiler warns about this because the output is ambiguous. The NFA compiler (`nfaCompiler.ts`) mirrors this same "no value unless -actually needed" principle via the shared `deriveValue` -(`grammarValueDeriver.ts`): `deriveEffectiveValue` forwards a rule's +actually needed" principle via the shared `deriveEffectiveValue` +(`grammarValueDeriver.ts`), which forwards a rule's single variable-bearing part (whatever its shape - a single-part rule, or a multi-part rule reshaped by `factorCommonPrefixes`), and only treats an ambiguous or missing implicit value as a hard error when the diff --git a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts index ea037ec9b6..a40eddc2c6 100644 --- a/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts +++ b/ts/packages/actionGrammar/test/grammarValueDeriver.spec.ts @@ -55,6 +55,19 @@ describe("deriveEffectiveValue (requireValue: false)", () => { }); }); + it("forwards the variable-bearing part's value regardless of its position", () => { + const rule: GrammarRule = { + parts: [ + createWildcardPart("x", "wildcard"), + createStringPart(["foo"]), + ], + }; + expect(deriveEffectiveValue(rule)).toEqual({ + type: "variable", + name: "x", + }); + }); + it("forwards a nested RulesPart's captured variable", () => { const nested = createRulesPart( [{ parts: [createStringPart(["foo"])] }], @@ -81,6 +94,17 @@ describe("deriveEffectiveValue (requireValue: false)", () => { expect(deriveEffectiveValue(rule)).toBeUndefined(); }); + it("returns undefined when 3+ parts carry a variable and there is no explicit value", () => { + const rule: GrammarRule = { + parts: [ + createStringPart(["foo"], "x"), + createStringPart(["bar"], "y"), + createStringPart(["baz"], "z"), + ], + }; + expect(deriveEffectiveValue(rule)).toBeUndefined(); + }); + it("returns undefined for a multi-part rule where nothing carries a variable", () => { const rule: GrammarRule = { parts: [createStringPart(["foo"]), createStringPart(["bar"])],