fix(eval): build the vector index so semantic benchmarks can report a number; stop tests writing to $HOME

code_review_graph/cli.py

@@ -840,6 +840,28 @@ def main() -> None:
     eval_cmd.add_argument("--all", action="store_true", dest="run_all", help="Run all benchmarks")
     eval_cmd.add_argument("--report", action="store_true", help="Generate report from results")
     eval_cmd.add_argument("--output-dir", default=None, help="Output directory for results")
+    eval_cmd.add_argument(
+        "--embed",
+        action="store_true",
+        help=(
+            "Build the vector index after each graph build. Required by the "
+            "agent_baseline, search_quality and multi_hop_retrieval "
+            "benchmarks: without it their natural-language questions hit "
+            "FTS5 only and return zero results (default: disabled)"
+        ),
+    )
+    eval_cmd.add_argument(
+        "--embed-provider",
+        choices=["local", "openai", "google", "minimax"],
+        default=None,
+        help="Provider for --embed (default: local, needs "
+             "code-review-graph[embeddings])",
+    )
+    eval_cmd.add_argument(
+        "--embed-model",
+        default=None,
+        help="Model for --embed (default: the provider's own default)",
+    )
 
     # detect-changes
     detect_cmd = sub.add_parser(
@@ -1262,6 +1284,9 @@ def main() -> None:
                 repos=repos,
                 benchmarks=benchmarks,
                 output_dir=getattr(args, "output_dir", None),
+                embed=getattr(args, "embed", False),
+                embedding_provider=getattr(args, "embed_provider", None),
+                embedding_model=getattr(args, "embed_model", None),
             )
             print(f"\nCompleted {len(results)} benchmark(s).")
             print("Run 'code-review-graph eval --report' to generate tables.")

code_review_graph/constants.py

@@ -4,6 +4,7 @@
 
 import math
 import os
+from pathlib import Path
 
 
 def _bounded_float_env(
@@ -71,3 +72,28 @@ def _bounded_float_env(
 IMPACT_SCORE_FLOOR = _bounded_float_env(
     "CRG_IMPACT_SCORE_FLOOR", 0.05, lower=0.0, upper=1.0,
 )
+
+
+#: Overrides the per-user state directory that holds ``registry.json``,
+#: ``watch.toml``, ``daemon.pid``, ``daemon-state.json`` and ``logs/``.
+#: Follows the same convention as CRG_DATA_DIR.
+CRG_HOME_ENV = "CRG_HOME"
+
+_DEFAULT_CRG_HOME = Path.home() / ".code-review-graph"
+
+
+def crg_home() -> Path:
+    """Return the per-user state directory for code-review-graph.
+
+    ``$CRG_HOME`` wins when set and non-empty; otherwise
+    ``~/.code-review-graph``.
+
+    Resolved per call rather than captured in a module-level constant. An
+    import-time constant cannot be redirected afterwards, which is what let
+    the test suite write into the real home directory of whoever ran it: by
+    the time a fixture set the variable, the value had already been frozen.
+    """
+    override = os.environ.get(CRG_HOME_ENV, "").strip()
+    if override:
+        return Path(override).expanduser()
+    return _DEFAULT_CRG_HOME

code_review_graph/daemon.py

@@ -32,15 +32,71 @@
     except ImportError:
         tomllib = None  # type: ignore[assignment]
 
+from .constants import crg_home
+
 logger = logging.getLogger(__name__)
 
 # ---------------------------------------------------------------------------
 # Config file location
 # ---------------------------------------------------------------------------
 
-CONFIG_PATH: Path = Path.home() / ".code-review-graph" / "watch.toml"
-PID_PATH: Path = Path.home() / ".code-review-graph" / "daemon.pid"
-STATE_PATH: Path = Path.home() / ".code-review-graph" / "daemon-state.json"
+def default_config_path() -> Path:
+    """Path to ``watch.toml`` under the per-user state directory."""
+    return crg_home() / "watch.toml"
+
+
+def default_pid_path() -> Path:
+    """Path to the daemon PID file."""
+    return crg_home() / "daemon.pid"
+
+
+def default_state_path() -> Path:
+    """Path to the persisted daemon state."""
+    return crg_home() / "daemon-state.json"
+
+
+def default_log_dir() -> Path:
+    """Directory for per-repo daemon logs."""
+    return crg_home() / "logs"
+
+
+# These four were module-level constants built from Path.home(). They resolve
+# per call now so $CRG_HOME can redirect them: an import-time constant is
+# frozen before any caller — a test fixture, a sandboxed run — gets the chance
+# to set the variable, which is how the test suite ended up writing into the
+# real home directory of whoever ran it.
+#
+# The PEP 562 shim below keeps the old attribute names working for anything
+# that already imported them: both ``daemon.CONFIG_PATH`` and
+# ``from …daemon import CONFIG_PATH`` route through ``__getattr__``, and
+# ``__dir__`` keeps them visible to introspection. Not covered: ``import *``
+# (this module defines no ``__all__``, and adding one would change what the
+# star exports for every other name) and static analysers, which cannot see
+# dynamic attributes. Both are acceptable — these were never public API, and
+# the alternative is deleting the names outright.
+_LAZY_PATHS = {
+    "CONFIG_PATH": default_config_path,
+    "PID_PATH": default_pid_path,
+    "STATE_PATH": default_state_path,
+}
+
+
+def __getattr__(name: str) -> Path:
+    if name in _LAZY_PATHS:
+        return _LAZY_PATHS[name]()
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__() -> list[str]:
+    """Include the lazy names so ``dir()`` and tab-completion still find them.
+
+    ``__getattr__`` alone covers attribute access and ``from … import X``,
+    but names absent from module globals are otherwise invisible to
+    ``dir()``, ``from … import *`` and static analysers.
+    """
+    return sorted(set(globals()) | set(_LAZY_PATHS))
+
+
 _HEALTH_CHECK_INTERVAL = 30
 
 # ---------------------------------------------------------------------------
@@ -66,7 +122,7 @@ class DaemonConfig:
     session_name: str = "crg-watch"
     """Logical daemon name (used in log messages and status output)."""
 
-    log_dir: Path = field(default_factory=lambda: Path.home() / ".code-review-graph" / "logs")
+    log_dir: Path = field(default_factory=default_log_dir)
     """Directory for per-repo log files."""
 
     poll_interval: int = 2
@@ -85,7 +141,7 @@ def load_config(path: Path | None = None) -> DaemonConfig:
     """Load daemon configuration from a TOML file.
 
     Args:
-        path: Explicit config path.  Falls back to :data:`CONFIG_PATH`.
+        path: Explicit config path.  Falls back to :func:`default_config_path`.
 
     Returns:
         A fully-validated :class:`DaemonConfig`.
@@ -99,7 +155,7 @@ def load_config(path: Path | None = None) -> DaemonConfig:
             "Install it with:  pip install tomli"
         )
 
-    config_path = path or CONFIG_PATH
+    config_path = path or default_config_path()
 
     if not config_path.exists():
         logger.info("Config file not found at %s — using defaults", config_path)
@@ -225,9 +281,9 @@ def save_config(config: DaemonConfig, path: Path | None = None) -> None:
 
     Args:
         config: The daemon configuration to persist.
-        path:   Explicit config path.  Falls back to :data:`CONFIG_PATH`.
+        path:   Explicit config path.  Falls back to :func:`default_config_path`.
     """
-    config_path = path or CONFIG_PATH
+    config_path = path or default_config_path()
     config_path.parent.mkdir(parents=True, exist_ok=True)
     config_path.write_text(_serialize_toml(config), encoding="utf-8")
     logger.info("Config saved to %s", config_path)
@@ -248,7 +304,7 @@ def add_repo_to_config(
     Args:
         repo_path:   Path to the repository (will be resolved to absolute).
         alias:       Optional short name.  Derived from dirname if *None*.
-        config_path: Explicit config file path.  Falls back to :data:`CONFIG_PATH`.
+        config_path: Explicit config file path.  Falls back to :func:`default_config_path`.
 
     Returns:
         The updated :class:`DaemonConfig`.
@@ -294,7 +350,7 @@ def remove_repo_from_config(
 
     Args:
         path_or_alias: Either the absolute/relative repo path or its alias.
-        config_path:   Explicit config file path.  Falls back to :data:`CONFIG_PATH`.
+        config_path:   Explicit config file path.  Falls back to :func:`default_config_path`.
 
     Returns:
         The updated :class:`DaemonConfig`.
@@ -323,14 +379,14 @@ def remove_repo_from_config(
 
 def write_pid(pid: int | None = None, path: Path | None = None) -> None:
     """Write the current (or given) PID to the PID file."""
-    pid_path = path or PID_PATH
+    pid_path = path or default_pid_path()
     pid_path.parent.mkdir(parents=True, exist_ok=True)
     pid_path.write_text(str(pid or os.getpid()), encoding="utf-8")
 
 
 def read_pid(path: Path | None = None) -> int | None:
     """Read the daemon PID from disk. Returns None if missing/invalid."""
-    pid_path = path or PID_PATH
+    pid_path = path or default_pid_path()
     if not pid_path.exists():
         return None
     try:
@@ -341,7 +397,7 @@ def read_pid(path: Path | None = None) -> int | None:
 
 def clear_pid(path: Path | None = None) -> None:
     """Remove the PID file."""
-    pid_path = path or PID_PATH
+    pid_path = path or default_pid_path()
     try:
         pid_path.unlink(missing_ok=True)
     except OSError:
@@ -455,7 +511,7 @@ def load_state(path: Path | None = None) -> dict[str, Any]:
     Returns a dict mapping alias to ``{"pid": int, "path": str}``.
     Returns an empty dict if the file is missing or corrupt.
     """
-    state_path = path or STATE_PATH
+    state_path = path or default_state_path()
     if not state_path.exists():
         return {}
     try:
@@ -599,8 +655,8 @@ def __init__(
         config_path: Path | None = None,
     ) -> None:
         self._config: DaemonConfig = config or load_config(config_path)
-        self._config_path: Path = config_path or CONFIG_PATH
-        self._state_path: Path = STATE_PATH
+        self._config_path: Path = config_path or default_config_path()
+        self._state_path: Path = default_state_path()
         self._children: dict[str, subprocess.Popen[bytes]] = {}
         self._current_repos: dict[str, WatchRepo] = {}
         self._config_watcher: ConfigWatcher | None = None

code_review_graph/eval/benchmarks/agent_baseline.py

@@ -180,10 +180,20 @@ def aggregate(results: list[dict]) -> dict:
     """Aggregate over rows where both sides of the comparison exist."""
     ok = [r for r in results if r.get("status") == "ok"]
     ratios = [float(r["baseline_to_graph_ratio"]) for r in ok]
+    no_graph = sum(1 for r in results if r.get("status") == "no_graph_results")
     return {
         "total_rows": len(results),
         "ok_rows": len(ok),
         "error_rows": sum(1 for r in results if r.get("status") == "error"),
+        # Excluded rows are reported, not just dropped: a run where the graph
+        # answered nothing must not be readable as "no result" when it is
+        # really "every query failed". A high no_graph_results_rows count
+        # against a populated graph means the vector index is missing —
+        # re-run the eval with --embed.
+        "no_graph_results_rows": no_graph,
+        "no_baseline_match_rows": sum(
+            1 for r in results if r.get("status") == "no_baseline_match"
+        ),
         "median_baseline_to_graph_ratio": (
             round(statistics.median(ratios), 1) if ratios else None
         ),