diff --git a/packages/nemo_evaluator_sdk/examples/gym/README.md b/packages/nemo_evaluator_sdk/examples/gym/README.md index 0189e03eb3..b2b4c0c74d 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/README.md +++ b/packages/nemo_evaluator_sdk/examples/gym/README.md @@ -42,7 +42,7 @@ Working from a Gym checkout also makes its components take precedence over the p Useful flags: `--resources-server`, `--agent`, `--model-type` (`inference_provider` for OpenAI-compatible **chat** endpoints; `openai_model` uses the OpenAI **Responses API** and 500s against chat-only endpoints), `--num-repeats`, `--dataset`, `--output-dir`. -For the full set of knobs the underlying `gym env start` / `gym eval run` commands accept, see the [NeMo Gym documentation](https://github.com/NVIDIA-NeMo/Gym). Anything `GymRuntimeConfig` does not expose as a field can be passed through with its `env_overrides` escape hatch — nested data such as `{"model": {"temperature": 0.7}}`, flattened to Hydra's override grammar and applied to `gym env start`. +For the full set of knobs the underlying `gym env start` / `gym eval run` commands accept, see the [NeMo Gym documentation](https://github.com/NVIDIA-NeMo/Gym). Anything `GymRuntimeConfig` does not expose as a field can be passed through with its `hydra_params` escape hatch — nested data such as `{"model": {"temperature": 0.7}}`, flattened to Hydra's override grammar and applied to `gym env start`. Each run writes its bundle to a fresh temporary directory by default. Pass `--output-dir` to choose one, but give every run its own: the runner refuses to reuse a directory that already holds Gym rollout output (Gym appends to its failures sidecar, so reusing one would mix runs) and raises rather than clearing a prior run's results. diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/config.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/config.py index 21a3dabecd..200cd41884 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/config.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/config.py @@ -128,10 +128,10 @@ def _flatten_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> list[ return arguments -def _redact_env_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> dict[str, Any]: +def _redact_hydra_params(overrides: Mapping[str, Any], _prefix: str = "") -> dict[str, Any]: """Redact credential-looking values from overrides before they are recorded as provenance. - ``env_overrides`` is a free-form escape hatch forwarded to Gym, so nothing stops a caller passing + ``hydra_params`` is a free-form escape hatch forwarded to Gym, so nothing stops a caller passing ``{"model": {"api_key": "sk-..."}}``. ``RunnerInfo.config`` is persisted into the run bundle, so a value that looks like a credential must not be written there. @@ -147,7 +147,7 @@ def _redact_env_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> di for key, value in overrides.items(): path = f"{_prefix}{key}" if isinstance(value, Mapping): - redacted[key] = _redact_env_overrides(value, f"{path}.") + redacted[key] = _redact_hydra_params(value, f"{path}.") elif any(marker in path.casefold() for marker in _SECRET_KEY_MARKERS): redacted[key] = _REDACTED elif isinstance(value, (list, tuple)): @@ -158,9 +158,9 @@ def _redact_env_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> di def _redact_list_item(item: Any, path: str) -> Any: - """Redact inside one element of a list-valued override. See :func:`_redact_env_overrides`.""" + """Redact inside one element of a list-valued override. See :func:`_redact_hydra_params`.""" if isinstance(item, Mapping): - return _redact_env_overrides(item, f"{path}.") + return _redact_hydra_params(item, f"{path}.") if isinstance(item, (list, tuple)): return [_redact_list_item(nested, path) for nested in item] return item @@ -186,11 +186,11 @@ def _selection_args(config: GymRuntimeConfig, work_dir: Path) -> list[str]: # env we're running. Assumes the agent config's top-level key equals the agent name (the # simple_agent convention) *and* that the resources-server is registered under the # environment's own name — not universally true, so self-contained or differently-named - # servers set bind_resources_server=False and bind themselves via env_overrides. + # servers set bind_resources_server=False and bind themselves via hydra_params. selection.append( f"+{config.agent}.responses_api_agents.{config.agent}.resources_server.name={config.resources_server}" ) - selection.extend(_flatten_overrides(config.env_overrides)) + selection.extend(_flatten_overrides(config.hydra_params)) # Gym is a Hydra app, so each invocation writes a timestamped run directory — by default # `outputs//