feat(swebench-pro): initial support for ScaleAI/SWE-bench_Pro

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "SWE-bench_Pro-os"]
+	path = SWE-bench_Pro-os
+	url = https://github.com/scaleapi/SWE-bench_Pro-os

configs/agents/mini_swe_swebench_pro.yaml

@@ -0,0 +1,30 @@
+step_limit: 250
+cost_limit: 0.0
+cwd: "/app"
+timeout: 180
+environment_class: modal
+action_regex: '```mswea_bash_command\s*\n(.*?)\n```'
+system_template: |
+  You are a software engineer working in a Linux container.
+  The repository is checked out at /app and all dependencies are installed.
+  Fix the reported issue by modifying the source files.
+  Every response must contain exactly one bash command in this format:
+
+  ```mswea_bash_command
+  your_command_here
+  ```
+
+  When you are done and confident the issue is fixed, submit by running:
+  ```mswea_bash_command
+  echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT
+  ```
+instance_template: |
+  Please solve this task: {{task}}
+
+  The repository is at /app. Use bash commands to explore the code, edit files, and run tests.
+  When the issue is fixed, submit with:
+  ```mswea_bash_command
+  echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT
+  ```
+extra_model_kwargs:
+  drop_params: true

configs/labeling/laguna_xs2_swebench_pro_test_labeler.yaml

@@ -0,0 +1,12 @@
+# Label 16 test trajectories from Laguna-XS.2 on SWE-bench Pro
+# Usage:
+#   sbatch slurm/swebench_pro_label.sh --config configs/labeling/laguna_xs2_swebench_pro_test_labeler.yaml
+
+trajectory_dir: generations/swebench_pro/laguna_xs2_test
+output_dir: labels/swebench_pro/laguna_xs2_test
+scripts_dir: SWE-bench_Pro-os
+modal_app_name: program-probes-labeler-pro
+sandbox_timeout: 3600
+eval_timeout: 600
+resume: true
+n_workers: 16

configs/runs/laguna_xs2_swebench_pro_full.yaml

@@ -0,0 +1,44 @@
+# Full run: Laguna-XS.2 on all 731 SWE-bench Pro instances, 10 runs each, 4xA100 thin
+#
+# Submit as 5-shard array job:
+#   sbatch --array=0-4 slurm/swebench_pro_run_thin.sh \
+#     --run-config configs/runs/laguna_xs2_swebench_pro_full.yaml
+
+# --- Model / vLLM ---
+model_id: poolside/Laguna-XS.2
+served_model_name: laguna-xs2
+tensor_parallel_size: 4
+max_model_len: 262144
+dtype: bfloat16
+gpu_memory_utilization: 0.9
+vllm_port: 18000
+vllm_startup_timeout_s: 3600
+vllm_log_path: logs/vllm_laguna_xs2_swebench_pro_full.log
+vllm_extra_args:
+  - "--reasoning-parser"
+  - "poolside_v1"
+  - "--enable-auto-tool-choice"
+  - "--tool-call-parser"
+  - "poolside_v1"
+  - "--default-chat-template-kwargs"
+  - '{"enable_thinking": true}'
+
+# --- Generation (thinking mode, coding) ---
+temperature: 0.7
+top_p: 0.95
+top_k: 20
+
+# --- Agent ---
+step_limit: 100
+command_timeout: 180
+
+# --- Execution ---
+n_runs: 10               # 10 trajectories per instance (pass@10)
+n_instances: -1          # all instances in this shard
+n_workers: 20            # 20 parallel Modal containers per shard (5 shards × 20 = 100 total)
+modal_app_name: program-probes-swebench-pro
+modal_timeout: 1800      # 30 min per sandbox
+output_dir: generations/swebench_pro/laguna_xs2_full
+
+# --- SWE-bench Pro ---
+swe_bench_pro_scripts_dir: SWE-bench_Pro-os

configs/runs/laguna_xs2_swebench_pro_test.yaml

@@ -0,0 +1,44 @@
+# Test run: Laguna-XS.2 on 16 SWE-bench Pro instances, 4xA100 thin
+#
+# Submit with:
+#   sbatch --gpus=4 slurm/swebench_pro_run.sh \
+#     --run-config configs/runs/laguna_xs2_swebench_pro_test.yaml
+
+# --- Model / vLLM ---
+model_id: poolside/Laguna-XS.2
+served_model_name: laguna-xs2
+tensor_parallel_size: 4
+max_model_len: 262144
+dtype: bfloat16
+gpu_memory_utilization: 0.9
+vllm_port: 18000
+vllm_startup_timeout_s: 900
+vllm_log_path: logs/vllm_laguna_xs2_swebench_pro_test.log
+vllm_extra_args:
+  - "--reasoning-parser"
+  - "poolside_v1"
+  - "--enable-auto-tool-choice"
+  - "--tool-call-parser"
+  - "poolside_v1"
+  - "--default-chat-template-kwargs"
+  - '{"enable_thinking": true}'
+
+# --- Generation (thinking mode, coding) ---
+temperature: 0.7
+top_p: 0.95
+top_k: 20
+
+# --- Agent ---
+step_limit: 100
+command_timeout: 180     # seconds per bash command in Modal sandbox
+
+# --- Execution ---
+n_runs: 1                # one attempt per instance
+n_instances: 16          # stop after 16 unique instances
+n_workers: 16            # 16 parallel agent threads; vLLM batches their requests
+modal_app_name: program-probes-swebench-pro
+modal_timeout: 1800      # seconds for the full sandbox lifecycle
+output_dir: generations/swebench_pro/laguna_xs2_test
+
+# --- SWE-bench Pro ---
+swe_bench_pro_scripts_dir: SWE-bench_Pro-os   # local clone of github.com/scaleapi/SWE-bench_Pro-os

configs/runs/qwen36_27b_swebench_pro_full.yaml

@@ -0,0 +1,41 @@
+# Full run: Qwen3.6-27B on all 731 SWE-bench Pro instances, 8xA100 80GB
+#
+# Submit as 4-shard array job:
+#   sbatch --array=0-3 -t 02:00:00 slurm/swebench_pro_run.sh \
+#     --run-config configs/runs/qwen36_27b_swebench_pro_full.yaml
+
+# --- Model / vLLM ---
+model_id: Qwen/Qwen3.6-27B
+served_model_name: qwen36-27b
+tensor_parallel_size: 8
+max_model_len: 262144
+dtype: bfloat16
+gpu_memory_utilization: 0.9
+vllm_port: 18000
+vllm_startup_timeout_s: 900
+vllm_log_path: logs/vllm_qwen36_27b_swebench_pro_full.log
+vllm_extra_args:
+  - "--reasoning-parser"
+  - "qwen3"
+  - "--enable-auto-tool-choice"
+  - "--tool-call-parser"
+  - "qwen3_coder"
+
+# --- Generation ---
+temperature: 1.0
+top_p: 0.95
+
+# --- Agent ---
+step_limit: 100
+command_timeout: 180
+
+# --- Execution ---
+n_runs: 1
+n_instances: -1          # all instances in this shard
+n_workers: 8
+modal_app_name: program-probes-swebench-pro
+modal_timeout: 3600
+output_dir: generations/swebench_pro/qwen36_27b_full
+
+# --- SWE-bench Pro ---
+swe_bench_pro_scripts_dir: SWE-bench_Pro-os

configs/runs/qwen36_27b_swebench_pro_test.yaml

@@ -0,0 +1,41 @@
+# Test run: Qwen3.6-27B on 16 SWE-bench Pro instances, 8xA100 80GB
+#
+# Submit with:
+#   sbatch slurm/swebench_pro_run.sh --run-config configs/runs/qwen36_27b_swebench_pro_test.yaml
+
+# --- Model / vLLM ---
+model_id: Qwen/Qwen3.6-27B
+served_model_name: qwen36-27b
+tensor_parallel_size: 8
+max_model_len: 262144
+dtype: bfloat16
+gpu_memory_utilization: 0.9
+vllm_port: 18000
+vllm_startup_timeout_s: 900
+vllm_log_path: logs/vllm_qwen36_27b_swebench_pro.log
+vllm_extra_args:
+  - "--reasoning-parser"
+  - "qwen3"
+  - "--enable-auto-tool-choice"
+  - "--tool-call-parser"
+  - "qwen3_coder"
+
+# --- Generation ---
+temperature: 1.0
+top_p: 0.95
+# max_new_tokens: not set — no per-call cap, lets thinking models use the full remaining context
+
+# --- Agent ---
+step_limit: 100
+command_timeout: 180     # seconds per bash command in Modal sandbox
+
+# --- Execution ---
+n_runs: 1                # one attempt per instance
+n_instances: 16          # stop after 16 unique instances
+n_workers: 16            # 16 parallel agent threads; vLLM batches their requests
+modal_app_name: program-probes-swebench-pro
+modal_timeout: 1800      # seconds for the full sandbox lifecycle
+output_dir: generations/swebench_pro/qwen36_27b_test
+
+# --- SWE-bench Pro ---
+swe_bench_pro_scripts_dir: SWE-bench_Pro-os   # local clone of github.com/scaleapi/SWE-bench_Pro-os

configs/tasks/swe_bench_pro.yaml

@@ -0,0 +1,3 @@
+dataset: ScaleAI/SWE-bench_Pro
+adapter: swe_bench_pro
+execution_timeout: 300

run_labeler_pro.py

@@ -0,0 +1,106 @@
+"""Run the SWE-bench Pro labeler on generated agent trajectories.
+
+Replays each edit step of agent trajectories in Modal sandboxes, computing
+probe labels (compiles, test_results) at every intermediate code state using
+the Pro eval harness (entryscript.sh → output.json).
+
+Usage:
+    uv run python run_labeler_pro.py --config configs/labeling/laguna_xs2_swebench_pro_test_labeler.yaml
+
+The config YAML fields are documented in SwebenchProLabelerConfig (src/configs.py).
+"""
+
+import argparse
+import json
+import random
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+from src.configs import SwebenchProLabelerConfig, load_config
+from src.labeling.swebench_pro_labeler import label_pro_trajectory, load_pro_instance
+
+
+def _process(
+    tf: Path,
+    iid: str,
+    cfg: SwebenchProLabelerConfig,
+    out_dir: Path,
+    scripts_dir: Path,
+    jitter: float = 0.0,
+) -> str:
+    label_path = out_dir / f"{tf.stem}_labels.json"
+    if cfg.resume and label_path.exists():
+        return f"{tf.name}: skipped (labels exist)"
+    if jitter > 0:
+        time.sleep(random.uniform(0, jitter))
+    instance = load_pro_instance(iid)
+    label_pro_trajectory(
+        tf,
+        instance=instance,
+        scripts_dir=scripts_dir,
+        output_path=label_path,
+        modal_app_name=cfg.modal_app_name,
+        sandbox_timeout=cfg.sandbox_timeout,
+        eval_timeout=cfg.eval_timeout,
+    )
+    return f"{tf.name}: done"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Label SWE-bench Pro agent trajectories")
+    parser.add_argument(
+        "--config", required=True,
+        help="Path to SwebenchProLabelerConfig YAML (see configs/labeling/)",
+    )
+    args = parser.parse_args()
+
+    cfg: SwebenchProLabelerConfig = load_config(args.config, SwebenchProLabelerConfig)
+
+    traj_dir = Path(cfg.trajectory_dir)
+    out_dir = Path(cfg.output_dir)
+    scripts_dir = Path(cfg.scripts_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    traj_files = sorted(
+        f for f in traj_dir.glob("*.json")
+        if not f.name.startswith("index") and "_labels" not in f.name
+    )
+    if cfg.single:
+        traj_files = [f for f in traj_files if f.name == cfg.single]
+
+    # Extract instance IDs: strip _run<N> suffix if present
+    def _iid(tf: Path) -> str:
+        stem = tf.stem
+        # e.g. django__django-12345_run01 → django__django-12345
+        if "_run" in stem:
+            stem = stem[: stem.rfind("_run")]
+        return stem
+
+    if cfg.instances:
+        traj_files = [f for f in traj_files if _iid(f) in cfg.instances]
+
+    print(f"Labeling {len(traj_files)} trajectories from {traj_dir} → {out_dir}", flush=True)
+
+    jitter = 2.0 if cfg.n_workers > 1 else 0.0
+
+    if cfg.n_workers == 1:
+        for tf in traj_files:
+            iid = _iid(tf)
+            result = _process(tf, iid, cfg, out_dir, scripts_dir, jitter=0.0)
+            print(result, flush=True)
+    else:
+        with ThreadPoolExecutor(max_workers=cfg.n_workers) as pool:
+            futures = {
+                pool.submit(_process, tf, _iid(tf), cfg, out_dir, scripts_dir, jitter): tf.name
+                for tf in traj_files
+            }
+            for fut in as_completed(futures):
+                try:
+                    print(fut.result(), flush=True)
+                except Exception as exc:
+                    print(f"{futures[fut]}: ERROR — {exc}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

run_probe.py

@@ -1,7 +1,7 @@
 import argparse
 import json
 from src.configs import ModelConfig, load_config
-from src.probe import run_sweep, run_final, create_sweep
+from src.probe import run_sweep, run_final, run_eval, create_sweep
 
 
 def main():
@@ -50,6 +50,12 @@ def main():
     final_p.add_argument("--pos-weight", type=float, default=1.0,
                         help="Positive class weight multiplier (only used with --loss weighted_cross_entropy)")
 
+    eval_p = subparsers.add_parser("eval", help="Evaluate pre-trained weights on n_eval_bins bins (no retraining)")
+    eval_p.add_argument("--weights-run-id", default=None,
+                        help="Run ID whose weights.pt to load (defaults to --run-id)")
+    eval_p.add_argument("--output-run-id", required=True,
+                        help="Run ID under which to save results.pt")
+
     args = parser.parse_args()
     if args.n_eval_bins is not None and args.n_eval_bins != args.n_bins and args.n_bins != 1:
         parser.error("--n-eval-bins requires --n-bins 1 or --n-eval-bins equal to --n-bins")
@@ -81,6 +87,20 @@ def main():
             fixed_params=args.fixed,
             shuffle_labels=args.shuffle_labels,
         )
+    elif args.mode == "eval":
+        run_eval(
+            weights_run_id=args.weights_run_id or args.run_id,
+            output_run_id=args.output_run_id,
+            probe_name=args.probe,
+            probe_layers=model_cfg.probe_layers,
+            seed=args.seed,
+            cache_dir=args.cache_dir,
+            cache_run_id=cache_run_id,
+            results_dir=args.results_dir,
+            probe_arch=args.probe_arch,
+            n_eval_bins=args.n_eval_bins or 10,
+            eval_bin_axis=args.eval_bin_axis,
+        )
     else:
         run_final(
             run_id=args.run_id,