evaluate.py

import os
import sys
import time
import gc
from pathlib import Path

# Add project root to python path
sys.path.insert(0, str(Path(__file__).resolve().parent))

from app.orchestrator.pipeline import PipelineOrchestrator
from app.config.settings import settings
from app.utils.logger import get_logger

logger = get_logger(__name__)

# List of 15 lightweight to medium Python repositories for actual E2E evaluation
EVAL_REPOS = [
    "https://github.com/psf/requests",
    "https://github.com/pallets/flask",
    "https://github.com/pallets/click",
    "https://github.com/urllib3/urllib3",
    "https://github.com/encode/httpx",
    "https://github.com/tiangolo/fastapi",
    "https://github.com/pallets/jinja",
    "https://github.com/pydantic/pydantic",
    "https://github.com/certifi/python-certifi",
    "https://github.com/pallets/werkzeug",
    "https://github.com/pytest-dev/pytest",
    "https://github.com/tox-dev/tox",
    "https://github.com/celery/celery",
    "https://github.com/python-attrs/attrs",
    "https://github.com/bottlepy/bottle"
]

def get_memory_usage() -> float:
    """Gets memory usage of the current process in MB."""
    try:
        import psutil
        process = psutil.Process(os.getpid())
        return process.memory_info().rss / (1024 * 1024)
    except ImportError:
        return 0.0

def main():
    logger.info("Starting Sentinel AI Lite Pipeline Live Evaluation...")
    orchestrator = PipelineOrchestrator()
    
    results = []
    settings.REPORTS_DIR.mkdir(parents=True, exist_ok=True)
    
    for idx, repo_url in enumerate(EVAL_REPOS, 1):
        logger.info(f"[{idx}/{len(EVAL_REPOS)}] Running live audit on: {repo_url}")
        
        # Trigger GC to get a clean memory reading
        gc.collect()
        start_mem = get_memory_usage()
        start_time = time.time()
        
        try:
            # Run the actual pipeline
            report = orchestrator.run_pipeline(repo_url)
            runtime = time.time() - start_time
            end_mem = get_memory_usage()
            mem_used = max(0.0, end_mem - start_mem)
            
            # Count metrics
            vuln_count = len(report.vulnerabilities.vulnerabilities)
            reachable_count = sum(1 for a in report.risk_assessment if a.reachability_verdict == "Reachable")
            possibly_reachable = sum(1 for a in report.risk_assessment if a.reachability_verdict == "Possibly Reachable")
            
            # Estimate LLM latency and cost
            # Input tokens are limited; Gemini API cost is ~$0.000075 per call (approx 1k tokens input/output)
            api_cost = len(report.risk_assessment) * 0.000075
            
            results.append({
                "repo_url": repo_url,
                "status": report.status,
                "runtime_sec": round(runtime, 2),
                "mem_mb": round(mem_used, 1),
                "vulns_detected": vuln_count,
                "reachable": reachable_count,
                "possibly_reachable": possibly_reachable,
                "api_cost_usd": round(api_cost, 6),
                "errors": len(report.errors)
            })
            
            logger.info(f"Finished {repo_url} in {runtime:.1f}s. Vulns: {vuln_count}, Reachable: {reachable_count}")
            
        except Exception as e:
            logger.error(f"Failed to audit {repo_url}: {e}", exc_info=True)
            results.append({
                "repo_url": repo_url,
                "status": "FAILED",
                "runtime_sec": round(time.time() - start_time, 2),
                "mem_mb": 0.0,
                "vulns_detected": 0,
                "reachable": 0,
                "possibly_reachable": 0,
                "api_cost_usd": 0.0,
                "errors": 1
            })

    # Compile the real markdown output
    eval_path = settings.REPORTS_DIR / "evaluation_results.md"
    logger.info(f"Writing actual evaluation report to {eval_path}")
    
    with open(eval_path, "w", encoding="utf-8") as f:
        f.write("# Sentinel AI Lite - System Evaluation Report\n\n")
        f.write("This report presents the actual runtime performance and vulnerability detection metrics collected across 15 real Python repositories.\n\n")
        
        f.write("## Evaluation Summary Table\n\n")
        f.write("| # | Repository | Status | Runtime (s) | Memory (MB) | Vulns Detected | Reachable | Possibly Reachable | API Cost ($) | Errors |\n")
        f.write("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n")
        
        total_runtime = 0.0
        total_mem = 0.0
        total_vulns = 0
        total_reachable = 0
        total_possibly = 0
        total_cost = 0.0
        total_errors = 0
        success_count = 0
        
        for idx, res in enumerate(results, 1):
            repo_name = res['repo_url'].replace('https://github.com/', '')
            f.write(
                f"| {idx} | `{repo_name}` | **{res['status']}** | {res['runtime_sec']} | {res['mem_mb']} | "
                f"{res['vulns_detected']} | {res['reachable']} | {res['possibly_reachable']} | ${res['api_cost_usd']:.6f} | {res['errors']} |\n"
            )
            
            if res['status'] in ["COMPLETED", "PARTIAL_COMPLETED"]:
                total_runtime += res['runtime_sec']
                total_mem += res['mem_mb']
                total_vulns += res['vulns_detected']
                total_reachable += res['reachable']
                total_possibly += res['possibly_reachable']
                total_cost += res['api_cost_usd']
                total_errors += res['errors']
                success_count += 1
                
        # Write Averages/Totals
        if success_count > 0:
            avg_runtime = total_runtime / success_count
            avg_mem = total_mem / success_count
            
            f.write(
                f"| | **Average / Total** | | **{avg_runtime:.2f}s** | **{avg_mem:.1f}MB** | "
                f"**{total_vulns}** | **{total_reachable}** | **{total_possibly}** | **${total_cost:.6f}** | **{total_errors}** |\n\n"
            )
        
        f.write("## Metrics Definitions\n\n")
        f.write("- **Runtime**: Total execution duration of the scanner, including Git operations, parser traversals, OSV lookups, and reachability computations.\n")
        f.write("- **Memory**: RAM usage overhead during pipeline execution.\n")
        f.write("- **Vulns Detected**: Total number of unique package vulnerabilities queried from OSV.dev.\n")
        f.write("- **Reachable**: Call chains linking entry points directly to vulnerable modules.\n")
        
        f.write("\n## Evaluation Run Context\n\n")
        f.write(f"- **Scan Date**: {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime())}\n")
        f.write(f"- **Total Audited Repositories**: {len(EVAL_REPOS)}\n")

    logger.info("Evaluation completed successfully.")

if __name__ == "__main__":
    main()