server.py

#!/usr/bin/env python3
"""
server.py

Local web server that wraps media_crawler.MediaCrawler in a controllable
background job, and serves a browser dashboard (static/index.html) to
start/stop crawls and watch live progress.

Run:
    python3 server.py
Then open:
    http://127.0.0.1:5000

Only one crawl runs at a time in this v1 -- starting a new one while one
is running returns an error. Downloaded files are saved under
./runs/<timestamp>/media/ and served back to the browser at /files/...
so the dashboard can show real thumbnails.
"""

import os
import threading
import time
from datetime import datetime

from flask import Flask, jsonify, request, send_from_directory, abort

from media_crawler import MediaCrawler, DEFAULT_USER_AGENT

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
RUNS_DIR = os.path.join(BASE_DIR, "runs")
STATIC_DIR = os.path.join(BASE_DIR, "static")
os.makedirs(RUNS_DIR, exist_ok=True)

app = Flask(__name__, static_folder=STATIC_DIR, static_url_path="/static")


class CrawlJobManager:
    """Holds at most one active/most-recent crawl job and exposes thread-safe
    control (start/stop) and read (status/manifest) operations for the API."""

    def __init__(self):
        self.lock = threading.Lock()
        self.crawler = None
        self.thread = None
        self.stop_event = None
        self.run_id = None

    def is_running(self):
        with self.lock:
            return self.crawler is not None and self.crawler.status == "running"

    def start(self, params):
        with self.lock:
            if self.crawler is not None and self.crawler.status == "running":
                raise RuntimeError("A crawl is already running. Stop it first.")

            run_id = datetime.now().strftime("%Y%m%d-%H%M%S")
            output_dir = os.path.join(RUNS_DIR, run_id)

            allowed_domains = "any" if params.get("no_domain_restriction") else params.get("allowed_domains")

            self.stop_event = threading.Event()
            self.crawler = MediaCrawler(
                seeds=params["seeds"],
                output_dir=output_dir,
                max_depth=params.get("max_depth", 2),
                max_pages=params.get("max_pages", 200),
                max_files=params.get("max_files") or None,
                allowed_domains=allowed_domains,
                max_domains=params.get("max_domains") or None,
                respect_robots=not params.get("ignore_robots", False),
                delay=params.get("delay", 0.5),
                timeout=params.get("timeout", 15),
                user_agent=params.get("user_agent") or DEFAULT_USER_AGENT,
                verbose=True,
                stop_event=self.stop_event,
            )
            self.run_id = run_id

            self.thread = threading.Thread(target=self.crawler.crawl, daemon=True)
            self.thread.start()
            return run_id

    def stop(self):
        with self.lock:
            if self.crawler is None or self.crawler.status != "running":
                raise RuntimeError("No crawl is currently running.")
            self.stop_event.set()

    def status(self):
        with self.lock:
            if self.crawler is None:
                return {"status": "idle"}
            s = self.crawler.get_status()
            s["run_id"] = self.run_id
            return s

    def manifest(self):
        with self.lock:
            if self.crawler is None:
                return []
            return self.crawler.get_manifest()

    def media_dir(self):
        with self.lock:
            if self.crawler is None:
                return None
            return self.crawler.output_dir


job = CrawlJobManager()


# ---------------------------------------------------------------------------
# API routes
# ---------------------------------------------------------------------------

@app.route("/")
def index():
    return send_from_directory(STATIC_DIR, "index.html")


@app.route("/api/start", methods=["POST"])
def api_start():
    body = request.get_json(force=True, silent=True) or {}

    raw_seeds = body.get("seeds", "")
    if isinstance(raw_seeds, str):
        seeds = [s.strip() for s in raw_seeds.replace(",", "\n").splitlines() if s.strip()]
    else:
        seeds = [s.strip() for s in raw_seeds if s.strip()]

    if not seeds:
        return jsonify({"error": "At least one seed URL is required."}), 400
    for s in seeds:
        if not (s.startswith("http://") or s.startswith("https://")):
            return jsonify({"error": f"Seed URL must start with http:// or https://: {s}"}), 400

    raw_domains = body.get("allowed_domains", "")
    if isinstance(raw_domains, str):
        allowed_domains = [d.strip() for d in raw_domains.replace(",", "\n").splitlines() if d.strip()]
    else:
        allowed_domains = [d.strip() for d in (raw_domains or []) if d.strip()]

    params = {
        "seeds": seeds,
        "max_depth": int(body.get("max_depth", 2)),
        "max_pages": int(body.get("max_pages", 200)),
        "max_files": int(body["max_files"]) if body.get("max_files") not in (None, "") else None,
        "allowed_domains": allowed_domains or None,
        "no_domain_restriction": bool(body.get("no_domain_restriction", False)),
        "max_domains": int(body["max_domains"]) if body.get("max_domains") not in (None, "") else None,
        "delay": float(body.get("delay", 0.5)),
        "ignore_robots": bool(body.get("ignore_robots", False)),
    }

    try:
        run_id = job.start(params)
    except RuntimeError as e:
        return jsonify({"error": str(e)}), 409
    except Exception as e:
        return jsonify({"error": f"Could not start crawl: {e}"}), 400

    return jsonify({"run_id": run_id})


@app.route("/api/stop", methods=["POST"])
def api_stop():
    try:
        job.stop()
    except RuntimeError as e:
        return jsonify({"error": str(e)}), 409
    return jsonify({"ok": True})


@app.route("/api/status")
def api_status():
    return jsonify(job.status())


@app.route("/api/manifest")
def api_manifest():
    return jsonify(job.manifest())


@app.route("/files/<path:subpath>")
def serve_downloaded_file(subpath):
    media_dir = job.media_dir()
    if media_dir is None:
        abort(404)
    # media_dir is an absolute path under RUNS_DIR; send_from_directory
    # guards against path traversal outside of it.
    return send_from_directory(media_dir, subpath)


if __name__ == "__main__":
    app.run(host="127.0.0.1", port=5000, debug=False, threaded=True)