"""Run crawler tests with timing - stream output live."""
import subprocess, sys, time, re, shutil
from pathlib import Path
from datetime import datetime

LOGDIR = Path(__file__).parent / "timing_logs"
LOGDIR.mkdir(parents=True, exist_ok=True)

def run(label: str, args: list[str], timeout: int = 900) -> Path:
    log_path = LOGDIR / f"{label}.log"
    t0 = time.time()
    print(f"\n{'='*60}")
    print(f"[{datetime.now().strftime('%H:%M:%S')}] START: {label}")
    print(f"Command: py crawler.py {' '.join(args)}")
    print(f"{'='*60}", flush=True)

    all_stdout: list[str] = []
    all_stderr: list[str] = []

    proc = subprocess.Popen(
        [sys.executable, "crawler.py"] + args,
        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
        text=True, encoding="utf-8", errors="replace",
        cwd=str(Path(__file__).parent),
        bufsize=1,
    )

    import threading
    def read_stream(stream, lines_list, prefix=""):
        for line in iter(stream.readline, ""):
            lines_list.append(line)
            sys.stdout.write(prefix + line)

    stdout_thread = threading.Thread(target=read_stream, args=(proc.stdout, all_stdout))
    stderr_thread = threading.Thread(target=read_stream, args=(proc.stderr, all_stderr, "ERR> "))
    stdout_thread.start()
    stderr_thread.start()

    try:
        proc.wait(timeout=timeout)
    except subprocess.TimeoutExpired:
        proc.kill()
        elapsed = time.time() - t0
        msg = f"\nTIMEOUT after {timeout}s, wall={elapsed:.1f}s\n"
        sys.stdout.write(msg)
        all_stderr.append(msg)

    stdout_thread.join(timeout=5)
    stderr_thread.join(timeout=5)

    elapsed = time.time() - t0
    combined = "".join(all_stdout)

    # Extract metrics
    pages = re.findall(r"Done: (\d+)/(\d+) pages saved", combined)
    imgs = re.findall(r"image download summary.*?: (\d+)/(\d+) candidates saved", combined, re.I)
    rewritten = re.findall(r"Link rewriting complete: (\d+) rewritten, (\d+) unlinked across (\d+) files", combined)
    scores = re.findall(r"alignment=(\d)", combined)
    consent = re.findall(r"Consent verification returned status=(\S+)", combined)
    unchanged = re.findall(r"Content unchanged for (\S+); skipping", combined)

    summary = {
        "wall_seconds": round(elapsed, 1),
        "pages": pages[-1][1] if pages else "?",
        "pages_ok": pages[-1][0] if pages else "?",
        "images_saved": sum(int(m[0]) for m in imgs),
        "images_total": sum(int(m[1]) for m in imgs),
        "links_rewritten": int(rewritten[-1][0]) if rewritten else 0,
        "links_unlinked": int(rewritten[-1][1]) if rewritten else 0,
        "quality_scores": scores,
        "consent": consent[0] if consent else "?",
        "skipped_unchanged": len(unchanged),
    }

    with open(log_path, "w", encoding="utf-8") as f:
        f.write(f"TIMING REPORT: {label}\n")
        f.write(f"{'='*60}\n")
        for k, v in summary.items():
            f.write(f"  {k}: {v}\n")
        f.write(f"{'='*60}\n\nFULL OUTPUT:\n")
        f.write(combined)
        f.write("\n\nSTDERR:\n")
        f.write("".join(all_stderr))

    print(f"\n[DONE {label}] {elapsed:.1f}s | {summary['pages_ok']}/{summary['pages']} pages | {summary['images_saved']} imgs | skipped={summary['skipped_unchanged']}")
    sys.stdout.flush()
    return log_path


if __name__ == "__main__":
    SITES3 = [
        "https://www.tt.com",
        "https://www.heise.de/ct",
        "https://www.prolicht.at/en/inspiration/segment/office/",
    ]

    run("1_three_sites_forced", ["--fetch-urls"] + SITES3 + [
        "--markdown", "true", "--headless", "--fresh", "--verbose"
    ])

    run("2_three_sites_update", ["--fetch-urls"] + SITES3 + [
        "--markdown", "true", "--headless", "--keep-crawl", "true", "--verbose"
    ])

    prodir = Path(r"D:\repos\crawler\sites_crawled\www.prolicht.at")
    if prodir.exists():
        shutil.rmtree(str(prodir))

    run("3_prolicht_30_forced", [
        "--fetch-sitemap", "https://www.prolicht.at",
        "--limit", "30", "--markdown", "true", "--headless", "--fresh", "--verbose"
    ])

    run("4_prolicht_30_update", [
        "--fetch-sitemap", "https://www.prolicht.at",
        "--limit", "30", "--markdown", "true", "--headless", "--keep-crawl", "true", "--verbose"
    ])

    # Print overview
    print("\n" + "="*60)
    print("TIMING OVERVIEW")
    print("="*60)
    for log in sorted(LOGDIR.glob("*.log")):
        text = log.read_text(encoding="utf-8")
        for line in text.splitlines():
            line = line.strip()
            if line.startswith("wall_seconds") or line.startswith("pages") or line.startswith("images_") or line.startswith("skipped"):
                print(f"  {log.stem}: {line}")
    print("="*60)
