#!/usr/bin/env python
"""Crawler REST API client: crawl pages and download markdown + images locally.

Usage:
  # Single page
  py crawler_client.py https://www.prolicht.at

  # Recursive documentation crawl
  py crawler_client.py https://www.jetbrains.com/help/youtrack/devportal/youtrack-rest-api.html --recursive --limit 30
"""

from __future__ import annotations

import argparse
import hashlib
import os
import re
import sys
import time
from pathlib import Path
from urllib.parse import urlparse

import requests

DEFAULT_HOST = "crawler.prototype.prolicht.digital"
DEFAULT_OUT = Path(__file__).resolve().parent / "crawled_pages"

IMG_MD_RE = re.compile(r"!\[[^\]]*\]\((https://[^)\s]+)(?:\s+\"[^\"]*\")?\s*\)")
IMG_SRC_RE = re.compile(r'src="(https://[^"]+/l/[^"]+)"')
FILENAME_RE = re.compile(r'filename="?([^"]+)"?')
CT_IMAGE_RE = re.compile(r"image/(\w+)")
PUBLIC_BASE = "https://crawler.prototype.prolicht.digital"


def _localize_url(url: str, base: str) -> str:
    if base and url.startswith(PUBLIC_BASE):
        return base.rstrip("/") + url[len(PUBLIC_BASE):]
    return url


def _strip_query_and_fragment(url: str) -> str:
    p = urlparse(url)
    return f"{p.scheme}://{p.hostname}{p.path}"


def _stem_from_url(url: str) -> str:
    path = urlparse(url).path.strip("/")
    if not path or path.lower() in {"index", "index.html", "index.htm"}:
        return "index"
    stem = path.rsplit("/", 1)[-1]
    if stem.lower().endswith(".html"):
        stem = stem[:-5]
    return stem.replace(":", "_").replace("?", "_").replace("&", "_") or "index"


def _download_binary(url: str, timeout: int = 30, base: str = "") -> tuple[bytes, str, str] | None:
    resp = requests.get(url, timeout=timeout, allow_redirects=False)
    while resp.status_code in (301, 302, 307, 308):
        location = resp.headers.get("Location", "")
        if base:
            location = _localize_url(location, base)
        resp = requests.get(location, timeout=timeout, allow_redirects=False)
    resp.raise_for_status()

    ct = (resp.headers.get("Content-Type") or "application/octet-stream").split(";")[0].strip().lower()

    if not ct.startswith("image/"):
        return None

    if not resp.content:
        return None

    cd = resp.headers.get("Content-Disposition", "")
    filename = "unknown"
    m = FILENAME_RE.search(cd)
    if m:
        filename = m.group(1)
    else:
        ext = ct.split("/")[-1]
        h = hashlib.md5(url.encode()).hexdigest()[:8]
        filename = f"img_{h}.{ext}"

    return resp.content, ct, filename


def _download_markdown(url: str, timeout: int = 30, base: str = "") -> str:
    resp = requests.get(url, timeout=timeout, allow_redirects=False)
    while resp.status_code in (301, 302, 307, 308):
        location = resp.headers.get("Location", "")
        if base:
            location = _localize_url(location, base)
        resp = requests.get(location, timeout=timeout, allow_redirects=False)
    resp.raise_for_status()
    return resp.text


def _api_post(base: str, path: str, payload: dict, timeout: int = 300, retries: int = 2) -> dict:
    url = f"{base.rstrip('/')}{path}"
    for attempt in range(retries):
        try:
            r = requests.post(url, json=payload, timeout=timeout)
            try:
                r.raise_for_status()
            except requests.exceptions.HTTPError:
                print("Server response:")
                print(r.text)
                raise
            return r.json()
        except requests.exceptions.ReadTimeout:
            if attempt < retries - 1:
                wait = 30 * (attempt + 1)
                print(f"  server timeout, retrying in {wait}s...")
                time.sleep(wait)
        except requests.exceptions.HTTPError as exc:
            code = exc.response.status_code if exc.response is not None else 0
            if code in (502, 504) and attempt < retries - 1:
                wait = 30 * (attempt + 1)
                print(f"  gateway timeout ({code}), retrying in {wait}s...")
                time.sleep(wait)
            else:
                raise
    raise RuntimeError(f"Failed after {retries} attempts: {url}")


# ---------------------------------------------------------------------------
def crawl_single(base: str, url: str, out_dir: Path, max_images: int = 5, timeout: int = 300, respect_robots: bool = True) -> tuple[bool, str]:
    result = _api_post(base, "/crawl/single", {
        "url": url, "markdown": True, "images": True, "max_images": max_images,
        "respect_robots": respect_robots,
    }, timeout=timeout)

    status = result.get("status", "error")
    domain = result.get("domain", "")
    markdown_url = result.get("markdown_url", "")
    file_stem = _stem_from_url(result.get("url", url))

    if status != "ok" or not markdown_url:
        print(f"  FAILED: status={status}")
        return False, file_stem

    domain_dir = out_dir / domain
    domain_dir.mkdir(parents=True, exist_ok=True)
    images_dir = domain_dir / "images"
    images_dir.mkdir(parents=True, exist_ok=True)

    local_md_url = _localize_url(markdown_url, base)
    print(f"  downloading: {local_md_url}")
    md_text = _download_markdown(local_md_url, timeout=timeout, base=base)

    img_urls = set()
    for m in IMG_MD_RE.finditer(md_text):
        img_urls.add(m.group(1))
    for m in IMG_SRC_RE.finditer(md_text):
        img_urls.add(m.group(1))

    downloaded = {}
    for img_url in img_urls:
        local_img_url = _localize_url(img_url, base)
        try:
            result = _download_binary(local_img_url, timeout=30, base=base)
            if result is None:
                continue
            data, ct, fname = result
            local_path = images_dir / fname
            local_path.write_bytes(data)
            downloaded[img_url] = f"images/{fname}"
            print(f"    img: {fname}  ({len(data)} bytes)")
        except Exception as exc:
            print(f"    img FAILED: {img_url[:80]}...  {exc}")

    for remote, local in downloaded.items():
        md_text = md_text.replace(remote, local)

    md_path = domain_dir / f"{file_stem}.md"
    md_path.write_text(md_text, encoding="utf-8")
    print(f"  saved: {md_path}  ({len(downloaded)} images)")
    return True, file_stem


def crawl_sitemap(
    base: str, url: str, out_dir: Path,
    limit: int = 0, max_depth: int = 0, threads: int = 4,
    max_images: int = 5, timeout: int = 600,
    screenshots: str | bool = False,
    include_subdomains: bool = False,
    include_page_data: bool = False,
    interactive_mode: str = "off",
    keep_crawl: bool = False,
    download_assets: bool = False,
    respect_robots: bool = True,
) -> bool:
    result = _api_post(base, "/crawl/sitemap", {
        "url": url,
        "markdown": True,
        "images": True,
        "max_images": max_images,
        "limit": limit,
        "max_depth": max_depth,
        "recursive": True,
        "threads": threads,
        "screenshots": screenshots,
        "include_subdomains": include_subdomains,
        "include_page_data": include_page_data,
        "interactive_mode": interactive_mode,
        "keep_crawl": keep_crawl,
        "download_assets": download_assets,
        "respect_robots": respect_robots,
    }, timeout=timeout)

    print(f"  status: {result.get('status')}  total: {result.get('total')}  ok: {result.get('succeeded')}  failed: {result.get('failed')}")
    failed = result.get("failed_urls", [])
    if failed:
        print("  failed URLs:")
        for u in failed:
            print(f"    {u}")

    markdown_urls = result.get("markdown_urls", {})
    if not markdown_urls:
        print("  No markdown_urls in response")
        return False

    ok = 0
    fail = 0
    total = len(markdown_urls)

    for page_url, md_url in markdown_urls.items():
        if not md_url:
            print(f"  SKIP (no markdown URL): {page_url}")
            fail += 1
            continue

        domain = urlparse(page_url).hostname or "unknown"
        domain_dir = out_dir / domain
        domain_dir.mkdir(parents=True, exist_ok=True)
        images_dir = domain_dir / "images"
        images_dir.mkdir(parents=True, exist_ok=True)
        file_stem = _stem_from_url(page_url)

        local_md_url = _localize_url(md_url, base)
        print(f"  [{ok + fail + 1}/{total}] downloading: {local_md_url}")
        try:
            md_text = _download_markdown(local_md_url, timeout=timeout, base=base)
        except Exception as exc:
            print(f"    markdown FAILED: {exc}")
            fail += 1
            continue

        img_urls = set()
        for m in IMG_MD_RE.finditer(md_text):
            img_urls.add(m.group(1))
        for m in IMG_SRC_RE.finditer(md_text):
            img_urls.add(m.group(1))

        downloaded = {}
        for img_url in img_urls:
            local_img_url = _localize_url(img_url, base)
            try:
                result = _download_binary(local_img_url, timeout=30, base=base)
                if result is None:
                    continue
                data, ct, fname = result
                local_path = images_dir / fname
                local_path.write_bytes(data)
                downloaded[img_url] = f"images/{fname}"
                print(f"    img: {fname}  ({len(data)} bytes)")
            except Exception as exc:
                print(f"    img FAILED: {img_url[:80]}...  {exc}")

        for remote, local in downloaded.items():
            md_text = md_text.replace(remote, local)

        md_path = domain_dir / f"{file_stem}.md"
        md_path.write_text(md_text, encoding="utf-8")
        print(f"    saved: {md_path}  ({len(downloaded)} images)")
        ok += 1

    print(f"\nDone: {ok} pages OK, {fail} errors")
    return fail == 0


# ---------------------------------------------------------------------------
def main():
    p = argparse.ArgumentParser(description="Crawler REST API client")
    p.add_argument("url", help="URL to crawl")
    p.add_argument("--recursive", "-r", action="store_true", help="Sitemap crawl (recursive)")
    p.add_argument("--limit", type=int, default=0, help="Max pages (0 = unlimited)")
    p.add_argument("--max-depth", type=int, default=0, help="Max recursive depth (0 = unlimited)")
    p.add_argument("--max-images", type=int, default=5, help="Max images per page")
    p.add_argument("--threads", type=int, default=4, help="Concurrent threads (sitemap mode)")
    p.add_argument("--screenshots", type=str, default="false", help="Screenshot mode: false, true/full, or viewport_initial")
    p.add_argument("--include-subdomains", action="store_true", help="Follow links to subdomains of the start URL")
    p.add_argument("--inline-pages", action="store_true", help="Include page data (markdown+images+links) in the API response")
    p.add_argument("--interactive-mode", type=str, default="off", choices=["off", "discover_only", "full"], help="JS interaction mode (default: off)")
    p.add_argument("--keep-crawl", action="store_true", help="Skip reprocessing when page content digest matches cached version")
    p.add_argument("--download-assets", action="store_true", help="Download CSS/JS/fonts from browser cache for offline HTML viewing")
    p.add_argument("--respect-robots", action="store_true", default=True, help="Respect robots.txt (default: true)")
    p.add_argument("--no-respect-robots", dest="respect_robots", action="store_false", help="Ignore robots.txt and crawl anyway")
    p.add_argument("--hostname", default=DEFAULT_HOST, help=f"Crawler hostname (default: {DEFAULT_HOST})")
    p.add_argument("--output", "-o", type=Path, default=DEFAULT_OUT, help=f"Output directory (default: {DEFAULT_OUT})")
    p.add_argument("--timeout", type=int, default=600, help="Total timeout in seconds")
    args = p.parse_args()

    host = args.hostname
    if not host.startswith("http"):
        proto = "http" if host.startswith(("localhost", "127.")) else "https"
        host = f"{proto}://{host}"
    host = host.rstrip("/")

    out_dir = Path(args.output)
    out_dir.mkdir(parents=True, exist_ok=True)

    print(f"Host: {host}")
    print(f"Output: {out_dir.resolve()}")
    print()

    if args.recursive:
        screenshots_val = args.screenshots
        if screenshots_val.lower() == "true":
            screenshots_val = True
        elif screenshots_val.lower() == "false":
            screenshots_val = False

        print(f"Crawling (sitemap): {args.url}")
        print(f"  depth={args.max_depth}  limit={args.limit}  images={args.max_images}  threads={args.threads}")
        print(f"  screenshots={screenshots_val}  subdomains={args.include_subdomains}  interactive={args.interactive_mode}")
        ok = crawl_sitemap(
            host, args.url, out_dir,
            limit=args.limit, max_depth=args.max_depth, threads=args.threads,
            max_images=args.max_images, timeout=args.timeout,
            screenshots=screenshots_val,
            include_subdomains=args.include_subdomains,
            include_page_data=args.inline_pages,
            interactive_mode=args.interactive_mode,
            keep_crawl=args.keep_crawl,
            download_assets=args.download_assets,
            respect_robots=args.respect_robots,
        )
    else:
        print(f"Crawling (single): {args.url}")
        ok, _ = crawl_single(
            host, args.url, out_dir,
            max_images=args.max_images, timeout=args.timeout,
            respect_robots=args.respect_robots,
        )

    sys.exit(0 if ok else 1)


if __name__ == "__main__":
    main()
