"""MCP research server: search Google (serper.dev), filter with DeepSeek, crawl via API.

Must not import crawler.py, file_server.py, or nodriver.
"""

from __future__ import annotations

import json
import logging
import os
import time
from urllib.parse import urlparse

import requests
import uvicorn
from fastmcp import FastMCP
from fastmcp.server.http import create_streamable_http_app
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request

# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
CRAWLER_API_BASE_URL = os.getenv("CRAWLER_API_BASE_URL", "http://crawler:8114").rstrip("/")
MCP_PUBLIC_BASE_URL = os.getenv("MCP_PUBLIC_BASE_URL", "http://localhost:8115").rstrip("/")
SERPER_API_KEY = os.getenv("SERPER_API_KEY", "")
DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_API_KEY", "")
DEEPSEEK_MODEL = os.getenv("DEEPSEEK_MODEL", "deepseek-chat")
HTTP_TIMEOUT = float(os.getenv("MCP_HTTP_TIMEOUT_SECONDS", "180"))

mcp = FastMCP(
    "Crawler Research",
    instructions=(
        "Use crawl_page for a single URL. Use research for multi-page site research. "
        "This MCP server calls the crawler API and does not crawl directly."
    ),
)


@mcp.tool
def ping() -> str:
    """Fast connectivity check — responds instantly."""
    return "pong"


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
async def _ctx_info(ctx: Context | None, message: str) -> None:
    if ctx is not None:
        await ctx.info(message)


async def _ctx_progress(ctx: Context | None, progress: int, total: int) -> None:
    if ctx is not None:
        await ctx.report_progress(progress=progress, total=total)


def _require_env() -> None:
    if not SERPER_API_KEY:
        raise RuntimeError("SERPER_API_KEY is not set")
    if not DEEPSEEK_API_KEY:
        raise RuntimeError("DEEPSEEK_API_KEY is not set")


def _normalize_site(site: str) -> str:
    value = site.strip().lower()
    if value.startswith(("http://", "https://")):
        value = urlparse(value).hostname or value
    return value.split("/")[0].rstrip(".").strip()


def _same_or_subdomain(url: str, site: str) -> bool:
    try:
        host = urlparse(url).hostname or ""
    except Exception:
        return False
    host = host.lower().rstrip(".")
    site = site.lower().rstrip(".")
    return host == site or host.endswith("." + site)


def _api_post(path: str, payload: dict, timeout: float | None = None) -> dict:
    url = f"{CRAWLER_API_BASE_URL.rstrip('/')}{path}"
    resp = requests.post(url, json=payload, timeout=timeout or HTTP_TIMEOUT)
    if not resp.ok:
        raise RuntimeError(f"API {path} returned {resp.status_code}: {resp.text[:500]}")
    return resp.json()


def _api_get(path: str, timeout: float | None = None) -> dict:
    url = f"{CRAWLER_API_BASE_URL.rstrip('/')}{path}"
    resp = requests.get(url, timeout=timeout or HTTP_TIMEOUT)
    if not resp.ok:
        raise RuntimeError(f"API {path} returned {resp.status_code}")
    return resp.json()


def _api_delete(path: str, timeout: float | None = None) -> dict:
    url = f"{CRAWLER_API_BASE_URL.rstrip('/')}{path}"
    resp = requests.delete(url, timeout=timeout or HTTP_TIMEOUT)
    if not resp.ok:
        raise RuntimeError(f"API {path} returned {resp.status_code}")
    return resp.json()


def _google_search(question: str, site: str, num: int = 20) -> list[dict]:
    query = f"site:{site} {question}"
    resp = requests.post(
        "https://google.serper.dev/search",
        json={"q": query, "num": num},
        headers={"X-API-KEY": SERPER_API_KEY, "Content-Type": "application/json"},
        timeout=30,
    )
    resp.raise_for_status()
    data = resp.json()
    return [
        {
            "url": r.get("link", ""),
            "title": r.get("title", ""),
            "snippet": r.get("snippet", ""),
            "position": r.get("position"),
        }
        for r in data.get("organic", [])
        if r.get("link")
    ]


def _deepseek_ask(prompt: str, timeout: float = 15) -> str:
    resp = requests.post(
        "https://api.deepseek.com/v1/chat/completions",
        headers={"Authorization": f"Bearer {DEEPSEEK_API_KEY}"},
        json={
            "model": DEEPSEEK_MODEL,
            "messages": [{"role": "user", "content": prompt}],
            "max_tokens": 120,
            "temperature": 0.0,
        },
        timeout=timeout,
    )
    resp.raise_for_status()
    return resp.json()["choices"][0]["message"]["content"].strip()


def _filter_relevant(question: str, candidates: list[dict], max_pages: int = 10) -> list[dict]:
    relevant: list[dict] = []
    for c in candidates:
        if max_pages > 0 and len(relevant) >= max_pages:
            break
        url = c.get("url", "")
        title = c.get("title", "")
        snippet = c.get("snippet", "")
        prompt = (
            "You decide if a Google result is useful for answering a user question.\n"
            "Return only compact JSON with keys: relevant, reason.\n\n"
            f"Question: {question}\n"
            f"URL: {url}\n"
            f"Title: {title}\n"
            f"Snippet: {snippet}\n"
        )
        try:
            reply = _deepseek_ask(prompt)
        except Exception as exc:
            relevant.append({
                "url": url, "title": title, "snippet": snippet,
                "position": c.get("position"),
                "relevance_answer": False,
                "relevance_reason": f"DeepSeek error: {exc}",
            })
            continue

        try:
            parsed = json.loads(reply)
            rel_bool = bool(parsed.get("relevant", False))
            reason = str(parsed.get("reason", ""))
        except Exception:
            low = reply.lower()
            rel_bool = "yes" in low and "no" not in low[:30]
            reason = reply

        result = {
            "url": url, "title": title, "snippet": snippet,
            "position": c.get("position"),
            "relevance_answer": rel_bool,
            "relevance_reason": reason,
        }
        if rel_bool:
            relevant.append(result)
    return relevant


def _relevance_by_url(relevant: list[dict]) -> dict[str, dict]:
    return {r["url"]: r for r in relevant}


def _attach_relevance(pages: list[dict], relevant: list[dict]) -> list[dict]:
    by_url = _relevance_by_url(relevant)
    for page in pages:
        url = page.get("url", "")
        rel = by_url.get(url)
        if rel:
            page["relevance_reason"] = rel.get("relevance_reason", "")
    return pages


# ---------------------------------------------------------------------------
# Tools
# ---------------------------------------------------------------------------
@mcp.tool
def crawl_page(
    url: str,
    max_images: int = 5,
    max_inline_chars: int = 0,
    recrawl: bool = True,
) -> dict:
    errors: list[str] = []

    try:
        if recrawl:
            data = _api_post("/crawl/single", {
                "url": url,
                "markdown": True,
                "images": True,
                "screenshots": False,
                "quality_check": False,
                "max_images": max_images,
                "include_page_data": True,
                "max_inline_chars": max_inline_chars,
            })
            page = data.get("page") or {}
            source = "crawl_api"
        else:
            page = _api_post("/artifacts/page", {
                "url": url,
                "include_markdown": True,
                "include_links": True,
                "include_images": True,
                "max_images": max_images,
                "max_inline_chars": max_inline_chars,
            })
            source = "artifact_api"
    except Exception as exc:
        errors.append(str(exc))
        return {"url": url, "status": "error", "source": "api", "page": {}, "errors": errors}

    page_errors = page.get("errors") if isinstance(page, dict) else []
    if isinstance(page_errors, list):
        errors.extend(str(e) for e in page_errors)

    return {
        "url": url,
        "status": page.get("status", "error") if isinstance(page, dict) else "error",
        "source": source,
        "page": page,
        "errors": errors,
    }


@mcp.tool
def research(
    question: str,
    site: str,
    max_pages: int = 10,
    max_images: int = 3,
    max_inline_chars: int = 12000,
    recrawl: bool = True,
) -> dict:
    errors: list[str] = []
    site_norm = _normalize_site(site)
    max_pages = max(1, min(int(max_pages), 30))

    try:
        _require_env()
    except Exception as exc:
        return {
            "question": question, "site": site_norm, "status": "error",
            "search_results": 0, "relevant_found": 0,
            "pages_requested": 0, "pages_crawled": 0, "pages_failed": 0,
            "pages": [], "failed_pages": [],
            "errors": [str(exc)],
        }

    try:
        search_results = _google_search(question, site_norm, num=min(max_pages * 2, 20))
    except Exception as exc:
        return {
            "question": question, "site": site_norm, "status": "error",
            "search_results": 0, "relevant_found": 0,
            "pages_requested": 0, "pages_crawled": 0, "pages_failed": 0,
            "pages": [], "failed_pages": [],
            "errors": [f"search failed: {exc}"],
        }

    search_results = [r for r in search_results if _same_or_subdomain(r.get("url", ""), site_norm)]
    relevant = _filter_relevant(question, search_results, max_pages=max_pages)
    urls = [r["url"] for r in relevant]

    if not urls:
        return {
            "question": question, "site": site_norm, "status": "ok",
            "search_results": len(search_results), "relevant_found": 0,
            "pages_requested": 0, "pages_crawled": 0, "pages_failed": 0,
            "pages": [], "failed_pages": [],
            "errors": [],
        }

    try:
        if recrawl:
            api_data = _api_post("/crawl/urls", {
                "urls": urls,
                "markdown": True, "images": True,
                "screenshots": False, "quality_check": False,
                "max_images": max_images, "threads": 4,
                "include_page_data": True,
                "max_inline_chars": max_inline_chars,
            })
        else:
            api_data = _api_post("/artifacts/pages", {
                "urls": urls,
                "include_markdown": True, "include_links": True,
                "include_images": True,
                "max_images": max_images,
                "max_inline_chars": max_inline_chars,
            })
    except Exception as exc:
        return {
            "question": question, "site": site_norm, "status": "error",
            "search_results": len(search_results), "relevant_found": len(relevant),
            "pages_requested": len(urls), "pages_crawled": 0, "pages_failed": len(urls),
            "pages": [],
            "failed_pages": [{"url": u, "error": str(exc)} for u in urls],
            "errors": [str(exc)],
        }

    pages = api_data.get("pages", []) if isinstance(api_data, dict) else []
    pages = _attach_relevance(pages, relevant)
    ok_pages = [p for p in pages if p.get("status") == "ok"]
    failed_pages = [
        {"url": p.get("url", ""), "error": "; ".join(p.get("errors", [])) or "crawl failed"}
        for p in pages
        if p.get("status") != "ok"
    ]

    return {
        "question": question, "site": site_norm,
        "status": "ok" if not failed_pages else "partial",
        "search_results": len(search_results), "relevant_found": len(relevant),
        "pages_requested": len(urls),
        "pages_crawled": len(ok_pages), "pages_failed": len(failed_pages),
        "pages": pages, "failed_pages": failed_pages,
        "errors": errors,
    }


@mcp.tool
def inspect_page(url: str) -> dict:
    errors: list[str] = []
    try:
        session = _api_post("/browser/sessions", {"url": url, "viewport_width": 1920, "viewport_height": 1080})
        sid = session.get("session_id", "")
    except Exception as exc:
        return {"status": "error", "errors": [str(exc)]}
    actions = []
    dom = {}
    try:
        act_resp = _api_get(f"/browser/sessions/{sid}/actions")
        actions = act_resp.get("actions", [])
        dom_resp = _api_get(f"/browser/sessions/{sid}/dom?max_nodes=100")
        dom = dom_resp
    except Exception:
        pass
    return {
        "status": "ok", "session_id": sid, "url": url,
        "title": dom.get("title", ""),
        "visible_text_preview": dom.get("visible_text_preview", "")[:2000],
        "node_count": dom.get("node_count", 0),
        "nodes": dom.get("nodes", [])[:50],
        "actions": actions[:50],
        "errors": errors,
    }


@mcp.tool
def analyze_page_with_gemini(session_id: str) -> dict:
    try:
        result = _api_post(f"/analysis/gemini/page", {
            "session_id": session_id,
            "analysis_type": "page_parser_discovery",
            "include_screenshot": True, "include_dom": True, "include_actions": True,
        }, timeout=60)
        return result
    except Exception as exc:
        return {"status": "error", "session_id": session_id, "errors": [str(exc)]}


@mcp.tool
def test_selector(
    session_id: str,
    selector: str,
    return_text: bool = True,
    limit: int = 20,
) -> dict:
    try:
        return _api_post("/parser/test-selector", {
            "session_id": session_id, "selector": selector, "return_text": return_text, "limit": limit,
        }, timeout=15)
    except Exception as exc:
        return {"status": "error", "errors": [str(exc)]}


@mcp.tool
def test_extractor(
    session_id: str,
    extractor: dict,
    limit: int = 50,
) -> dict:
    try:
        return _api_post("/parser/test-extractor", {
            "session_id": session_id, "extractor": extractor, "limit": limit,
        }, timeout=15)
    except Exception as exc:
        return {"status": "error", "errors": [str(exc)]}


@mcp.tool
def create_parser_project(
    domain: str,
    parser_name: str,
    parser_type: str = "product_discovery",
) -> dict:
    try:
        return _api_post("/parser/projects", {
            "domain": domain, "parser_name": parser_name, "parser_type": parser_type,
        }, timeout=10)
    except Exception as exc:
        return {"status": "error", "errors": [str(exc)]}


@mcp.tool
def save_parser_manifest(
    project_id: str,
    manifest: dict,
) -> dict:
    try:
        return _api_put(f"/parser/projects/{project_id}/manifest", {"manifest": manifest})
    except Exception as exc:
        return {"status": "error", "errors": [str(exc)]}


@mcp.tool
def validate_parser(
    project_id: str,
    test_urls: list[str],
    required_fields: list[str] | None = None,
) -> dict:
    try:
        checks = {}
        if required_fields:
            checks["required_fields"] = required_fields
        checks["min_products"] = 1
        return _api_post(f"/parser/projects/{project_id}/validate", {
            "test_urls": test_urls, "checks": checks,
        }, timeout=180)
    except Exception as exc:
        return {"status": "error", "errors": [str(exc)]}


@mcp.tool
def click_browser_action(
    session_id: str,
    action_id: str,
) -> dict:
    try:
        result = _api_post(
            f"/browser/sessions/{session_id}/actions/{action_id}/click",
            {},
            timeout=30,
        )
        return result
    except Exception as exc:
        return {"status": "error", "session_id": session_id, "errors": [str(exc)]}


@mcp.tool
def close_browser_session(session_id: str) -> dict:
    try:
        _api_delete(f"/browser/sessions/{session_id}")
        return {"status": "ok", "session_id": session_id, "closed": True}
    except Exception as exc:
        return {"status": "error", "errors": [str(exc)]}


def _api_put(path: str, payload: dict, timeout: float | None = None) -> dict:
    url = f"{CRAWLER_API_BASE_URL.rstrip('/')}{path}"
    resp = requests.put(url, json=payload, timeout=timeout or HTTP_TIMEOUT)
    if not resp.ok:
        raise RuntimeError(f"API {path} returned {resp.status_code}")
    return resp.json()


# ---------------------------------------------------------------------------
# Transport
# ---------------------------------------------------------------------------
logger = logging.getLogger("mcp_research")


class RequestLogMiddleware(BaseHTTPMiddleware):
    async def dispatch(self, request: Request, call_next):
        if request.url.path == "/mcp" and request.method.upper() == "POST":
            started = time.perf_counter()
            try:
                response = await call_next(request)
                elapsed_ms = (time.perf_counter() - started) * 1000.0
                level = logging.WARNING if response.status_code >= 400 else logging.INFO
                logger.log(level, "mcp request method=%s path=%s status=%s dur=%.1fms",
                           request.method, request.url.path, response.status_code, elapsed_ms)
                return response
            except Exception as exc:
                logger.log(logging.ERROR, "mcp error method=%s path=%s error=%s",
                           request.method, request.url.path, str(exc))
                raise
        return await call_next(request)


app = create_streamable_http_app(server=mcp, streamable_http_path="/mcp", stateless_http=True)
app.add_middleware(RequestLogMiddleware)


if __name__ == "__main__":
    host = os.getenv("MCP_HOST", "0.0.0.0")
    port = int(os.getenv("MCP_PORT", "8115"))
    print(f"MCP server on {host}:{port} (/mcp)", flush=True)
    uvicorn.run(app, host=host, port=port)
