"""Web crawler payload models for Kafka topics."""
from pydantic import BaseModel


class UrlReadyPayload(BaseModel):
    """Payload for web.url.ready — dispatched by web-engine to web-fetcher."""

    url_id: str
    url: str
    seed_id: str
    seed_config: dict | None = None
    parent_url: str | None = None
    depth: int = 0
    priority: int = 0
    metadata: dict | None = None

    model_config = {"extra": "allow"}


class RawReadyPayload(BaseModel):
    """Payload for web.raw.ready — emitted by web-fetcher with inline HTML."""

    url_id: str
    url: str
    seed_id: str
    seed_config: dict | None = None
    parent_url: str | None = None  # Page that linked to this URL (carried from UrlReadyPayload)
    html: str
    status: int
    headers: dict | None = None
    depth: int = 0
    fetched_at: str | None = None
    # -- Request metadata -----------------------------------------------
    final_url: str | None = None  # URL after redirects
    request_method: str = "GET"  # HTTP method used
    redirect_chain: list[str] | None = None  # Intermediate URLs during redirects
    retry_count: int = 0  # Number of retry attempts before success
    latency_ms: int | None = None  # HTTP request duration in milliseconds
    # -- Content hash ---------------------------------------------------
    raw_hash: str | None = None  # SHA-256 hex digest of the HTML content
    # -- Persistence reference ------------------------------------------
    web_html_raw_id: str | None = None  # DB-assigned id after persistence (set by persistence-worker)

    model_config = {"extra": "allow"}


class DiscoveryReadyPayload(BaseModel):
    """Payload for web.discovery.ready — emitted by web-discovery."""

    seed_id: str
    seed_config: dict | None = None
    source_url: str
    discovery_depth: int = 0
    new_urls: list[str] = []
    url_priorities: dict[str, int] | None = None
    duplicate_urls: list[str] = []
    blocked_urls: list[str] = []
    invalid_urls: list[str] = []
    external_urls: list[str] = []
    depth_exceeded_urls: list[str] = []

    model_config = {"extra": "allow"}


class CrawlMetricPayload(BaseModel):
    """Payload for web.crawl.metrics — operational telemetry."""

    seed_id: str
    seed_config: dict | None = None
    url: str | None = None
    status: str | None = None
    status_code: int | None = None
    error_message: str | None = None
    fetch_duration_ms: int | None = None
    fetched_at: str | None = None

    model_config = {"extra": "allow"}


class ParsedReadyPayload(BaseModel):
    """Payload for web.parsed.ready — emitted by web-parser with extracted content."""

    url: str
    seed_id: str
    seed_config: dict | None = None
    title: str | None = None
    description: str | None = None
    author: str | None = None
    published_time: str | None = None
    main_image: str | None = None
    content: str | None = None
    content_html: str | None = None
    keywords: list[str] | None = None
    tags: list[str] | None = None
    category: str | None = None
    images: list[dict] | None = None
    tables: list[dict] | None = None
    links: list[dict] | None = None
    schema_data: list[dict] | None = None
    statistics: dict | None = None
    custom_content: dict | None = None
    parsed_at: str | None = None
    diagnostics: dict | None = None
    # -- Upstream reference ---------------------------------------------
    web_html_raw_id: str | None = None  # DB id of the source raw HTML row (from web.raw.persisted)
    version: int = 1

    model_config = {"extra": "allow"}


class ParsedPersistedPayload(BaseModel):
    """Payload for web.parsed.persisted — emitted by persistence-worker after DB write.

    Contains the id assigned at persistence time plus the original parsed fields.
    """

    id: str
    url: str
    seed_id: str
    title: str | None = None
    description: str | None = None
    author: str | None = None
    published_time: str | None = None
    main_image: str | None = None
    content: str | None = None
    keywords: list[str] | None = None
    tags: list[str] | None = None
    category: str | None = None
    statistics: dict | None = None
    custom_content: dict | None = None
    parsed_at: str | None = None
    persisted_at: str | None = None
    # -- Persistence reference ------------------------------------------
    web_parsed_page_id: str | None = None  # DB-assigned id after persistence (set by persistence-worker)
    # -- Upstream reference ---------------------------------------------
    web_html_raw_id: str | None = None  # DB id of the source raw HTML row (from web.raw.persisted)
    raw_url_id: str | None = None  # DB id of the source web_html_raw row (maps to raw_url_id column)
    version: int = 1

    model_config = {"extra": "allow"}


class WebDownloadReadyPayload(BaseModel):
    """Published by web-engine → consumed by web-downloader.

    Carries the source URL of the media to download plus context
    about the page it was discovered on. web-engine handles rate
    limiting, politeness, and scheduling before dispatching.
    """

    url_id: str | None = None  # URL row ID in web_discovered_urls (for status updates)
    url: str  # Source URL of the media file to download
    parent_url: str  # Page where the media was discovered
    seed_id: str  # Seed identifier
    seed_config: dict | None = None  # Seed configuration (for rule-based decisions)
    media_type: str  # "image", "video", "document", "attachment"
    expected_content_hash: str | None = None  # Known SHA-256 (enables skip-if-exists)
    original_filename: str | None = None  # Filename from Content-Disposition or URL path
    page_depth: int = 0  # Crawl depth of the source page
    priority: int = 0  # Download priority (lower = higher)
    metadata: dict | None = None  # Arbitrary extra context
    discovered_at: str | None = None  # ISO 8601
    source: str | None = None  # Which service emitted this: "web-fetcher", "web-engine", etc.

    model_config = {"extra": "allow"}


class WebMediaReadyPayload(BaseModel):
    """Published by web-downloader → consumed by persistence-worker.

    Emitted after the media has been successfully downloaded and
    stored in RustFS. Contains the object key so downstream services
    can generate presigned URLs or reference the stored object.
    """

    url_id: str | None = None  # URL row ID in web_discovered_urls (for status updates)
    source_url: str  # Original source URL of the media
    parent_url: str  # Page where the media was discovered
    seed_id: str  # Seed identifier
    seed_config: dict | None = None  # Seed configuration
    media_type: str  # "image", "video", "document", "attachment"
    storage_bucket: str  # "web"
    storage_key: str  # Object key in RustFS (e.g., "web/images/ab/abc123.jpg")
    content_hash: str  # SHA-256 of the downloaded content
    etag: str | None = None  # HTTP ETag from source server response
    content_disposition: str | None = None  # Raw Content-Disposition header from source server
    content_type: str | None = None  # MIME type (e.g., "image/jpeg")
    content_length: int  # Size in bytes
    original_filename: str | None = None  # Original filename
    metadata: dict | None = None  # Arbitrary extra context
    downloaded_at: str | None = None  # ISO 8601 — when download completed
    download_duration_ms: int | None = None  # Total time (download + upload) in ms
    # -- Request metadata -----------------------------------------------
    final_url: str | None = None  # URL after redirects
    request_method: str = "GET"  # HTTP method used
    redirect_chain: list[str] | None = None  # Intermediate URLs during redirects
    retry_count: int = 0  # Number of retry attempts before success
    latency_ms: int | None = None  # HTTP request duration in milliseconds
    status_code: int | None = None  # HTTP response status code
    response_headers: dict | None = None  # Full HTTP response headers

    model_config = {"extra": "allow"}


class WebDownloadMetricPayload(BaseModel):
    """Published by web-downloader → consumed by persistence-worker.

    Per-media operational telemetry — download/upload timing, status
    tracking, error attribution.  Keyed by ``source_url`` for upsert
    into the ``web_metrics`` Doris table.

    Warehouse-only — no PostgreSQL or Elasticsearch backing.
    """

    source_url: str  # Source URL of the media — natural business key
    seed_id: str  # Seed identifier
    media_type: str  # "image", "video", "document", "attachment"
    status: str  # "downloaded", "uploaded", "published", "error"
    content_type: str | None = None  # MIME type (e.g., "image/jpeg")
    content_length: int | None = None  # Size in bytes
    download_duration_ms: int | None = None  # HTTP download time in ms
    upload_duration_ms: int | None = None  # RustFS upload time in ms
    total_duration_ms: int | None = None  # Total pipeline time in ms
    error_message: str | None = None  # Error details when status="error"
    downloaded_at: str | None = None  # ISO 8601 — when download completed

    model_config = {"extra": "allow"}
