"""Python beside the wikimedia-commons recipe: one open API asked four ways, and the order
the site shows.

**Commons is a JSON site with the door standing open.** Its `w/api.php` answers without a
key, without a token and without a wall - it answers a client that sends no name at all
(measured 2026-09-22). We ask under the program's own name anyway, and fetch the files under
it too: Wikimedia's robot policy asks every automated client to say who it is and not to
pass itself off as a browser, and `PROGRAM_AGENT` is exactly that.

**Why code at all, on a site this open:** the order, and the filters. A search answer lists
its files in no useful order and ranks them in a field, `index`; what the person saw on
`Special:MediaSearch` is the ranking, and the first ten of the page match the answer sorted
by `index` position for position. The filters of that page live in its address and turn into
words of the query the way the site's own code writes them. The declaration has neither a
sort nor that translation, so both live here. The nodes keep what is theirs: which address is
which, the project's name, and paging.

**Formats are asked for in `formatversion=2`**, where `query.pages` is a list. Version one
returns an object keyed by page id, and a declaration walking it would iterate once over the
object itself and quietly find nothing.

**The API says no with a `200`.** An error arrives as a body with an `error` field and no
pages, which the nodes would read as the end of the listing - a run that stops early and
looks complete. So a refusal is raised here, as `flickr.py` does for its `stat: fail`.
"""

import json
import re
from html import unescape
from urllib.parse import parse_qs, quote, unquote, urlsplit

from nd_crawler import SiteHandler
from nd_sdk.manners import PROGRAM_AGENT

#: The site and its API.
SITE = "https://commons.wikimedia.org"
API = f"{SITE}/w/api.php"

#: **The program's own name, on the API and on every file.** The robot policy asks for it on
#: both hosts; gallery-dl fetches Commons under its own name too.
NAMED = {"user-agent": PROGRAM_AGENT}

#: **What the site's own search asks for each tab of `Special:MediaSearch`.** Taken from its
#: source (`SpecialMediaSearch::search`, extension MediaSearch) and measured 2026-09-22: the
#: audio tab for «mozart» reports 997 hits and so does `filetype:audio mozart`, with the same
#: eight files in the same order. Without the filter the same word answers 60 821 hits led by
#: photographs - that is, a different door from the one the person used.
FILETYPES = {
    "image": "filetype:bitmap|drawing",
    "video": "filetype:video",
    "audio": "filetype:audio",
    "other": "filetype:multimedia|office|archive|3d",
}

#: The tab that lists pages rather than files.
PAGES_TAB = "page"

#: **The filters of the page that are words of the query as they stand.** The site writes
#: `name:value` for each (`getTermWithFilters`), and so do we: «tiger» with the licence filter
#: "No restrictions" reports 39 985 on the page and 39 985 from the API with
#: `haslicense:unrestricted` (measured 2026-09-22), against 135 576 without it. Leaving them
#: out would download mostly what the person had filtered away.
KEYWORDS = ("filemime", "fileres", "haslicense")

#: **The assessment filter becomes a statement**, one per value the page offers - read off the
#: page's own options on Commons (`searchOptions.json`, 2026-09-22).
ASSESSMENTS = {
    "any-assessment": "haswbstatement:P6731=Q63348049|P6731=Q63348069|P6731=Q63348040|P6731=Q6998859",
    "featured-image": "haswbstatement:P6731=Q63348049",
    "quality-image": "haswbstatement:P6731=Q63348069",
    "valued-image": "haswbstatement:P6731=Q63348040",
    "picture-of-the-day": "haswbstatement:P6731=Q6998859",
}

#: "Recency" on the page, as the site asks for it; everything else is its default, relevance.
RECENCY = "create_timestamp_desc"

#: What a filter's value may hold. No space, so a value cannot add a word of its own to the
#: query; the page offers nothing outside this (`png`, `500,1000`, `>1000`, `unrestricted`).
_FILTER_VALUE = re.compile(r"[A-Za-z0-9<>=,.+-]+")

#: **The site's search stops at ten thousand**, whatever it reports: at `gsroffset=10000` the
#: API answers `cirrussearch-offset-too-large` (measured 2026-09-22).
SEARCH_CEILING = 10_000
TOO_FAR = "cirrussearch-offset-too-large"

#: Everything a file needs, in one answer: where it is, what it is, and what the page says
#: about it. No second request per file.
FACTS = "url|size|mime|user|timestamp|extmetadata"

#: **Only the metadata we read.** The full set carries every category and assessment of a
#: file, and an answer that outgrows the API's limit comes back without the details of some
#: files (`iicontinue`). Five hundred files with everything weigh 1.4 MB against a limit of
#: 8 MB (measured 2026-09-22); asking for four fields keeps that limit out of reach.
METADATA = "ImageDescription|Artist|LicenseShortName|DateTimeOriginal"

#: Asked for and given: five hundred, both in the search and in the lists (measured).
PER_PAGE = 500

#: The namespaces this recipe meets: a file and a category.
FILE = 6
CATEGORY = 14

_TAGS = re.compile(r"<[^>]+>")
_SPACES = re.compile(r"\s+")
#: **Hidden blocks are not text.** Commons writes a machine-readable copy of a date into a
#: `display: none` element (`date QS:P,+1898-00-00T00:00:00Z/7,...`), and stripping only the
#: tags would leave it standing in the words.
_HIDDEN = re.compile(r"<(div|span)\b[^>]*display:\s*none[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)

_MONTHS = {
    name: number
    for number, name in enumerate(
        ("january", "february", "march", "april", "may", "june", "july",
         "august", "september", "october", "november", "december"),
        start=1,
    )
}
_ISO_DAY = re.compile(r"\b(\d{4})-(\d{2})-(\d{2})\b")
_DAY_MONTH_YEAR = re.compile(r"\b(\d{1,2}) ([A-Za-z]+),? (\d{4})\b")
_MONTH_DAY_YEAR = re.compile(r"\b([A-Za-z]+) (\d{1,2}), (\d{4})\b")
#: A year, or a year and a month, standing alone - `1895`, `2006-05`.
_YEAR = re.compile(r"\d{4}(-\d{2})?")


class Commons(SiteHandler):
    """One API, four questions, and every answer announced in the order the site shows."""

    def ask(self, step, engine):
        """The API call this step is really about, under the program's own name."""
        address = self._address(step, engine)
        if address is None:
            return None
        return engine.request(address, headers=NAMED)

    def _address(self, step, engine):
        """What to ask for, or `None` when the address is none of the shapes the nodes name."""
        asked = {
            "action": "query",
            "format": "json",
            "formatversion": "2",
            "prop": "imageinfo",
            "iiprop": FACTS,
            "iiextmetadatafilter": METADATA,
        }
        if step.payload.get("empty"):
            return _whose_redirect(_caught(engine, "category"))
        if title := _caught(engine, "file"):
            asked["titles"] = f"File:{title}"
        elif category := _caught(engine, "category"):
            asked |= {
                "generator": "categorymembers",
                # Files and subcategories in one answer: a category page shows both, and the
                # subcategories are where Commons keeps most of itself.
                "gcmtitle": f"Category:{category}",
                "gcmtype": "file|subcat",
                "gcmlimit": str(PER_PAGE),
            }
            if step.cursor:
                asked["gcmcontinue"] = step.cursor
        elif who := _caught(engine, "who"):
            asked |= {
                "generator": "allimages",
                "gaiuser": who,
                "gaisort": "timestamp",
                # Newest first, as `Special:ListFiles` shows them. The API's own direction is
                # the opposite, and a free project's share of files would be the oldest ones.
                "gaidir": "descending",
                "gailimit": str(PER_PAGE),
            }
            if step.cursor:
                asked["gaicontinue"] = step.cursor
        else:
            wanted = _search(step)
            if wanted is None:
                return None
            asked |= wanted
        spelled = "&".join(f"{name}={quote(value, safe='|')}" for name, value in asked.items())
        return f"{API}?{spelled}"

    def read(self, step, answer, engine):
        """Announce this answer's files in the site's order, and walk into its subcategories
        while the depth allows.

        **Subcategories first, then files, each in the order the site gives them** - which is
        how a category page lays itself out, its subcategories in a block above its media. The
        subcategories only queue steps of their own, so their files still come after these.
        """
        try:
            said = json.loads(answer.body)
        except ValueError:
            return None
        if not isinstance(said, dict):
            return None
        if refused := said.get("error"):
            return _refused(refused, engine)
        if step.payload.get("empty"):
            return _go_where_it_points(said, engine)
        pending = (said.get("continue") or {}).get("iicontinue")
        if pending:
            # The generator does not move on until every file of this batch has its details,
            # and the nodes page by the generator alone: carrying on would lose the rest of the
            # batch without a word. Out of reach with `METADATA`, and loud should it happen.
            raise RuntimeError(
                "Commons could not fit the details of every file into one answer "
                f"(continues at {pending}), so this part of the listing stops here"
            )
        _say_the_ceiling(step, said, engine)
        found = (said.get("query") or {}).get("pages")
        rows = _in_site_order([one for one in (found or []) if isinstance(one, dict)], engine)
        deeper = _whole(engine.param("subcategories"))
        level = _whole(step.payload.get("level"))
        first = not step.cursor and not level and not step.payload.get("arrived")
        if first and not rows and _caught(engine, "category"):
            # A second question rather than one more field on the first: the listing is a
            # generator, and the page itself is not among what it answers about.
            engine.follow_up(f"{step.url.split('?')[0]}?empty=1", payload={"empty": True})
            return None
        for one in rows:
            if one.get("ns") == CATEGORY:
                if level < deeper:
                    _walk_into(one, level, engine)
            elif one.get("missing") and _caught(engine, "file"):
                engine.log(f"{one.get('title') or 'This file'} is not on Commons (deleted or renamed)")
            elif one.get("ns") == FILE or one.get("imageinfo"):
                _announce(one, engine)
        return None


def _whose_redirect(category):
    """What to ask of a category with nothing in it: whether it only points at another.

    `{{Category redirect}}` leaves the page standing and empties it, so `Category:Hokusai` is a
    successful run of nothing, and the person is not told that `Category:Katsushika Hokusai` is
    where the files are (2026-10-29). Its one link into the category namespace is the target.
    """
    asked = {
        "action": "query",
        "format": "json",
        "formatversion": "2",
        "titles": f"Category:{category}",
        "prop": "templates|links",
        "tltemplates": "Template:Category redirect",
        "plnamespace": str(CATEGORY),
        "pllimit": "2",
    }
    spelled = "&".join(f"{name}={quote(value, safe='|')}" for name, value in asked.items())
    return f"{API}?{spelled}"


def _go_where_it_points(said, engine):
    """Walk into the target of a redirect, once, or say the category is empty."""
    found = (said.get("query") or {}).get("pages") or []
    page = found[0] if found and isinstance(found[0], dict) else {}
    links = [str(one.get("title") or "") for one in page.get("links") or [] if isinstance(one, dict)]
    title = str(page.get("title") or "This category")
    if not page.get("templates") or len(links) != 1:
        engine.log(f"{title} has no files and no subcategories")
        return None
    engine.log(f"{title} only points to {links[0]}: downloading from there")
    # `arrived` stops here: a redirect to a redirect is somebody's mistake, not a chain to walk.
    engine.follow_up(f"{SITE}/wiki/{quote(links[0].replace(' ', '_'), safe=':/')}", payload={"arrived": True})
    return None


def _search(step):
    """The search the person was looking at, asked of the API the way the site asks it.

    **The tab and the filters are part of the question**, and the page builds its query from
    them in one fixed way (`SpecialMediaSearch::search`): the tab's `filetype:`, `-fileres:0`
    for everything but sound (files the site cannot measure are left out of its own results),
    each filter that is a search keyword as `name:value`, the assessment as a statement, then
    the words; "Recency" is a sort rather than a word.

    The address's own offset (`&continue=40`, the site's "load more") is deliberately not
    carried over: somebody who paged through the results and pressed download wants the
    search, not the rest of it. Paging is the run's own, five hundred at a time, and the site
    stops anybody at ten thousand - past that the API answers `cirrussearch-offset-too-large`
    with no files.
    """
    query = parse_qs(urlsplit(step.start or step.url).query)

    def given(name):
        return (query.get(name) or [""])[0].strip()

    words = given("search")
    if not words:
        return None
    kind = given("type").lower() or "image"
    if kind == PAGES_TAB:
        raise RuntimeError(
            "The Categories and Pages tab lists pages rather than files. "
            "Open a category it found and download from there"
        )
    parts = [FILETYPES.get(kind, FILETYPES["image"])]
    if kind != "audio":
        parts.append("-fileres:0")
    for name in KEYWORDS:
        if value := given(name):
            parts.append(f"{name}:{_filter_value(name, value)}")
    if assessment := given("assessment"):
        if assessment not in ASSESSMENTS:
            raise RuntimeError(f"Commons has no assessment filter called {assessment!r}")
        parts.append(ASSESSMENTS[assessment])
    parts.append(words)
    return {
        "generator": "search",
        "gsrsearch": " ".join(parts),
        "gsrnamespace": str(FILE),
        "gsrlimit": str(PER_PAGE),
        "gsroffset": str((max(step.page, 1) - 1) * PER_PAGE),
        "gsrsort": RECENCY if given("sort") == "recency" else "relevance",
        "gsrinfo": "totalhits",
    }


def _filter_value(name, value):
    """A filter's value as the query can take it, or a refusal that names it."""
    if not _FILTER_VALUE.fullmatch(value):
        raise RuntimeError(f"Commons filter {name} has a value this search cannot use: {value!r}")
    return value


def _refused(refused, engine):
    """The API's own error: the end of a search at the site's ceiling, or a refusal."""
    code = str(refused.get("code") or "") if isinstance(refused, dict) else ""
    if code == TOO_FAR:
        # Handled, and nothing came of it: the nodes do not run, so no next page is queued.
        engine.log(f"The search ends here: Commons gives no more than {SEARCH_CEILING:,} files")
        return True
    info = str(refused.get("info") or "") if isinstance(refused, dict) else str(refused)
    raise RuntimeError(f"Commons refused this step: {code or 'error'} ({info})")


def _say_the_ceiling(step, said, engine):
    """One line on the first page of a search that finds more than the site will give.

    **A search for «tiger» reports 135 575 files and yields ten thousand.** Without a word the
    run looks complete at 9 761, and the person has no way to know the rest was never on
    offer; the number was being asked for (`gsrinfo=totalhits`) and not read.
    """
    if max(step.page, 1) != 1:
        return
    total = (said.get("query") or {}).get("searchinfo", {}).get("totalhits")
    if isinstance(total, int) and total > SEARCH_CEILING:
        engine.log(f"Commons finds {total:,} files for this search and gives the first {SEARCH_CEILING:,}")


def _in_site_order(rows, engine):
    """The answer as the site orders it: a search by rank, uploads by time, a category by name.

    **None of them is the order of the answer itself.** A generator's pages come in page-id
    order: `Category:Tigers in art` as answered matched the site's order in none of its 277
    places, and even with `gaidir=descending` the uploads came May, July, April (measured
    2026-09-22). A category sorts by name and puts its subcategories first; by the name alone
    that matches the site in 262 of the 277 places of `Category:Tigers in art` and in all 38 of
    `Category:Panthera tigris altaica`. The misses are pages filed under a sort key of their
    own, which a generator does not report - common among subcategories, rare among files.
    """
    if any("index" in one for one in rows):
        return sorted(rows, key=lambda one: one.get("index") or 0)
    if engine.caught("who"):
        return sorted(rows, key=_uploaded, reverse=True)
    return sorted(rows, key=lambda one: (one.get("ns") != CATEGORY, _by_name(one)))


def _uploaded(row):
    """The moment the file's current version arrived, which is what `Special:ListFiles` sorts by."""
    return str(((row.get("imageinfo") or [{}])[0]).get("timestamp") or "")


def _by_name(row):
    """A name compared the way Commons compares it: upper-cased, letter by letter.

    The site says so itself - `categorycollation: uppercase` in `meta=siteinfo` - and it is
    not the numeric collation of the larger Wikipedias: `043Aa.` comes before `1897` and `24h`
    after both, which reading numbers by value would reverse.
    """
    return str(row.get("title") or "").split(":", 1)[-1].upper()


def _walk_into(row, level, engine):
    """A subcategory as a step of its own, its files under a folder named after it.

    **Commons keeps its content in subcategories**, so a run that stopped at the category it
    was given would come back empty and look like a success: `Category:Felidae` holds no files
    at all and thirty-four subcategories, and `Category:Panthera tigris` holds 139 files and
    29 subcategories whose own files number 1246 (measured 2026-09-22).
    """
    title = str(row.get("title") or "")
    name = title.split(":", 1)[-1] if ":" in title else title
    engine.follow_up(
        f"{SITE}/wiki/{quote(title.replace(' ', '_'), safe=':/')}",
        payload={"level": level + 1},
        segment=name,
    )


def _announce(row, engine):
    """One file as the site holds it: the original, named as the site names it."""
    image = (row.get("imageinfo") or [{}])[0]
    address = str(image.get("url") or "")
    if not address:
        return
    facts = image.get("extmetadata") or {}
    title = str(row.get("title") or "")
    meta = {
        # The author as the file page credits them, and the licence it names - the two facts
        # somebody downloading from Commons has to carry away with the file.
        "author": _plain(_said(facts, "Artist")) or str(image.get("user") or ""),
        "license": _plain(_said(facts, "LicenseShortName")),
        "uploaded": str(image.get("timestamp") or "")[:10],
        "taken": _day(_plain(_said(facts, "DateTimeOriginal"))),
        "uploader": str(image.get("user") or ""),
        # Sound reports a width and height of zero, which the filter below drops, and a length
        # to the hundred-trillionth of a second (`149.99510204081633`).
        "width": image.get("width"),
        "height": image.get("height"),
        "duration": round(image["duration"], 1) if isinstance(image.get("duration"), int | float) else None,
    }
    engine.download(
        # The API decorates the address with its own tracking; the file is the same without it.
        address.split("?")[0],
        # The title without its namespace is what the site calls the file, extension and all -
        # and the namer knows not to add a second one (`nd_sdk.transfer.naming`).
        name=title.split(":", 1)[-1] if title.lower().startswith("file:") else title,
        description=_plain(_said(facts, "ImageDescription")),
        page=str(image.get("descriptionurl") or ""),
        # **No `expect`.** It guards against a page served in place of a file, and the address
        # here is the file's own, from the API; what it could do is drop a real file whose host
        # names its type differently from the API, silently, as filtered rather than failed.
        headers=NAMED,
        meta={name: value for name, value in meta.items() if value not in (None, "", 0)},
    )


def _day(text):
    """The day a date names, as `YYYY-MM-DD`, or a year (and month) standing alone; nothing
    otherwise.

    **Better absent than wrong.** Commons writes a date as a person would - "Taken on 1
    November 2008, 13:02:01", "17 October 2004 (according to Exif data)", "circa 1895",
    "1917 – 1918" - and cutting the text at ten characters made "Taken on 1" and "1917 – 191"
    out of a sixth of them (34 of 488 for «tiger», 15 of 56 among the Met's, measured
    2026-09-22). A range, an "after" or a "circa" is not a day, and is left out.
    """
    if found := _ISO_DAY.search(text):
        return _checked(*(int(part) for part in found.groups()))
    if found := _DAY_MONTH_YEAR.search(text):
        day, month, year = found.groups()
        if month.lower() in _MONTHS:
            return _checked(int(year), _MONTHS[month.lower()], int(day))
    if found := _MONTH_DAY_YEAR.search(text):
        month, day, year = found.groups()
        if month.lower() in _MONTHS:
            return _checked(int(year), _MONTHS[month.lower()], int(day))
    return text if _YEAR.fullmatch(text) else ""


def _checked(year, month, day):
    """A day that exists, written the one way, or nothing."""
    if 1 <= month <= 12 and 1 <= day <= 31:
        return f"{year:04d}-{month:02d}-{day:02d}"
    return ""


def _said(facts, name):
    """One field of `extmetadata`, which wraps every value in an object of its own."""
    field = facts.get(name)
    return str((field or {}).get("value") or "") if isinstance(field, dict) else ""


def _plain(text):
    """The markup Commons writes into its metadata, as words.

    Descriptions and credits arrive as HTML - a link around the author's name, a `<span>`
    around "Own work", a hidden machine-readable copy of a date - and a file's description is
    text.
    """
    return _SPACES.sub(" ", _TAGS.sub(" ", unescape(_HIDDEN.sub(" ", text)))).strip()


def _caught(engine, name):
    """What the node caught out of the address, percent-decoded and readable.

    Commons writes its titles into addresses with underscores for spaces and percent escapes
    for everything else; the API wants them back as they are written on the page.
    """
    value = engine.caught(name)
    return unquote(str(value)).replace("_", " ").strip() if value else ""


def _whole(value):
    """A whole number out of whatever the parameter or the payload holds, or zero."""
    try:
        return int(str(value))
    except (TypeError, ValueError):
        return 0
