#!/usr/bin/env python3
"""packnorm — THE ONE NORMALISER, and what a figure is.

Reference implementation of SPEC-R0A1-PACKS.md §3.0. `pack.py`, `pack_gate.py`,
the judge and R1 all cut text the same way or they disagree about what the page
says; this module is the single place that decides.

Nothing here calls a model, reads the network, or looks at a file. It is pure
text, so it is testable, and `test_packnorm.py` beside it tests it.

THE TWO PLACES THE SPEC LEAVES ROOM, both pinned here rather than guessed at a
call site (each is named in the bench report so the operator can rule):

  * "Excluded as figures: a bare 4-digit year" — a bare four-digit integer is
    not necessarily a year, so the exclusion is pinned to 1900-2099
    (`_YEAR_RANGE`). `1500` (a VA rating) stays a figure; `2026` does not.
  * "ends in a figure" is defined on "the bullet's last whitespace-delimited
    token", but three of the spec's own five examples (`71.4 %`, `138 W`, and
    `1.9×` when spaced) put the unit in a SECOND token. So the terminal token
    may be a UNIT TAIL — a digitless final token of at most four characters
    drawn from `%°×/A-Za-z` — in which case the figure token is the pair. Both
    readings are computed: `ends_in_figure()` uses the unit-tail rule,
    `ends_in_figure_strict()` is the literal one, and the bench reports both.
"""

from __future__ import annotations

import html
import os
import re
import unicodedata
from dataclasses import dataclass

VERSION = "packnorm-1.0"

# ---------------------------------------------------------------- normalise

_ANCHOR_TRAILER = re.compile(r"\s*\{#[A-Za-z0-9_-]+\}\s*$", re.MULTILINE)
_HEADING_MARKER = re.compile(r"^[ \t]{0,3}#{1,6}[ \t]+", re.MULTILINE)
_QUOTE_MARKER = re.compile(r"^[ \t]{0,3}(?:>[ \t]?)+", re.MULTILINE)
_LIST_MARKER = re.compile(r"^[ \t]{0,3}(?:[-*+][ \t]+|[0-9]+[.)][ \t]+)", re.MULTILINE)
_MD_LINK = re.compile(r"!?\[([^\]]*)\]\(([^)]*)\)")
_EMPHASIS = re.compile(r"(\*{1,3}|_{1,3}|`{1,3}|~~)")
# a pipe-table delimiter row: | --- | :---: | ---: |
_TABLE_DELIM_ROW = re.compile(r"^[ \t]{0,3}\|?[ \t]*:?-{2,}:?[ \t]*(\|[ \t]*:?-{2,}:?[ \t]*)*\|?[ \t]*$",
                              re.MULTILINE)
_ESCAPED_PUNCT = re.compile(r"\\([\\`*_{}\[\]()#+\-.!|~>])")

_DASHES = "‐‑‒–—―−"
_SQUOTES = "‘’‚‛′"
_DQUOTES = "“”„‟″"
# Zs is folded by category at run time; these are the named extras the spec lists.
_SPACEY = "  \t"

_WS_RUN = re.compile(r"[ \t]+")
_BLANK_RUN = re.compile(r"\n{3,}")


def _fold_char(ch: str) -> str:
    if ch in _SPACEY or unicodedata.category(ch) == "Zs":
        return " "
    if ch in _DASHES:
        return "-"
    if ch in _SQUOTES:
        return "'"
    if ch in _DQUOTES:
        return '"'
    if ch == "…":
        return "..."
    return ch


def normalise(text: str, *, fold_case: bool = False) -> str:
    """SPEC §3.0's five steps, in order. NOT NFKC (it rewrites 24 of 39 twins).

    `× ≥ ≤ ≈` and superscripts are left exactly as written: they are part of a
    figure's unit and folding them would make a bullet's figure stop matching
    the article's.
    """
    t = text
    # (1) markdown furniture
    t = _ANCHOR_TRAILER.sub("", t)
    t = _TABLE_DELIM_ROW.sub("", t)          # before markers: a row is a whole line
    t = _HEADING_MARKER.sub("", t)
    t = _QUOTE_MARKER.sub("", t)
    t = _LIST_MARKER.sub("", t)
    t = _MD_LINK.sub(r"\1", t)
    t = _EMPHASIS.sub("", t)
    t = _ESCAPED_PUNCT.sub(r"\1", t)
    # (2) pipe tables — the delimiter rows are already gone; the bars become space
    t = t.replace("|", " ")
    # (3) HTML entities
    t = html.unescape(t)
    # (4) the character folds
    t = "".join(_fold_char(c) for c in t)
    # (5) collapse whitespace
    t = _WS_RUN.sub(" ", t)
    t = "\n".join(line.strip() for line in t.split("\n"))
    t = _BLANK_RUN.sub("\n\n", t)
    t = t.strip()
    return t.casefold() if fold_case else t


def flat(text: str, *, fold_case: bool = False) -> str:
    """`normalise` with newlines folded to spaces too.

    The substring checks ("is this token in the cited span?", "is this quote
    verbatim?") must not fail because a sentence wrapped across two source
    lines, so both sides of every such comparison go through this.
    """
    t = normalise(text, fold_case=fold_case)
    return re.sub(r"\s+", " ", t.replace("\n", " ")).strip()


# ------------------------------------------------- WHICH DEFINITION OF FIGURE

#: TWO DEFINITIONS OF "A FIGURE", AND WHICH ONE IS LIVE IS A RULING, NOT A TASTE.
#: **`v3` IS THE RULED DEFINITION** — D-20260908-40, ruled by the workshop 2026-09-09 —
#: and it is the default from that date.
#:
#: `v3`: a quantity token — ASCII DIGITS, or a NUMBER WORD (one…ninety-nine,
#: hundred, thousand, and the hyphenated compounds) — followed by up to three
#: UNIT WORDS, with the WHOLE span verbatim in the cited section after
#: normalisation. Bare years and bare-year ranges stay excluded. It was ruled on
#: measurement: the follow-up bench found 36 of 111 composed bullets ending in a
#: figure whose unit is a WORD (`56.9 milliseconds`, `4 minutes 49 seconds`,
#: `44 million parameters`) — each verbatim in its cited span and each rejected
#: by v2's four-character cap — and 20 more ending in a quantity the house voice
#: writes in words (`forty-eight in one read`, `seventy-two hours`) that a
#: `[0-9]`-only grammar cannot see at all. Isolated on its own in the 2026-09-09
#: re-run, the definition alone was worth +6 bullets and +3 pages.
#:
#: `v2` is SPEC §3.0 exactly as written and as this file was lifted: ASCII digits
#: only, with a unit TAIL of at most four characters from `%°×/A-Za-z`. It is no
#: longer the default. It stays SELECTABLE because the suites score both
#: readings against each other, and because a definition nobody can still run is
#: a definition nobody can check.
#:
#: `DEFINITION` is read at call time, the environment can flip it for a bench
#: (`S2S_PACK_FIGURE_DEF=v2`), every `pack.json` records which one wrote it, and
#: `pack_gate` judges a committed pack by the definition THE PACK records — so a
#: page always says what rule its lines were held to, and the ambient setting
#: can never re-judge a pack behind its own back.
DEFINITIONS = ("v2", "v3")
DEFINITION = os.environ.get("S2S_PACK_FIGURE_DEF", "v3")
if DEFINITION not in DEFINITIONS:                   # fail closed, never silently
    raise RuntimeError("packnorm: S2S_PACK_FIGURE_DEF=%r is not one of %s"
                       % (DEFINITION, list(DEFINITIONS)))


def _definition(explicit: str | None) -> str:
    d = explicit or DEFINITION
    if d not in DEFINITIONS:
        raise ValueError("packnorm: unknown figure definition %r" % (d,))
    return d


#: The number words v3 admits. One to ninety-nine (spelled or hyphenated), plus
#: the two scale words the house voice actually uses. `one` is excluded from
#: being a figure for the same reason a bare `1` is.
_UNITS_WORDS = ("one two three four five six seven eight nine ten eleven twelve "
                "thirteen fourteen fifteen sixteen seventeen eighteen nineteen").split()
_TENS_WORDS = "twenty thirty forty fifty sixty seventy eighty ninety".split()
_SCALE_WORDS = ("hundred", "thousand", "million", "billion")
NUMBER_WORDS = frozenset(_UNITS_WORDS + _TENS_WORDS + list(_SCALE_WORDS) + [
    "%s-%s" % (t, u) for t in _TENS_WORDS for u in _UNITS_WORDS[:9]])

#: v3's unit tail: up to three DIGITLESS words after the number. A word, not a
#: symbol, and length-free — `milliseconds` is as legitimate a unit as `%`.
V3_MAX_UNIT_WORDS = 3

#: And the whole terminal span stays a FIGURE rather than a sentence.
V3_MAX_SPAN_TOKENS = 6
_WORDY = re.compile(r"^[A-Za-z][A-Za-z\-]*$|^[%°×/]{1,2}$")


def is_number_word(tok: str) -> bool:
    t = tok.lower().strip(".,")
    return t in NUMBER_WORDS and t != "one"


# ------------------------------------------------------------------ figures

# THE ORDER IS THE SPEC'S. Python alternation is leftmost-first at each start
# position, so listing them in this order gives the earlier pattern precedence
# exactly as "the first match in order of" requires.
_FIGURE_ALTERNATIVES = (
    r"[0-9]{4}-[0-9]{2}-[0-9]{2}",
    r"[0-9]+:[0-9]{2}(?::[0-9]{2})?",
    r"[0-9]+-for-[0-9]+",
    r"[0-9]+/[0-9]+",
    r"[0-9]+-[0-9]+",
    r"[0-9]{1,3}(?:,[0-9]{3})+",
    r"[0-9]+(?:\.[0-9]+)?",
)
FIGURE_RE = re.compile("|".join(_FIGURE_ALTERNATIVES))   # [0-9], never \d

_TRAILING_PUNCT = ".,"
_YEAR_RANGE = (1900, 2099)
_BARE_YEAR = re.compile(r"^[0-9]{4}$")


def _strip_trailing(fig: str) -> str:
    return fig.rstrip(_TRAILING_PUNCT)


#: §9.5's second ruling: a bare-year RANGE is excluded exactly like a bare year.
#: `2019-2021` is a span of time, not a measurement, and a line ending in one is
#: not a receipt. It was pinned in the addendum and unimplemented until the
#: 2026-09-08 audit named it.
_BARE_YEAR_RANGE = re.compile(r"^([0-9]{4})-([0-9]{4})$")


def is_excluded_figure(fig: str) -> bool:
    """A bare 4-digit year, a bare year RANGE, a bare `0`, a bare `1`."""
    if fig in ("0", "1"):
        return True
    if _BARE_YEAR.match(fig):
        return _YEAR_RANGE[0] <= int(fig) <= _YEAR_RANGE[1]
    m = _BARE_YEAR_RANGE.match(fig)
    if m:
        return all(_YEAR_RANGE[0] <= int(g) <= _YEAR_RANGE[1] for g in m.groups())
    return False


def figures(text: str, *, already_normalised: bool = False) -> list[str]:
    """Every figure in `text`, in order, exclusions removed.

    Duplicates are kept: `40 of 56` is two figures and the gate checks both.
    """
    t = text if already_normalised else flat(text)
    out = []
    for m in FIGURE_RE.finditer(t):
        fig = _strip_trailing(m.group(0))
        if fig and not is_excluded_figure(fig):
            out.append(fig)
    return out


def has_figure(token: str) -> bool:
    return bool(figures(token))


# --------------------------------------------------- ends in a figure, twice

_UNIT_TAIL = re.compile(r"^[%°×/A-Za-z]{1,4}$")


@dataclass(frozen=True)
class EndsInFigure:
    ok: bool
    token: str          # the figure-bearing terminal token, as checked
    figure: str         # the figure inside it ("" when there is none)
    in_span: bool       # was that whole token present in the cited span?
    used_unit_tail: bool
    reason: str


def _terminal_token(claim: str) -> tuple[str, bool]:
    """The claim's terminal figure token, and whether a unit tail was absorbed."""
    toks = flat(claim).rstrip(_TRAILING_PUNCT).split(" ")
    toks = [t for t in toks if t]
    if not toks:
        return "", False
    last = toks[-1].rstrip(_TRAILING_PUNCT)
    if has_figure(last):
        return last, False
    if len(toks) >= 2 and _UNIT_TAIL.match(last) and has_figure(toks[-2]):
        return f"{toks[-2].rstrip(_TRAILING_PUNCT)} {last}", True
    return last, False


def _terminal_token_v3(claim: str, span_text: str = "") -> tuple[str, bool]:
    """v3's terminal figure span: the LONGEST trailing run of at most
    `V3_MAX_SPAN_TOKENS` tokens that starts at a number (ASCII digits or a
    number word) and is VERBATIM in the cited section.

    Longest-verbatim is what makes the compound quantities this corpus writes
    come back whole — `4 minutes 49 seconds`, `44 million parameters`,
    `forty-eight in one read` — without a grammar for each of them. The verbatim
    requirement is the whole safety property: a span the article does not
    contain, character for character, never wins, however well it scans.

    With no span to check against (a caller that only wants the shape), the
    shortest number-anchored suffix comes back.
    """
    toks = [t for t in flat(claim).rstrip(_TRAILING_PUNCT).split(" ") if t]
    if not toks:
        return "", False
    hay = flat(span_text) if span_text else ""
    lo = max(0, len(toks) - V3_MAX_SPAN_TOKENS)
    shortest = ""
    for i in range(lo, len(toks)):
        head = toks[i].rstrip(_TRAILING_PUNCT)
        if not (has_figure(head) or is_number_word(head)):
            continue
        # A BARE SCALE WORD CANNOT ANCHOR A SPAN. `million parameters` is
        # verbatim in an article that says `44 million parameters`, so a claim
        # of `45 million parameters` would otherwise pass on the tail alone —
        # the wrong number, hidden behind a true phrase. A scale word only
        # counts when the number in front of it is part of the span.
        if head.lower().strip(".,") in _SCALE_WORDS and i > 0:
            prev = toks[i - 1].rstrip(_TRAILING_PUNCT)
            if has_figure(prev) or is_number_word(prev):
                # The span starting at `prev` was already offered, and the loop
                # runs longest-first, so if it did not match verbatim then this
                # shorter one must not be allowed to stand in for it: the
                # article says `44 million parameters` and the claim says `45`.
                continue
        cand = " ".join([head] + [t.rstrip(_TRAILING_PUNCT) for t in toks[i + 1:]])
        units = [t for t in toks[i + 1:] if not has_figure(t)]
        if len(units) > V3_MAX_UNIT_WORDS:
            continue
        if not shortest or len(cand) < len(shortest):
            shortest = cand
        if hay and flat(cand) in hay:
            return cand, len(toks) - 1 > i
    return (shortest or toks[-1].rstrip(_TRAILING_PUNCT)), False


def ends_in_figure(claim: str, span_text: str, *,
                   definition: str | None = None) -> EndsInFigure:
    """SPEC §3.0's rule under whichever definition is live (see `DEFINITION`).

    v2 is the spec as written: digits, with a unit tail of at most four
    characters. v3 is D-20260908-40 as proposed: digits or a number word, with
    up to three unit WORDS. Both readings end at the same place — the whole
    terminal span must appear, character for character, in the cited section.
    """
    d = _definition(definition)
    token, tail = (_terminal_token_v3(claim, span_text) if d == "v3"
                   else _terminal_token(claim))
    if not token:
        return EndsInFigure(False, "", "", False, tail, "empty")
    figs = figures(token)
    head = token.split(" ")[0]
    if not figs and not (d == "v3" and is_number_word(head)):
        return EndsInFigure(False, token, "", False, tail, "terminal token carries no figure")
    present = flat(token) in flat(span_text)
    return EndsInFigure(present, token, (figs[-1] if figs else head), present, tail,
                        "" if present else "terminal token not verbatim in the cited span")


def ends_in_figure_strict(claim: str, span_text: str) -> EndsInFigure:
    """The literal reading: the LAST whitespace-delimited token, no unit tail."""
    toks = [t for t in flat(claim).split(" ") if t]
    if not toks:
        return EndsInFigure(False, "", "", False, False, "empty")
    last = toks[-1].rstrip(_TRAILING_PUNCT)
    figs = figures(last)
    if not figs:
        return EndsInFigure(False, last, "", False, False, "last token carries no figure")
    present = flat(last) in flat(span_text)
    return EndsInFigure(present, last, figs[-1], present, False,
                        "" if present else "last token not verbatim in the cited span")


def figures_all_present(claim: str, span_text: str) -> tuple[bool, list[str]]:
    """Every figure in `claim` present verbatim in `span_text`. Returns the misses."""
    span = flat(span_text)
    missing = [f for f in figures(claim) if f not in span]
    return (not missing), missing


def quote_is_verbatim(quote: str, span_text: str) -> bool:
    """The judge's quote, checked case-folded per §3.0 step (5)."""
    q = flat(quote, fold_case=True)
    return bool(q) and q in flat(span_text, fold_case=True)


def word_count(text: str) -> int:
    return len([w for w in flat(text).split(" ") if w])