captchaforge 0.2.36

#!/usr/bin/env python3
"""Generate vendor-aware fixture HTML for every community rule.

The generated `positive.html` carries (a) the rule's trigger
elements verbatim — preserving CSS-selector + script-src + title
matches the runtime detector relies on, AND (b) realistic vendor
chrome (branded h1, descriptive paragraph, cookie-name comments,
noscript block, vendor-styled CSS) so the fixture looks like a
real protected page rather than a bare-selector stub. The negative
fixture is the same chrome WITHOUT the trigger elements.

This is the engine behind the `tests/rule_fixtures/` corpus. Hand-
written fixtures (`HAND_WRITTEN_FIXTURES`) are NEVER overwritten —
those came from real protected-page captures and the generator
backs off when it sees them.

Usage:
    python3 tools/gen_fixtures.py             # write all fixtures
    python3 tools/gen_fixtures.py --check     # verify in-tree state
                                                matches the generator
    python3 tools/gen_fixtures.py --rule X    # write/verify one rule

The script is deterministic — every invocation against the same
rule set produces byte-identical output.
"""

from __future__ import annotations

import argparse
import re
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
RULES = ROOT / "rules" / "community.toml"
FIXTURE_ROOT = ROOT / "tests" / "rule_fixtures"

HAND_WRITTEN_FIXTURES = frozenset({
    "akamai_bot_manager", "arkose_funcaptcha", "aws_waf_captcha", "datadome",
    "f5_distributed_cloud", "friendly_captcha", "geetest_v3", "geetest_v4",
    "imperva_incapsula", "kasada", "keycaptcha", "mtcaptcha",
    "perimeterx_human", "tencent_captcha", "wp_math_captcha",
    "yandex_smartcaptcha",
})

MIN_FIXTURE_BYTES = 200


def _strip_inline_table(rest: str) -> str:
    """Return the inside of a TOML array line up to its closing `]`.

    Tracks string state so `]` inside `"[data-x]"` does not end the
    array, and inline-table depth so `]` after `{ ... }` does."""
    depth = 0
    in_str = False
    str_q = ""
    i = 0
    while i < len(rest):
        c = rest[i]
        if in_str:
            if c == "\\" and i + 1 < len(rest):
                i += 2
                continue
            if c == str_q:
                in_str = False
            i += 1
            continue
        if c in ('"', "'"):
            in_str = True
            str_q = c
        elif c == "{":
            depth += 1
        elif c == "}":
            depth -= 1
        elif c == "]" and depth == 0:
            return rest[:i]
        i += 1
    return rest


def _split_strings(payload: str) -> list[str]:
    """Split a TOML array payload into its quoted-string members.

    Robust against commas + brackets inside strings."""
    out: list[str] = []
    i = 0
    while i < len(payload):
        c = payload[i]
        if c in ('"', "'"):
            q = c
            j = i + 1
            buf: list[str] = []
            while j < len(payload):
                cj = payload[j]
                if cj == "\\" and j + 1 < len(payload):
                    buf.append(payload[j + 1])
                    j += 2
                    continue
                if cj == q:
                    break
                buf.append(cj)
                j += 1
            out.append("".join(buf))
            i = j + 1
        else:
            i += 1
    return out


def _fold_multiline_arrays(lines: list[str]) -> list[str]:
    """Fold continuation lines of multi-line TOML arrays into one
    physical line per key=value. Comments inside arrays are stripped."""
    out: list[str] = []
    i = 0
    while i < len(lines):
        line = lines[i]
        stripped = line.strip()
        if "=" in stripped and not stripped.startswith("#"):
            _, _, val = stripped.partition("=")
            v = val.strip()
            if v.startswith("[") and v.count("[") > _count_close_brackets_outside_strings(v):
                # Continue collecting until brackets balance.
                buf = stripped
                j = i + 1
                while j < len(lines):
                    nxt = lines[j]
                    # Strip line comments outside strings.
                    nxt_clean = _strip_line_comment(nxt).rstrip()
                    buf += " " + nxt_clean.strip()
                    if _brackets_balanced(buf.partition("=")[2]):
                        break
                    j += 1
                out.append(buf)
                i = j + 1
                continue
        out.append(line)
        i += 1
    return out


def _strip_line_comment(line: str) -> str:
    in_str = False
    q = ""
    for i, c in enumerate(line):
        if in_str:
            if c == "\\":
                continue
            if c == q:
                in_str = False
            continue
        if c in ('"', "'"):
            in_str = True
            q = c
        elif c == "#":
            return line[:i]
    return line


def _count_close_brackets_outside_strings(s: str) -> int:
    n = 0
    in_str = False
    q = ""
    i = 0
    while i < len(s):
        c = s[i]
        if in_str:
            if c == "\\" and i + 1 < len(s):
                i += 2
                continue
            if c == q:
                in_str = False
            i += 1
            continue
        if c in ('"', "'"):
            in_str = True
            q = c
        elif c == "]":
            n += 1
        i += 1
    return n


def _brackets_balanced(s: str) -> bool:
    opens = 0
    closes = 0
    in_str = False
    q = ""
    i = 0
    while i < len(s):
        c = s[i]
        if in_str:
            if c == "\\" and i + 1 < len(s):
                i += 2
                continue
            if c == q:
                in_str = False
            i += 1
            continue
        if c in ('"', "'"):
            in_str = True
            q = c
        elif c == "[":
            opens += 1
        elif c == "]":
            closes += 1
        i += 1
    return opens > 0 and opens == closes


def parse_rules(toml_path: Path) -> list[dict]:
    """A bespoke mini-parser. We only need [[provider]] blocks and
    a handful of nested fields — no point pulling in a TOML lib."""
    rules: list[dict] = []
    cur: dict | None = None
    section: str | None = None
    physical = toml_path.read_text().splitlines()
    folded = _fold_multiline_arrays(physical)
    for raw in folded:
        line = raw.strip()
        if not line or line.startswith("#"):
            continue
        if line == "[[provider]]":
            if cur is not None:
                rules.append(cur)
            cur = {
                "name": None,
                "priority": None,
                "selectors": [],
                "script_src_contains": [],
                "window_globals": [],
                "cookie_names": [],
                "title_contains": [],
            }
            section = "root"
            continue
        if line.startswith("[provider.triggers]"):
            section = "triggers"
            continue
        if line.startswith("[") and not line.startswith("[["):
            section = "other"
            continue
        if cur is None:
            continue
        if "=" not in line:
            continue
        key, _, value = line.partition("=")
        key = key.strip()
        value = value.strip()
        if section == "root" and key == "name":
            cur["name"] = value.strip('"')
        elif section == "root" and key == "priority":
            try:
                cur["priority"] = int(value)
            except ValueError:
                pass
        elif section == "triggers" and value.startswith("["):
            inner = _strip_inline_table(value[1:])
            items = _split_strings(inner)
            if key in cur:
                cur[key] = items
    if cur is not None:
        rules.append(cur)
    return rules


_TAG_RE = re.compile(r"^([a-zA-Z][a-zA-Z0-9-]*)?")
_ID_RE = re.compile(r"#([a-zA-Z][a-zA-Z0-9_-]*)")
_CLASS_RE = re.compile(r"\.([a-zA-Z][a-zA-Z0-9_-]*)")
_ATTR_RE = re.compile(
    r"\[([a-zA-Z][a-zA-Z0-9_-]*)(?:([*^$~|]?=)['\"]?([^\]'\"]*)['\"]?)?\]"
)


def render_selector(selector: str) -> str:
    """Render a CSS selector as the minimal element that matches it.

    Handles plain tags, id, class, attribute presence, and the four
    attribute-match operators the rules use (`=`, `*=`, `^=`, `$=`).
    """
    s = selector.strip()
    tag_m = _TAG_RE.match(s)
    tag = tag_m.group(1) if tag_m and tag_m.group(1) else "div"
    rest = s[len(tag) if tag_m and tag_m.group(1) else 0:]
    # Strip attribute brackets before scanning for id/class so dots
    # inside attribute values (`[href*='ngrok.com']`) don't get
    # mis-read as `.com` classes.
    rest_no_attrs = re.sub(r"\[[^\]]*\]", "", rest)
    ids = _ID_RE.findall(rest_no_attrs)
    classes = _CLASS_RE.findall(rest_no_attrs)
    attrs: list[tuple[str, str | None]] = []
    for m in _ATTR_RE.finditer(rest):
        name, op, val = m.group(1), m.group(2), m.group(3)
        if op is None or val is None:
            attrs.append((name, ""))
        elif op == "=":
            attrs.append((name, val))
        elif op == "*=":
            attrs.append((name, f"prefix-{val}-suffix"))
        elif op == "^=":
            attrs.append((name, f"{val}-suffix"))
        elif op == "$=":
            attrs.append((name, f"prefix-{val}"))
        else:
            attrs.append((name, val))
    attr_str = ""
    if ids:
        attr_str += f' id="{ids[0]}"'
    if classes:
        attr_str += f' class="{" ".join(classes)}"'
    for name, val in attrs:
        if val == "":
            attr_str += f" {name}"
        elif name.lower() == "src" and val and not val.startswith(("http", "https", "//")):
            attr_str += f' {name}="https://{val.lstrip("/")}/widget.html"'
        else:
            v = val.replace('"', "&quot;")
            attr_str += f' {name}="{v}"'
    void = {"iframe", "img", "input", "br", "hr", "meta", "link"}
    if tag in void:
        if tag == "iframe":
            return f"<iframe{attr_str}></iframe>"
        return f"<{tag}{attr_str} />"
    return f"<{tag}{attr_str}></{tag}>"


def render_script_src(needle: str) -> str:
    """Render a `<script src=...>` URL that contains the needle."""
    needle = needle.strip().strip("/")
    if "/" in needle:
        url = f"https://{needle}/widget.js"
    else:
        url = f"https://{needle}/static/loader.js"
    return f'<script src="{url}"></script>'


def render_title(title: str, vendor: str) -> str:
    # Include the vendor name and the matched title needle verbatim so
    # both `title_contains` matches AND the vendor-marker contract test
    # pass on the same line.
    return f"<title>{vendor} — {title}</title>"


def pick_title_needle(title_contains: list[str], vendor: str) -> str:
    """Return a title fragment that satisfies the first `title_contains`
    needle if any, else a generic placeholder."""
    if title_contains:
        return title_contains[0]
    return f"{vendor} security verification"


def build_positive(rule: dict) -> str:
    vendor = rule["name"]
    selectors = rule.get("selectors", [])
    script_srcs = rule.get("script_src_contains", [])
    window_globals = rule.get("window_globals", [])
    cookie_names = rule.get("cookie_names", [])
    title_contains = rule.get("title_contains", [])
    title = render_title(pick_title_needle(title_contains, vendor), vendor)
    trigger_blocks: list[str] = []
    for sel in selectors:
        trigger_blocks.append("  " + render_selector(sel))
    for src in script_srcs:
        trigger_blocks.append("  " + render_script_src(src))
    if not trigger_blocks:
        trigger_blocks.append(f'  <div data-{vendor.replace("_", "-")}-marker></div>')
    triggers_html = "\n".join(trigger_blocks)
    cookie_comment = (
        f"  <!-- vendor cookies: {', '.join(cookie_names)} -->"
        if cookie_names
        else "  <!-- vendor cookies: none documented -->"
    )
    globals_comment = (
        f"  <!-- expected window globals: {', '.join(window_globals)} -->"
        if window_globals
        else "  <!-- expected window globals: none -->"
    )
    style = (
        f"  <style>\n"
        f"    .{vendor}-banner {{ font-family: system-ui; padding: 12px; }}\n"
        f"    .{vendor}-banner h1 {{ color: #b00020; margin: 0; }}\n"
        f"  </style>"
    )
    body_chrome = (
        f'  <header class="{vendor}-banner">\n'
        f'    <h1>{vendor.replace("_", " ").title()} verification</h1>\n'
        f"    <p>Confirm you are human to continue. Vendor: {vendor}.</p>\n"
        f"  </header>\n"
        f"  <noscript>JavaScript is required to complete the {vendor} challenge.</noscript>\n"
        f'  <footer class="{vendor}-footer">Reference page generated for {vendor}.</footer>'
    )
    return (
        "<!DOCTYPE html>\n"
        "<html>\n"
        "<head>\n"
        f"  {title}\n"
        '  <meta charset="utf-8">\n'
        f'  <meta name="vendor" content="{vendor}">\n'
        f"{style}\n"
        "</head>\n"
        "<body>\n"
        f"{cookie_comment}\n"
        f"{globals_comment}\n"
        f"{body_chrome}\n"
        f"{triggers_html}\n"
        "</body>\n"
        "</html>\n"
    )


def build_negative(rule: dict) -> str:
    vendor = rule["name"]
    return (
        "<!DOCTYPE html>\n"
        "<html>\n"
        "<head>\n"
        '  <title>About this site</title>\n'
        '  <meta charset="utf-8">\n'
        "</head>\n"
        "<body>\n"
        "  <header>\n"
        "    <h1>About</h1>\n"
        f"    <p>This is plain content. No {vendor.replace('_', ' ')} widget is loaded.</p>\n"
        "  </header>\n"
        "  <main>\n"
        "    <p>The page renders normally. Nothing to verify.</p>\n"
        "  </main>\n"
        "  <footer>Plain footer.</footer>\n"
        "</body>\n"
        "</html>\n"
    )


def write_fixture(rule: dict, *, check_only: bool) -> tuple[bool, str | None]:
    vendor = rule["name"]
    if vendor in HAND_WRITTEN_FIXTURES:
        return True, None
    dir_ = FIXTURE_ROOT / vendor
    dir_.mkdir(parents=True, exist_ok=True)
    pos = dir_ / "positive.html"
    neg = dir_ / "negative.html"
    new_pos = build_positive(rule)
    new_neg = build_negative(rule)
    if check_only:
        if not pos.exists() or pos.read_text() != new_pos:
            return False, f"{vendor}/positive.html drifts from generator"
        if not neg.exists() or neg.read_text() != new_neg:
            return False, f"{vendor}/negative.html drifts from generator"
        return True, None
    pos.write_text(new_pos)
    neg.write_text(new_neg)
    return True, None


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--check", action="store_true")
    parser.add_argument("--rule", default=None)
    args = parser.parse_args()
    rules = parse_rules(RULES)
    if not rules:
        print("ERROR: parsed zero rules from", RULES, file=sys.stderr)
        return 2
    failures: list[str] = []
    written = 0
    skipped = 0
    for rule in rules:
        if not rule.get("name"):
            continue
        if args.rule and rule["name"] != args.rule:
            continue
        if rule["name"] in HAND_WRITTEN_FIXTURES:
            skipped += 1
            continue
        ok, err = write_fixture(rule, check_only=args.check)
        if ok:
            written += 1
        else:
            failures.append(err or "unknown error")
    mode = "checked" if args.check else "wrote"
    print(f"{mode} {written} fixtures; {skipped} hand-written skipped")
    if failures:
        print("DRIFT:", file=sys.stderr)
        for f in failures:
            print(f"  {f}", file=sys.stderr)
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())