retch-cli 0.18.4

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
# Copyright (C) 2026 Ken Tobias
"""
Updates WIP.md after merging a feature branch to main.

WIP.md is an ongoing, rolling log written by hand. This script owns exactly ONE block of it:
the lines between the two marker comments below, which it regenerates from git and Cargo.toml
on every `just merge-pr`. Everything outside the markers is left byte-for-byte alone.

    <!-- BEGIN merge-pr state ... -->
    ...generated table: active branch, main HEAD, versions, newest tag, date...
    <!-- END merge-pr state -->

If the markers are absent the block is inserted once, at the top of the file. If they are
duplicated, unpaired or out of order the script refuses and changes nothing: guessing which
copy is "the real one" is exactly the failure this design replaces.

WHY A MARKED BLOCK, AND NOT THE OLD TWO SUBSTITUTIONS. The previous version rewrote the
first line anywhere in the file matching `### Active Branch:` and the first matching
`**main HEAD**:`. The file's structure moved on and the patterns did not:

- the live hand-off moved to a START HERE table and later to newest-first dated entries,
  none of which used either header, so both patterns matched STALE entries deep in the file
  (one ~500 lines down, the pointer ~4400 lines down) and rewrote those instead;
- it then printed "Active Branch set to none, main HEAD updated to ..." whether or not
  anything current had changed, so a no-op read as success;
- and any prose that QUOTED either header verbatim became the first match and was
  overwritten. That happened for real after #248, and WIP.md grew a standing rule never to
  quote the two strings - a rule about the text, patching a defect in the code.

A block the script owns outright cannot drift from the thing it describes, cannot be
confused with prose, and cannot be the wrong occurrence.

IT ALSO PRESERVES THE FILE'S LINE ENDINGS, AND THAT IS NOT INCIDENTAL - BUT THE REASON
CHANGED IN v0.18.1. `WIP.md` used to be deliberately CRLF, and this function existed to
defend that. It is now LF, like every other text file in the tree: **LF is the base model,
Windows is not**, and a survey of the whole three-repo fleet found retch's `WIP.md` was the
single CRLF file in any of them. So what this code defends now is the opposite decision -
it is what stops the conversion being undone silently by the next merge.

Preserving rather than hardcoding `\n` is still the right shape, for two reasons. It keeps
the script honest about what it found instead of imposing a house style on a file it does
not own; and a hardcoded `\n` would convert the file as a SIDE EFFECT of a merge, which is
precisely the accident v0.17.12 was about, just pointing the other way. The direction of an
accident does not make it deliberate. `--check-endings` is what asserts the decision.

An earlier version read with `read_text()` and wrote with `write_text()`. Both use
universal newlines, so `\r\n` became `\n` in memory and was written back as whatever the
PLATFORM prefers - which round-trips on Windows and silently converts the whole file on
Linux and macOS. That is why it went unnoticed: this repo's merges were historically cut
from a Windows host, and the first merge run from Linux converted all 4773 CRLF to LF.

A second trap, and the reason a naive "just read bytes" fix is not enough: in Python's
`re` a dot matches `\r` (it only excludes `\n`), so any `.*` run over un-normalised CRLF
text swallows the carriage return and the rewritten line comes back LF-terminated.
Normalising to `\n` before editing and re-applying the terminator on write avoids both.
"""

import datetime
import re
import subprocess
import sys
from pathlib import Path

BEGIN_MARKER = (
    "<!-- BEGIN merge-pr state: generated by scripts/update_wip.py on every `just merge-pr`."
    " Anything between these markers is overwritten; write notes outside them. -->"
)
END_MARKER = "<!-- END merge-pr state -->"

# Whole-line matches only. A marker mentioned mid-sentence is prose, not a marker; one that
# starts a line is treated as a marker even inside a code fence, and then the duplicate
# check refuses rather than guessing.
_BEGIN_RE = re.compile(r"^<!-- BEGIN merge-pr state\b.*-->[ \t]*$", re.MULTILINE)
_END_RE = re.compile(r"^" + re.escape(END_MARKER) + r"[ \t]*$", re.MULTILINE)


def read_preserving_newlines(path):
    """Return (text_with_lf_endings, dominant_terminator).

    Read as BYTES rather than text: `read_text()` applies universal newlines, which is
    exactly the conversion this function exists to avoid. The terminator is chosen by
    majority so a file with a few stray lone LFs in an otherwise CRLF document still
    round-trips as CRLF.
    """
    raw = path.read_bytes()
    crlf = raw.count(b"\r\n")
    lf = raw.count(b"\n") - crlf
    newline = "\r\n" if crlf > lf else "\n"
    return raw.decode("utf-8").replace("\r\n", "\n"), newline


def write_preserving_newlines(path, text, newline):
    """Write BYTES, re-applying `newline`. Never `write_text`, which re-translates."""
    path.write_bytes(text.replace("\n", newline).encode("utf-8"))


def classify_endings(raw):
    """Return (ok, detail) for a WIP.md's raw bytes. Pure, so the self-test can pin it.

    `WIP.md` is gitignored, so `.gitattributes` never applies to it and
    `scripts/text_check.py` - which walks `git ls-files` - cannot see it. It is therefore
    the one text file in the tree with no automatic protection, and since v0.18.1 it is
    also expected to be LF. This is what supplies the missing guard.

    A file with NO newline at all passes: an empty or single-line WIP.md carries no
    evidence either way, and a guard that fires on no evidence is a guard that gets
    deleted. Only an actual carriage return is a failure.
    """
    crlf = raw.count(b"\r\n")
    lone_cr = raw.count(b"\r") - crlf
    lf = raw.count(b"\n") - crlf
    if not crlf and not lone_cr:
        return True, f"LF ({lf} lines)" if lf else "no newlines (nothing to check)"
    return False, f"{crlf} CRLF pair(s), {lone_cr} lone CR, {lf} lone LF"


def check_endings(path):
    """Assert `path` is LF. Missing is a pass - WIP.md is per-machine and untracked."""
    if not path.exists():
        print(f"wip-endings: {path.name} not present (nothing to check)")
        return 0
    ok, detail = classify_endings(path.read_bytes())
    if ok:
        print(f"wip-endings: {path.name} is {detail}")
        return 0
    print(
        f"wip-endings: {path.name} carries carriage returns - {detail}.\n"
        "  LF is this tree's base model and WIP.md was converted in v0.18.1; a CRLF copy is\n"
        "  drift, most likely from an editor on a Windows host. Nothing else can catch it:\n"
        "  WIP.md is gitignored, so .gitattributes never applies and text_check.py cannot\n"
        "  see it. Repair it in place - NEVER via a temp file in /tmp, which would carry\n"
        "  user_tmp_t into this Syncthing folder (~/AGENTS.md sec.12):\n"
        "      python3 -c \"from pathlib import Path; p=Path('WIP.md'); "
        "p.write_bytes(p.read_bytes().replace(bytes([13,10]), bytes([10])))\"",
        file=sys.stderr,
    )
    return 1


def run(cmd, required=True):
    """Run `cmd` and return its stripped stdout.

    `required=False` returns None on failure instead of exiting - for facts that may
    legitimately be absent, such as a tag in a repository that has never been released.
    """
    try:
        # encoding="utf-8" (not the platform default) so commit subjects with
        # non-Latin-1 characters - e.g. "→" or an em-dash, both common in this
        # repo's merge subjects - decode correctly on Windows (cp1252) too.
        return subprocess.run(
            cmd, capture_output=True, text=True, encoding="utf-8", check=True
        ).stdout.strip()
    except subprocess.CalledProcessError as e:
        if not required:
            return None
        print(f"Command failed: {' '.join(cmd)}", file=sys.stderr)
        if e.stdout:
            print(e.stdout, file=sys.stderr)
        if e.stderr:
            print(e.stderr, file=sys.stderr)
        sys.exit(e.returncode)


def read_cargo_version(cargo_file):
    """Return the package version from a Cargo.toml, or None if unreadable.

    Matches the first top-level ``version = "..."`` line - the same heuristic
    ``just man`` uses - which is the ``[package]`` version (the workspace table
    has no version key).
    """
    try:
        for line in cargo_file.read_text(encoding="utf-8").splitlines():
            m = re.match(r'version\s*=\s*"([^"]+)"', line)
            if m:
                return m.group(1)
    except OSError:
        pass
    return None


def _cell(value):
    """Make `value` safe inside a Markdown table cell.

    A `|` in a commit subject would otherwise end the cell and shift every column after it,
    and a newline would end the row.
    """
    return value.replace("|", "\\|").replace("\n", " ")


def render_state_block(state):
    """Render the generated block, markers included, with LF line endings.

    `state` keys: branch, head_hash, head_subject, cli_version, sysinfo_version,
    newest_tag, date. Missing values render as `unknown`/`none` rather than raising, so a
    partially readable repository still produces a truthful block instead of no block.
    Pure: no I/O, so the self-test can pin the output exactly.
    """
    branch = state.get("branch")
    if branch in (None, "main"):
        active = "none — `main` is current"
    else:
        # merge-pr checks out main before calling this, so anything else is worth seeing.
        active = f"`{_cell(branch)}` — **not `main`**; check what is checked out"

    versions = [f"`retch-cli` {_cell(state.get('cli_version') or 'unknown')}"]
    if state.get("sysinfo_version"):
        versions.append(f"`retch-sysinfo` {_cell(state['sysinfo_version'])}")

    head = f"`{_cell(state.get('head_hash') or 'unknown')}`"
    if state.get("head_subject"):
        head += f" — {_cell(state['head_subject'])}"

    tag = state.get("newest_tag")
    rows = [
        ("Active branch", active),
        ("main HEAD", head),
        ("Version", " · ".join(versions)),
        ("Newest tag", f"`{_cell(tag)}`" if tag else "none"),
        ("Updated", f"{_cell(state.get('date') or 'unknown')}, by `just merge-pr`"),
    ]
    lines = [
        BEGIN_MARKER,
        "## Repository state (generated)",
        "",
        "| | |",
        "|---|---|",
        *[f"| **{label}** | {value} |" for label, value in rows],
        "",
        END_MARKER,
    ]
    return "\n".join(lines) + "\n"


def apply_state_block(text, block):
    """Return (new_text, action) with `block` placed in `text`.

    `text` must already be LF-normalised. action is "replaced" or "inserted". Raises
    ValueError, and changes nothing, when the markers are duplicated, unpaired or out of
    order - the caller decides what to do; this function never guesses.
    """
    begins = list(_BEGIN_RE.finditer(text))
    ends = list(_END_RE.finditer(text))

    if not begins and not ends:
        # First run on this WIP.md: put the block at the very top, where a reader starts.
        return block + "\n" + text, "inserted"
    if len(begins) != 1 or len(ends) != 1:
        raise ValueError(
            f"expected exactly one BEGIN and one END merge-pr state marker, found "
            f"{len(begins)} BEGIN and {len(ends)} END"
        )
    begin, end = begins[0], ends[0]
    if end.start() < begin.end():
        raise ValueError("the END merge-pr state marker comes before the BEGIN marker")

    # Consume the END marker's own newline, so the block's trailing newline replaces it
    # rather than doubling it.
    stop = end.end() + 1 if text[end.end():end.end() + 1] == "\n" else end.end()
    return text[:begin.start()] + block + text[stop:], "replaced"


def update_file(path, state):
    """Read `path`, place the rendered block, write it back. Returns the action taken."""
    text, newline = read_preserving_newlines(path)
    new_text, action = apply_state_block(text, render_state_block(state))
    write_preserving_newlines(path, new_text, newline)
    return action


def gather_state(root_dir):
    """Read the facts the block reports from git and the two manifests."""
    return {
        "branch": run(["git", "rev-parse", "--abbrev-ref", "HEAD"]),
        "head_hash": run(["git", "rev-parse", "--short", "HEAD"]),
        "head_subject": run(["git", "log", "-1", "--format=%s"]),
        "cli_version": read_cargo_version(root_dir / "Cargo.toml"),
        "sysinfo_version": read_cargo_version(root_dir / "crates" / "sysinfo" / "Cargo.toml"),
        # No tag yet is a legitimate state, not an error.
        "newest_tag": run(["git", "describe", "--tags", "--abbrev=0"], required=False),
        "date": datetime.date.today().isoformat(),
    }


def main():
    # WIP.md and this repo's commit subjects contain non-Latin-1 characters
    # (em-dashes, "→"). Force UTF-8 on stdout so the final status print does not
    # crash under a cp1252 console on Windows (where `just merge-pr` runs it).
    try:
        sys.stdout.reconfigure(encoding="utf-8")
    except (AttributeError, ValueError):
        pass

    root_dir = Path(__file__).resolve().parent.parent
    wip_file = root_dir / "WIP.md"
    if not wip_file.exists():
        print("WIP.md not found. Skipping update.", file=sys.stderr)
        return 0

    state = gather_state(root_dir)
    try:
        action = update_file(wip_file, state)
    except ValueError as e:
        print(f"WIP.md NOT updated: {e}. Fix the markers by hand; nothing was written.",
              file=sys.stderr)
        return 1
    # Say what actually happened, including where - the previous version reported success
    # while rewriting an entry nobody was reading.
    where = "at the top of the file (first run)" if action == "inserted" else "in place"
    print(f"WIP.md state block {action} {where}: main HEAD `{state['head_hash']}`, "
          f"v{state['cli_version']}. Hand-written entries below it are unchanged; add one "
          f"if this merge changed the open-task list.")
    return 0


def _self_test():
    """Assert the block behaviour and the line-ending round trip, in both directions.

    Several cases pin ways this could be WRONG rather than the way it is right: prose that
    quotes the old header strings must survive, malformed markers must refuse rather than
    guess, and a rewrite must not touch a single byte outside the markers.
    """
    import tempfile

    failures = []

    def check(name, cond, detail=""):
        if not cond:
            failures.append(f"{name}: {detail}")

    state = {
        "branch": "main", "head_hash": "abc1234", "head_subject": "Fix a | b (#1)",
        "cli_version": "1.2.3", "sysinfo_version": "0.1.9", "newest_tag": "v1.2.0",
        "date": "2026-01-02",
    }
    block = render_state_block(state)

    # --- rendering ---------------------------------------------------------------------
    check("block starts with BEGIN", block.startswith(BEGIN_MARKER + "\n"), repr(block[:40]))
    check("block ends with END", block.endswith(END_MARKER + "\n"), repr(block[-40:]))
    check("pipe in subject is escaped", "Fix a \\| b (#1)" in block, block)
    check("main renders as none", "none — `main` is current" in block, block)
    off_main = render_state_block({**state, "branch": "feature/x"})
    check("off-main branch is flagged", "**not `main`**" in off_main, off_main)
    no_tag = render_state_block({**state, "newest_tag": None, "sysinfo_version": None})
    check("no tag renders as none", "| **Newest tag** | none |" in no_tag, no_tag)
    check("absent sysinfo is omitted", "retch-sysinfo" not in no_tag, no_tag)
    check("the block never contains the old header strings",
          "### Active Branch:" not in block and "**main HEAD**:" not in block, block)

    # --- placement -----------------------------------------------------------------------
    # THE REGRESSION: prose quoting both old header strings. The old code rewrote the first
    # occurrence of each; this must leave them exactly as written.
    prose = ("## Old entry\n### Active Branch: feature/old\n"
             "**main HEAD**: `0000000` - ancient\nnotes\n")
    inserted, action = apply_state_block(prose, block)
    check("absent markers -> inserted", action == "inserted", action)
    check("inserted at the very top", inserted.startswith(BEGIN_MARKER), inserted[:60])
    check("prose after an insert is byte-identical", inserted.endswith("\n" + prose),
          repr(inserted[-80:]))

    newer = render_state_block({**state, "head_hash": "def5678", "head_subject": "Next"})
    replaced, action = apply_state_block(inserted, newer)
    check("present markers -> replaced", action == "replaced", action)
    check("replacement took effect", "`def5678`" in replaced and "`abc1234`" not in replaced,
          replaced)
    check("prose after a replace is byte-identical", replaced.endswith("\n" + prose),
          repr(replaced[-80:]))
    again, _ = apply_state_block(replaced, newer)
    check("replacing is idempotent", again == replaced, "a second run changed the file")

    middle = "# WIP\nintro\n" + block + "tail\n"
    moved, _ = apply_state_block(middle, newer)
    check("a block mid-file is replaced where it is",
          moved.startswith("# WIP\nintro\n" + BEGIN_MARKER) and moved.endswith(END_MARKER
                                                                                  + "\ntail\n"),
          repr(moved))

    mention = "Prose may mention " + END_MARKER + " mid-line.\n"
    kept, action = apply_state_block(mention, block)
    check("a mid-line mention is not a marker", action == "inserted" and kept.endswith(mention),
          action)

    for name, bad in [
        ("duplicate block", block + "x\n" + block),
        ("BEGIN without END", BEGIN_MARKER + "\nstuff\n"),
        ("END without BEGIN", "stuff\n" + END_MARKER + "\n"),
        ("END before BEGIN", END_MARKER + "\n" + BEGIN_MARKER + "\n"),
    ]:
        try:
            apply_state_block(bad, block)
            check(f"{name} refuses", False, "no ValueError raised")
        except ValueError:
            pass

    # --- line endings, end to end through update_file() -----------------------------------
    with tempfile.TemporaryDirectory() as d:
        tmp = Path(d)

        # CRLF in, CRLF out -- the defect v0.17.12 fixed, now asserted through the real path.
        f = tmp / "crlf.md"
        f.write_bytes(b"### Active Branch: feature/x\r\nbody\r\n**main HEAD**: `old`\r\n")
        check("CRLF first run inserts", update_file(f, state) == "inserted")
        check("CRLF second run replaces", update_file(f, state) == "replaced")
        raw = f.read_bytes()
        lone_lf = raw.count(b"\n") - raw.count(b"\r\n")
        check("CRLF survives, with no lone LF", lone_lf == 0 and raw.count(b"\r\n") > 3,
              f"crlf={raw.count(bytes([13, 10]))} lone_lf={lone_lf}")
        check("old header lines untouched in the file",
              b"### Active Branch: feature/x\r\n" in raw and b"**main HEAD**: `old`\r\n" in raw,
              repr(raw[-80:]))

        # LF in, LF out. This was the control when WIP.md was CRLF; since v0.18.1 it is the
        # LIVE case, and the CRLF case above is what became the control. Both are kept: the
        # property is "follow the file", and it has to hold in both directions or the next
        # merge silently converts whatever it was handed.
        f2 = tmp / "lf.md"
        f2.write_bytes(b"notes\n")
        update_file(f2, state)
        check("LF round-trips", b"\r" not in f2.read_bytes(), "a CR was introduced")

        # A real WIP.md-shaped LF file must survive a full insert+replace cycle with no CR,
        # which is the end-to-end claim the v0.18.1 conversion rests on.
        f4 = tmp / "converted.md"
        f4.write_bytes(b"# WIP\n" + b"a line\n" * 50)
        update_file(f4, state)
        update_file(f4, state)
        raw4 = f4.read_bytes()
        check("a converted WIP.md stays LF across two merges", b"\r" not in raw4,
              f"CR reappeared: {raw4.count(bytes([13]))}")
        check("classify_endings passes the converted file", classify_endings(raw4)[0],
              classify_endings(raw4)[1])

        # check_endings on an absent file is a PASS: WIP.md is per-machine and untracked, so
        # a fresh clone has none and a guard that failed there would fail every fresh clone.
        check("absent WIP.md passes", check_endings(tmp / "definitely-absent.md") == 0,
              "a missing file was treated as a failure")

        # A refusal writes nothing.
        f3 = tmp / "bad.md"
        before = (block + block).encode("utf-8")
        f3.write_bytes(before)
        try:
            update_file(f3, state)
            check("malformed file refuses", False, "no ValueError raised")
        except ValueError:
            check("a refusal leaves the file untouched", f3.read_bytes() == before, "file changed")

    # --- classify_endings, the guard behind `just wip-check` ------------------------------
    check("LF verdict", classify_endings(b"a\nb\n") == (True, "LF (2 lines)"),
          str(classify_endings(b"a\nb\n")))
    check("CRLF is refused", not classify_endings(b"a\r\nb\r\n")[0], "a CRLF file passed")
    check("a lone CR is refused", not classify_endings(b"a\rb\n")[0], "a lone CR passed")
    check("one stray CRLF in an LF file is still refused",
          not classify_endings(b"a\n" * 99 + b"b\r\n")[0],
          "a majority-LF file with one CRLF passed -- drift starts as one line")
    # No newline at all is not evidence of anything, so it must not fire.
    check("no newlines passes", classify_endings(b"")[0] and classify_endings(b"x")[0],
          "an empty or single-line file was flagged")

    # The `.*` trap: a dot matches \r, so a substitution on UN-normalised CRLF text eats the
    # carriage return. Kept as a property so nobody "simplifies" the normalisation away.
    naive = re.sub(r"### Active Branch:.*", "x", "### Active Branch: y\r\nbody\r\n", count=1)
    check("the .* trap is real", naive.count("\r\n") == 1,
          "expected the substitution to eat one CR -- if this fails, re's dot changed")

    if failures:
        for f_ in failures:
            print(f"  FAIL {f_}", file=sys.stderr)
        print(f"update_wip.py self-test FAILED ({len(failures)})", file=sys.stderr)
        return 1
    print("update_wip.py self-test passed")
    return 0


if __name__ == "__main__":
    if "--self-test" in sys.argv[1:]:
        sys.exit(_self_test())
    if "--check-endings" in sys.argv[1:]:
        sys.exit(check_endings(Path(__file__).resolve().parent.parent / "WIP.md"))
    sys.exit(main())