itok 0.3.2

context-cost estimator: token and window budgets for files and changes
Documentation
{
  # itok's pinned toolchain. Lives at the repo ROOT: this is a standalone
  # crate headed for opensource, so it carries an ordinary root flake like
  # any Rust project. The gate it serves is `hk.pkl`, run by hk (V64) --
  # one definition, reached the same way from a laptop and from CI.
  # Standard-only (V31): the rust toolchain plus nextest and llvm-cov --
  # the cargo-native gate. No host guard bins, because those do not travel.
  description = "itok -- context-cost estimator (dev shell)";

  # `nixpkgs-lock` is the fleet's SOLE nixpkgs authority, and nixpkgs follows
  # it rather than naming a rev of its own. One rev across every repo is what
  # makes the shared binary cache hit instead of rebuilding -- a second
  # nixpkgs edge here would silently fork it, and the fork costs a full
  # toolchain rebuild rather than an error anyone would notice.
  #
  # Following a BRANCH is not the drift it looks like: `flake.lock` is the
  # pin, so the toolchain only moves when someone runs `nix flake update`
  # and commits the result. The gate can always name the compiler that
  # produced a verdict.
  #
  # This does not breach the standalone rule (V13/V31): `flake.nix` is in
  # `Cargo.toml`'s exclude list, so none of these inputs reach a consumer of
  # the published crate. They bind the DEV SHELL, not the tool.
  inputs.nixpkgs-lock.url = "github:pr0d1r2/nixpkgs-lock";
  inputs.nixpkgs.follows = "nixpkgs-lock/nixpkgs";

  # The gate RUNNER, from its own flake rather than from nixpkgs.
  #
  # nixos 26.05 dropped `pkgs.hk` entirely -- the dev shell stopped
  # evaluating with `attribute 'hk' missing`, which is the whole gate gone in
  # one nixpkgs bump. Sourcing it here makes the runner an explicit, versioned
  # choice instead of a side effect of whatever nixpkgs happens to carry,
  # exactly the argument that already pins microlith to a tag below.
  #
  # `nixpkgs-lock` follows ours so the closure holds ONE nixpkgs. Without it
  # hk resolves its own copy of the same lock repo and the cache misses.
  inputs.hk.url = "github:pr0d1r2/nix-hk";
  inputs.hk.inputs.nixpkgs-lock.follows = "nixpkgs-lock";

  # The FORMAT owner, pinned to a released TAG rather than a branch: the
  # rules this spec is held to should change when the pin changes and at no
  # other time.
  #
  # v0.6.1 rather than v0.5.0 because 26.05 made the old pin UNBUILDABLE:
  # both earlier tags declare `rust-version = "1.96"` and this nixpkgs
  # carries rustc 1.95.0, so the shell died with `rustc 1.95.0 is not
  # supported by microlith@0.5.0`. 0.6.1 lowers its MSRV to 1.95 and takes
  # nixpkgs from the same fleet lock.
  #
  # It follows our `nixpkgs-lock`, not our `nixpkgs`: 0.6.1 consumes the
  # lock repo directly, so redirecting only `nixpkgs` left a SECOND
  # `nixpkgs-lock` node resolving the same repo twice. One authority, one
  # nixpkgs, one toolchain to download -- same wiring as `hk` above.
  # `nix-hk` follows ours too: microlith consumes it for its OWN dev shell,
  # and left alone it locks a different rev -- two hk builds in one closure
  # for a binary neither this shell nor `mth` needs twice.
  inputs.microlith.url = "github:pr0d1r2/microlith/v0.6.1";
  inputs.microlith.inputs.nixpkgs-lock.follows = "nixpkgs-lock";
  inputs.microlith.inputs.nix-hk.follows = "hk";

  outputs =
    {
      nixpkgs,
      microlith,
      hk,
      ...
    }:
    let
      # Same tier-1 systems as the host.
      systems = [
        "aarch64-darwin"
        "x86_64-darwin"
        "x86_64-linux"
        "aarch64-linux"
      ];
      forAll = f: nixpkgs.lib.genAttrs systems (s: f nixpkgs.legacyPackages.${s});

      # itok dogfoods itself (V15) -- so the dev shell must PROVIDE `itok`,
      # not merely the toolchain to build it. Deliberately a SHIM and not a
      # nix package in the shell's closure: a package would have to build
      # the crate in order to ENTER the shell, so a compile error would
      # lock you out of the very shell you need to fix it. `cargo run` is a
      # no-op when the build is fresh and rebuilds only on change, so the
      # shim always matches the working tree, which a pinned package never
      # does.
      #
      # The real `packages.default` for consumers is a SECOND output, not
      # a replacement (T99) -- see `itokPkg` below. Both exist because
      # they answer different questions: the shim answers "what does the
      # tree I am editing do", the package answers "what does the version
      # I depend on do". A consumer wants the second and must never be
      # handed the first; a contributor wants the first and would be
      # locked out of the shell by the second.
      # ITOK_MANIFEST is exported by the shellHook; ITOK_PROFILE=release
      # is the escape hatch when a debug tiktoken (--bpe) is too slow on a
      # large tree.
      itokShim =
        pkgs:
        pkgs.writeShellScriptBin "itok" ''
          set -eu
          manifest="''${ITOK_MANIFEST:-}"
          if [ -z "$manifest" ] || [ ! -f "$manifest" ]; then
            echo "itok(shim): ITOK_MANIFEST unset or missing -- re-enter the dev shell from the crate root" >&2
            exit 2
          fi
          profile=""
          [ "''${ITOK_PROFILE:-debug}" = "release" ] && profile="--release"
          exec cargo run --quiet $profile \
            --manifest-path "$manifest" --bin itok \
            ''${ITOK_FEATURES:+--features "$ITOK_FEATURES"} -- "$@"
        '';
      # The FORMAT owner (V7 of microlith): this spec's structural rules
      # live in one implementation, and itok CALLS it rather than porting
      # it -- the ported copy is what drifted while both gates stayed green.
      # The PACKAGE is `microlith`; the BINARY it installs is `mth`. Those
      # are two names on purpose, so keep them apart: paths and manifests
      # take `microlith`, anything invoked takes `mth`.
      #
      # This WAS a shim over a sibling `../microlith` checkout, because
      # before the release there was no public URL to name and an absolute
      # local path is the private reference V39 forbids shipping. There is
      # a URL now, so the pin is an ordinary flake input and the sibling
      # stops being required: a fresh public clone can enter this shell and
      # commit, which is the whole point -- the `mth` gate step hard-fails
      # when the binary is absent (nothing else checks SPEC.md structure,
      # so degrading would read as a pass). `hk.pkl` did not change, and
      # could not have: it names the binary, never a path.
      #
      # The wrapper exists for the HATCH, not the default. Default is the
      # pinned build, so the rules are whatever the tag says. Setting
      # MICROLITH_MANIFEST to a sibling checkout runs THAT instead, which
      # is how a format change gets tried against a real spec before it is
      # tagged. Deliberately opt-in: an auto-detected sibling would silently
      # outrank the pin whenever the directory happened to exist, and then
      # the gate's verdict would depend on the layout of the machine.
      mthWrapper =
        pkgs:
        let
          pinned = pkgs.rustPlatform.buildRustPackage {
            pname = "microlith";
            version = "0.6.1";
            src = microlith;
            cargoLock.lockFile = "${microlith}/Cargo.lock";
            doCheck = false;
          };
        in
        pkgs.writeShellScriptBin "mth" ''
          set -eu
          manifest="''${MICROLITH_MANIFEST:-}"
          if [ -n "$manifest" ]; then
            if [ ! -f "$manifest" ]; then
              echo "mth: MICROLITH_MANIFEST=$manifest does not exist -- unset it to use the pinned ${pinned.version}" >&2
              exit 2
            fi
            exec cargo run --quiet --manifest-path "$manifest" --bin mth -- "$@"
          fi
          exec ${pinned}/bin/mth "$@"
        '';
      # The reproducible build (V62). Possible only because the flake sits
      # at the repo root and can therefore see `Cargo.toml`/`src/`, and
      # because `Cargo.lock` is tracked -- flakes copy git-TRACKED files
      # into the store, and the build sandbox has no network, so
      # `cargoLock.lockFile` is what lets nix vendor exact crates offline.
      itokPkg =
        pkgs: features:
        pkgs.rustPlatform.buildRustPackage {
          pname = "itok";
          # Read from Cargo.toml so there is ONE version, not two that can
          # disagree (V64's rule, applied to a number).
          version = (builtins.fromTOML (builtins.readFile ./Cargo.toml)).package.version;
          # Only what the BUILD reads. With `src = ./.` every tracked file
          # is an input, so editing SPEC.md or README rebuilt the crate
          # from scratch. Tests are excluded too -- doCheck is false, and
          # they need a `.git` the store does not have.
          src = pkgs.lib.fileset.toSource {
            root = ./.;
            fileset = pkgs.lib.fileset.unions [
              ./src
              ./dev
              ./Cargo.toml
              ./Cargo.lock
            ];
          };
          cargoLock.lockFile = ./Cargo.lock;
          buildNoDefaultFeatures = features != null;
          buildFeatures = if features == null then [ ] else features;
          # The test suite shells out to git for HEAD~n (V33's `gitref`),
          # and a nix store source has no `.git` -- by design, that is what
          # makes the build reproducible. So the suite cannot run HERE; it
          # runs in the dev shell and in CI, where history exists (V37/B3
          # is the same constraint seen from the other side).
          doCheck = false;
          meta = {
            description = "context-cost estimator: token and window budgets for files and changes";
            homepage = "https://github.com/pr0d1r2/itok";
            license = pkgs.lib.licenses.mit;
            mainProgram = "itok";
          };
        };
    in
    {
      packages = forAll (pkgs: {
        # Default features: the dummy tier plus `--bpe` (V4).
        default = itokPkg pkgs null;
        # The zero-dependency core alone -- proves V23/V13 build clean.
        itok-minimal = itokPkg pkgs [ ];
        # Adds the LAN-exact rung (V22), with TLS (V23/V25).
        itok-ollama = itokPkg pkgs [
          "bpe"
          "ollama"
        ];
      });

      devShells = forAll (pkgs: {
        default = pkgs.mkShell {
          packages = [
            (itokShim pkgs)
            (mthWrapper pkgs)
            pkgs.rustc
            pkgs.cargo
            pkgs.clippy
            pkgs.rustfmt
            pkgs.cargo-nextest
            pkgs.cargo-llvm-cov
            # llvm-cov / llvm-profdata come from the LLVM package, wired
            # via the env vars cargo-llvm-cov looks for.
            pkgs.llvmPackages.llvm
            pkgs.git # itok shells out to `git ls-files` for tracked files.
            # The gate runner (V64): `hk.pkl` holds the ops, hk runs them
            # locally on pre-commit/pre-push and in CI via `hk check`.
            # Pinned here so every contributor on nix gets the same hk as
            # the vendored pkl schema was cut from.
            #
            # From the `hk` INPUT, not `pkgs.hk`: nixos 26.05 dropped the
            # nixpkgs package, and the runner of every rule in this repo is
            # not something to leave at the mercy of a channel.
            hk.packages.${pkgs.stdenv.hostPlatform.system}.default
            # Gate steps that need a real binary (V72). Pinned here so the
            # dev shell and CI run the same versions.
            pkgs.typos
            pkgs.actionlint
            # zizmor AUDITS the workflow; actionlint CHECKS it. Not a
            # replacement for the line above -- zizmor does no syntax or
            # type checking at all, and actionlint knows nothing about
            # token scope. The two findings that motivated adding it are
            # both structural rather than typographic: a workflow with no
            # `permissions:` block runs at the repository default, and a
            # checkout that keeps its credentials leaves a token in
            # `.git/config` for every later step to read. Neither is a
            # mistake actionlint is built to see.
            pkgs.zizmor
            pkgs.taplo
            pkgs.shellcheck
            # V72 again: `nixfmt --check` gates this very file.
            pkgs.nixfmt
            # Relative-link resolution across the docs set. Offline only --
            # it never touches the network, so it cannot fail on someone
            # else's 404.
            pkgs.lychee
            # Secret shapes `no-private-key` does not match. Here rather
            # than nowhere because a leaked token in a PUBLIC history is
            # irreversible, and this repo is headed for one.
            pkgs.ripsecrets
            # The version ladder (V70), checked instead of remembered.
            # Vacuous until the first `v*` tag, and it says so.
            pkgs.cargo-semver-checks
            # THE RELEASE, which is a tool and not a script. Everything T59
            # described as prose -- clean tree, allowed branch, tag scheme,
            # dry-run first, verify, publish, push -- this already does, and
            # `release.toml` configures it. `0.3.0-rc.1` went out by hand
            # from that prose: eight commands whose ORDER was remembered
            # rather than enforced, which is the shape of a rule with no
            # runner (V17). Propagated from microlith, which reached the
            # same conclusion and deleted the script it had started (V84).
            pkgs.cargo-release
          ];
          # Pin locale so tool output is deterministic across machines.
          LANG = "C.UTF-8";
          # Opts a bare `cargo test` in the dev shell in to the dogfood
          # tests, matching hk. Without it a contributor's local run silently
          # skips fourteen tests that CI runs, and "green here, red in the
          # gate" is the confusion this repo spends real effort avoiding.
          # See src/testutil.rs for why the gate is an env var.
          ITOK_DOGFOOD = "1";
          LLVM_COV = "${pkgs.llvmPackages.llvm}/bin/llvm-cov";
          LLVM_PROFDATA = "${pkgs.llvmPackages.llvm}/bin/llvm-profdata";
          # Resolved at ENTRY, not baked in: `.envrc` sits beside
          # `Cargo.toml`, and direnv enters with PWD = that directory --
          # in-repo (crates/itok/) and extracted (repo root) alike, so one
          # rule works in both layouts (V37: derive, never hardcode).
          shellHook = ''
            if [ -f "$PWD/Cargo.toml" ]; then
              export ITOK_MANIFEST="$PWD/Cargo.toml"
            else
              echo "itok(shell): no Cargo.toml in $PWD -- \`itok\` shim disabled" >&2
            fi
            # MICROLITH_MANIFEST is deliberately NOT exported here. `mth`
            # comes from the pinned input, and the sibling checkout is an
            # override a person chooses, not one the filesystem chooses for
            # them -- see the wrapper above.
            # Entering the shell installs the hooks, so a contributor
            # cannot forget to (V71: a gate you must remember to enable is
            # not a gate). Silent and idempotent -- Unix philosophy.
            #
            # Written by hand rather than via `hk install`, because the
            # command that writes assumes hk is on PATH. It is not,
            # outside this shell -- and a hook that hard-fails when its
            # runner is missing does not gate a commit, it blocks EVERY
            # git command in the repo (B6). So the command degrades: no
            # hk, no gate, one line to stderr saying so. Loud, not silent.
            #
            # SKIPPED when a global install exists: git aggregates
            # `hook.<name>.command` across scopes, so having both fires
            # every hook twice. The global install is the better setup, so
            # it wins when present.
            if ! git config --global --get-regexp '^hook\.hk-' >/dev/null 2>&1; then
              # SCOPE DIFFERS BY EVENT, and it is the one thing this
              # shared loop must not share (B32). `hk run` selects STAGED
              # files. At commit time that is exactly the change. At push
              # time it is NOTHING -- the commit emptied the index -- so
              # every step carrying a glob was skipped and the hook exited
              # 0 with `steps = all` declared and twelve of them never run.
              # A push publishes the whole tree, so the push-side gate
              # reads the whole tree (V120).
              #
              # `--from-hook` forwards git's own hook arguments, which hk
              # then treats as a FILE list, and `--all` refuses to sit
              # beside one. The push side does not need the refs anyway:
              # the whole tree is the scope, so it takes no arguments.
              for ev in pre-commit pre-push; do
                run="run $ev --from-hook \"\$@\""
                if [ "$ev" = "pre-push" ]; then run="run pre-push --all"; fi
                git config --local "hook.hk-$ev.event" "$ev"
                git config --local "hook.hk-$ev.command" \
                  "command -v hk >/dev/null 2>&1 || { echo 'hk not found -- gate skipped; enter the dev shell (direnv/nix develop) to run it' >&2; exit 0; }; test \"\''${HK:-1}\" = \"0\" || hk $run"
              done
            fi
          '';
        };
      });
    };
}