teksilo_corpus/lib.rs
1// SPDX-License-Identifier: MPL-2.0
2// SPDX-FileCopyrightText: 2026 FernTech
3
4//! The version-matched Teksilo documentation corpus.
5//!
6//! This crate is **data**, not logic: one prebuilt retrieval index over the
7//! hand-written guides and the worked examples — their text, chunked, with
8//! BM25 statistics and a dense vector per chunk — embedded at build time and
9//! parsed on first use. It carries no ML dependency; the vectors are computed
10//! at release time, so a consumer only ever encodes the *query*.
11//!
12//! It is versioned in lockstep with the framework, which is the whole point:
13//! `cargo install cargo-teksilo --version X` resolves `teksilo-corpus X`
14//! through cargo's own resolver, so an answer can never come from a
15//! different teksilo than the one an app pinned.
16//!
17//! # The corpus is one file
18//!
19//! `corpus/index.json` is the whole crate's data, embedded by
20//! [`INDEX_JSON`]. Every [`Chunk`] carries its own `text`, and its `path`
21//! names the **original** file in the teksilo repository —
22//! `docs/scroll-area.md`, `examples/simple_button/src/main.rs` — so a
23//! search result cites something that exists and resolves on GitHub.
24//!
25//! It did not always work that way. An earlier layout mirrored all 158
26//! guides and example sources into `corpus/` and stored only a line range
27//! into the copy. That put a second copy of every guide in the tree, one
28//! fuzzy-open away from the real one, and an edit made in the copy was
29//! discarded without a word by the next regeneration. Moving the text into
30//! the index was size-neutral — `index.json` grew by roughly what the
31//! copies weighed — and int8-quantised vectors had already bought the
32//! headroom that splitting the text out was originally for.
33//!
34//! [`Chunk::line_start`] / [`Chunk::line_end`] survive as **provenance**:
35//! an inclusive, 0-based range naming where in `path` this text was found,
36//! so a reader can be sent to the right lines. `tools/build_corpus.py`
37//! verifies every range against its source file before emitting the index,
38//! and the line split is [`str::lines`]' — on `\n` only, with a trailing
39//! `\r` stripped — so Rust and the generator agree on what a line index
40//! means.
41//!
42//! # Term strings are interned
43//!
44//! [`Index::terms`] is the sorted table of every token in the corpus;
45//! [`Chunk::tokens`] and [`Index::df`] are keyed by a term's index into it
46//! rather than repeating the string thousands of times. Resolve one with
47//! [`Index::term`], or a whole chunk's with [`Index::chunk_terms`]. In
48//! JSON those maps are objects with decimal string keys (`{"17":5}`),
49//! since JSON object keys must be strings; serde deserializes them
50//! straight into the `u32` keys used here.
51//!
52//! # The index is built in two passes, in this order
53//!
54//! 1. `python3 tools/build_corpus.py` chunks the guides and examples and
55//! writes `index.json` with `encoder`, `encoder_dim` and every
56//! [`Chunk::embedding`] as `null`. It regenerates the file **from
57//! scratch**, so it necessarily discards any vectors already there —
58//! except the ones it can carry forward by chunk-text hash, which is
59//! every vector whose text an edit did not touch.
60//! 2. `cargo teksilo build-vectors` encodes each chunk, quantises the
61//! result, and writes the vectors back into that same `index.json`,
62//! setting `encoder` and `encoder_dim`.
63//!
64//! Running them in the other order leaves a corpus with no vectors, which
65//! degrades to lexical search rather than breaking — but it is a
66//! regression, so the order is part of the release procedure, and
67//! `build_corpus.py --check` is deliberately blind to the three vector
68//! fields: it checks that the *chunking* is current, which is the thing
69//! that goes stale when a guide is edited.
70
71use std::collections::BTreeMap;
72use std::sync::OnceLock;
73
74use serde::{Deserialize, Serialize};
75
76/// The corpus index, embedded at build time.
77///
78/// `include_str!` rather than a directory embedding: the corpus is exactly
79/// one file, and a macro that says so cannot quietly start shipping a
80/// second one. It also makes "the file is missing" and "the file is not
81/// UTF-8" compile errors rather than runtime variants of [`CorpusError`].
82pub static INDEX_JSON: &str = include_str!("../corpus/index.json");
83
84/// The teksilo version this corpus was generated from.
85pub const CORPUS_VERSION: &str = env!("CARGO_PKG_VERSION");
86
87/// A chunk's dense vector: int8 values plus the scale that decodes them.
88///
89/// # Why int8, and why a per-vector scale
90///
91/// The encoder emits 384 `f32`s per chunk. Stored as JSON numbers that is
92/// ~4 MB for this corpus, which alone would push the crate past the 10 MB
93/// crates.io limit. Stored as int8 it is 384 bytes, base64-encodes to 512
94/// characters, and — because the encoder L2-normalises its output — loses
95/// almost nothing: a vector's cosine against its own quantisation is
96/// > 0.999 (see `cargo-teksilo`'s `vectors` tests, which assert it).
97///
98/// The scale is per-vector rather than corpus-wide because it is derived
99/// from *that* vector's largest magnitude, so every vector uses the full
100/// −127..=127 range regardless of how peaked it happens to be.
101///
102/// # JSON shape
103///
104/// ```json
105/// "embedding": {"scale": 0.00512, "q": "ABn/…"}
106/// ```
107///
108/// `q` is standard base64 (RFC 4648, padded) of the int8 values reinterpreted
109/// as bytes, two's complement, in dimension order. Base64 rather than a JSON
110/// array of decimal numbers: the array form is ~1600 bytes per chunk against
111/// base64's 512, which is the difference between shipping and not shipping.
112#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
113pub struct ChunkEmbedding {
114 /// Multiplier that turns a stored int8 back into the encoder's `f32`.
115 /// Always > 0; a zero vector is stored with `scale = 1.0` rather than
116 /// with a zero scale, so decoding never divides by it.
117 pub scale: f32,
118 /// The quantised values, base64 in JSON under the key `q`.
119 #[serde(rename = "q", with = "base64_i8")]
120 pub values: Vec<i8>,
121}
122
123impl ChunkEmbedding {
124 /// The approximate `f32` vector this was quantised from.
125 ///
126 /// Retrieval does not need this — a cosine can be taken on the int8
127 /// values directly, and `cargo-teksilo` does exactly that — but a
128 /// consumer doing its own vector maths wants the floats back.
129 pub fn dequantised(&self) -> Vec<f32> {
130 self.values
131 .iter()
132 .map(|&q| f32::from(q) * self.scale)
133 .collect()
134 }
135
136 /// Number of dimensions, for checking against [`Index::encoder_dim`].
137 pub fn dim(&self) -> usize {
138 self.values.len()
139 }
140}
141
142/// Base64 for the int8 payload, as a serde `with` module.
143///
144/// Hand-rolled rather than pulled in as a dependency: this crate is *data*
145/// and its whole value is that a consumer can depend on it without
146/// inheriting a tree. Standard alphabet, padded, so the field is readable
147/// by any other tool without a private convention to look up.
148mod base64_i8 {
149 use serde::{Deserialize, Deserializer, Serializer};
150
151 const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
152
153 /// `ALPHABET` inverted: byte value -> 6-bit symbol, or 0xFF for "not a
154 /// base64 character". Built at compile time so decoding is a table
155 /// lookup rather than a search.
156 const INVERSE: [u8; 256] = {
157 let mut table = [0xFFu8; 256];
158 let mut i = 0usize;
159 while i < 64 {
160 table[ALPHABET[i] as usize] = i as u8;
161 i += 1;
162 }
163 table
164 };
165
166 pub fn encode(bytes: &[u8]) -> String {
167 let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
168 for group in bytes.chunks(3) {
169 let b0 = group[0] as u32;
170 let b1 = *group.get(1).unwrap_or(&0) as u32;
171 let b2 = *group.get(2).unwrap_or(&0) as u32;
172 let packed = (b0 << 16) | (b1 << 8) | b2;
173 out.push(ALPHABET[(packed >> 18) as usize & 0x3F] as char);
174 out.push(ALPHABET[(packed >> 12) as usize & 0x3F] as char);
175 out.push(if group.len() > 1 {
176 ALPHABET[(packed >> 6) as usize & 0x3F] as char
177 } else {
178 '='
179 });
180 out.push(if group.len() > 2 {
181 ALPHABET[packed as usize & 0x3F] as char
182 } else {
183 '='
184 });
185 }
186 out
187 }
188
189 pub fn decode(text: &str) -> Result<Vec<u8>, &'static str> {
190 let body = text.trim_end_matches('=');
191 if !text.len().is_multiple_of(4) || text.len() - body.len() > 2 {
192 return Err("not a padded base64 string");
193 }
194 let mut out = Vec::with_capacity(body.len() / 4 * 3);
195 let mut acc = 0u32;
196 let mut bits = 0u32;
197 for &byte in body.as_bytes() {
198 let symbol = INVERSE[byte as usize];
199 if symbol == 0xFF {
200 return Err("invalid base64 character");
201 }
202 acc = (acc << 6) | u32::from(symbol);
203 bits += 6;
204 if bits >= 8 {
205 bits -= 8;
206 out.push((acc >> bits) as u8);
207 }
208 }
209 Ok(out)
210 }
211
212 pub fn serialize<S: Serializer>(values: &[i8], serializer: S) -> Result<S::Ok, S::Error> {
213 let bytes: Vec<u8> = values.iter().map(|&v| v as u8).collect();
214 serializer.serialize_str(&encode(&bytes))
215 }
216
217 pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Vec<i8>, D::Error> {
218 let text = String::deserialize(deserializer)?;
219 let bytes = decode(&text).map_err(serde::de::Error::custom)?;
220 Ok(bytes.into_iter().map(|b| b as i8).collect())
221 }
222}
223
224/// One retrieval chunk of the corpus, as written by `tools/build_corpus.py`.
225///
226/// `crate_name` is `#[serde(rename = "crate")]` because `crate` is a Rust
227/// keyword. `embedding` is written by a **second** pass —
228/// `cargo teksilo build-vectors` — which runs after `build_corpus.py` and
229/// fills it in for every chunk at once; see the crate-level docs.
230#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
231pub struct Chunk {
232 /// Position of this chunk in `Index::chunks`; stable within one
233 /// generated corpus (i.e. one `teksilo_version`).
234 pub id: usize,
235 /// `"guide"`, `"example"`, or `"footer"`.
236 ///
237 /// A `"footer"` is a guide's closing navigation list — "See also",
238 /// "Reference", "Code references". It is carried so that a document can be
239 /// reassembled from its chunks in full, and is excluded from retrieval:
240 /// being short and made of link paths, it outranks the prose it points at.
241 pub kind: String,
242 /// Path to the source file this chunk came from, **relative to the
243 /// teksilo repository root** (e.g. `"docs/layout-primitives.md"` or
244 /// `"examples/simple_button/src/main.rs"`). It is a real path in the
245 /// repository, so it is safe to print as a citation — the crate itself
246 /// does not ship the file, only this chunk's [`text`](Chunk::text).
247 pub path: String,
248 /// For a guide chunk, the open ancestor headings plus this chunk's own
249 /// heading. For an example chunk, a one-element label (an item
250 /// signature, or a `"lines A-B"` window descriptor for the windowed
251 /// fallback).
252 pub heading_path: Vec<String>,
253 /// This chunk's text, verbatim — exactly what was tokenized into
254 /// [`tokens`](Chunk::tokens) and encoded into
255 /// [`embedding`](Chunk::embedding). [`Chunk::text`] is the borrowing
256 /// accessor; the field is `pub` like every other one so a consumer can
257 /// still build a `Chunk` of its own.
258 pub text: String,
259 /// Index of this chunk's first line in `path`, 0-based and inclusive.
260 ///
261 /// Provenance into the original file, not storage: the text is right
262 /// there in [`Chunk::text`]. `tools/build_corpus.py` proves the range
263 /// names that text before emitting the index.
264 pub line_start: usize,
265 /// Index of this chunk's last line in `path`, 0-based and **inclusive**.
266 /// Equal to `line_start` for a one-line chunk.
267 pub line_end: usize,
268 /// The originating example crate's name, for an example chunk; `None`
269 /// for a guide chunk.
270 #[serde(rename = "crate")]
271 pub crate_name: Option<String>,
272 /// Per-chunk BM25 term frequencies, keyed by a term's index into
273 /// [`Index::terms`] rather than by the term itself. Terms are
274 /// lowercased, `[a-z0-9_]+`-tokenized, of length > 1, unstemmed.
275 pub tokens: BTreeMap<u32, u32>,
276 /// Sum of `tokens`' values — this chunk's length in tokens.
277 pub len: u32,
278 /// This chunk's dense vector, quantised — `None` in a corpus whose
279 /// [`Index::encoder`] is `None`, `Some` in every chunk of one whose
280 /// `encoder` is set. See [`ChunkEmbedding`].
281 pub embedding: Option<ChunkEmbedding>,
282}
283
284impl Chunk {
285 /// This chunk's text, borrowed out of the index.
286 ///
287 /// Infallible and free. It used to be an `Option<String>` joined out of
288 /// a copy of the source file embedded beside the index; the text is now
289 /// stored in the chunk itself, so there is nothing left to fail at and
290 /// nothing to allocate.
291 pub fn text(&self) -> &str {
292 &self.text
293 }
294}
295
296/// The full retrieval index, embedded as [`INDEX_JSON`].
297#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
298pub struct Index {
299 /// Format version of this index. Currently `2`.
300 ///
301 /// Bumped from `1` when the corpus stopped shipping copies of the guides
302 /// and examples: a chunk gained its own `text`, and `path` changed from a
303 /// corpus-internal name (`guides/foo.md`) to the real repository path
304 /// (`docs/foo.md`). A version 1 index will not deserialize here — it has no
305 /// `text` — which is the point of the field moving with the format.
306 ///
307 /// The crate and the corpus it embeds ship together, so a mismatch cannot
308 /// arise from a normal install; the field is for anyone reading a stray
309 /// `index.json` and needing to know what shape it is.
310 pub schema: u32,
311 /// The teksilo workspace version this corpus was generated from.
312 /// Matches [`CORPUS_VERSION`] for a corpus built from this crate's own
313 /// release, but is carried in the data too so a copy of `index.json`
314 /// examined outside the crate is still self-describing.
315 pub teksilo_version: String,
316 /// Name of the dense-vector encoder that produced every
317 /// [`Chunk::embedding`], e.g. `"BAAI/bge-small-en-v1.5"`; `None` in a
318 /// corpus that has not had `cargo teksilo build-vectors` run over it.
319 ///
320 /// **A consumer must compare this against the encoder it is about to
321 /// encode a query with, and refuse the vector path when they differ.**
322 /// Two unrelated encoders at the same dimension yield perfectly
323 /// well-formed cosines that mean nothing, so the failure is silent
324 /// rather than loud — there is no runtime symptom to notice.
325 pub encoder: Option<String>,
326 /// Dimensionality of that encoding, and therefore of every
327 /// [`ChunkEmbedding::values`]. `None` when `encoder` is.
328 pub encoder_dim: Option<u32>,
329 /// `chunks.len()`, kept as its own field so a consumer can sanity-check
330 /// the index without materializing `chunks` first.
331 pub chunk_count: usize,
332 /// Mean chunk length in tokens across the whole corpus — BM25's
333 /// `avgdl`.
334 pub avgdl: f64,
335 /// Every term in the corpus, **sorted**, interning the keys of
336 /// [`Index::df`] and every [`Chunk::tokens`]. Sorted, so a query term
337 /// can be binary-searched straight to its index — see
338 /// [`Index::term_index`].
339 pub terms: Vec<String>,
340 /// Corpus-wide document frequency: for each term *index*, the number of
341 /// chunks that term appears in at least once.
342 pub df: BTreeMap<u32, u32>,
343 /// Every chunk in the corpus, in a stable, deterministic order.
344 pub chunks: Vec<Chunk>,
345}
346
347impl Index {
348 /// The chunk with the given `id`, if one exists.
349 ///
350 /// Looks up by the `id` field rather than by vector position, so this
351 /// stays correct even if a future corpus version doesn't lay `chunks`
352 /// out in id order.
353 pub fn chunk(&self, id: usize) -> Option<&Chunk> {
354 self.chunks.iter().find(|chunk| chunk.id == id)
355 }
356
357 /// The text of the chunk with the given `id`, if one exists.
358 ///
359 /// Sugar for [`Index::chunk`] plus [`Chunk::text`]. `None` means no
360 /// chunk carries that `id` — the text itself is never missing.
361 pub fn chunk_text(&self, id: usize) -> Option<&str> {
362 Some(self.chunk(id)?.text())
363 }
364
365 /// The term string an index in [`Chunk::tokens`] or [`Index::df`]
366 /// refers to.
367 ///
368 /// Takes a `u32` to match those maps' key type, so iterating a chunk's
369 /// tokens needs no cast.
370 pub fn term(&self, idx: u32) -> Option<&str> {
371 self.terms.get(idx as usize).map(String::as_str)
372 }
373
374 /// The index of `term` in [`Index::terms`], if the corpus contains it.
375 ///
376 /// A binary search — `terms` is sorted — so this is the cheap way to
377 /// turn a tokenized query into the keys `df` and `tokens` speak.
378 pub fn term_index(&self, term: &str) -> Option<u32> {
379 self.terms
380 .binary_search_by(|candidate| candidate.as_str().cmp(term))
381 .ok()
382 .map(|idx| idx as u32)
383 }
384
385 /// Corpus-wide document frequency of `term`, by string. `0` if the
386 /// corpus never saw it.
387 pub fn document_frequency(&self, term: &str) -> u32 {
388 self.term_index(term)
389 .and_then(|idx| self.df.get(&idx))
390 .copied()
391 .unwrap_or(0)
392 }
393
394 /// A chunk's term frequencies with the terms resolved back to strings,
395 /// in ascending term order (which is ascending term-index order, since
396 /// [`Index::terms`] is sorted).
397 ///
398 /// Any token index that does not name a term in this index is skipped;
399 /// within one generated corpus there are none.
400 pub fn chunk_terms<'a>(&'a self, chunk: &Chunk) -> Vec<(&'a str, u32)> {
401 chunk
402 .tokens
403 .iter()
404 .filter_map(|(&idx, &count)| Some((self.term(idx)?, count)))
405 .collect()
406 }
407}
408
409/// An error parsing the embedded corpus index.
410///
411/// One variant, and it names the only failure `include_str!` leaves
412/// reachable: a missing file and non-UTF-8 bytes are both compile errors
413/// now, so the variants that used to report them at runtime are gone.
414#[derive(Debug, Clone, thiserror::Error)]
415#[non_exhaustive]
416pub enum CorpusError {
417 /// [`INDEX_JSON`]'s content is not valid per the [`Index`] schema.
418 #[error("failed to parse index.json: {0}")]
419 InvalidJson(String),
420}
421
422/// Parse and cache the embedded corpus index.
423///
424/// The first call parses [`INDEX_JSON`]; every later call returns the
425/// cached result. Parsing is infallible in practice — an [`Err`] here
426/// means the embedded index itself is malformed, which is a build-time
427/// packaging bug in `teksilo-corpus` rather than a condition a caller can
428/// recover from — but the [`Result`] is still surfaced rather than
429/// panicking, since a corpus consumer may want to degrade gracefully
430/// (e.g. an editor plugin falling back to no documentation search) instead
431/// of crashing the host process.
432pub fn index() -> Result<&'static Index, CorpusError> {
433 static CACHE: OnceLock<Result<Index, CorpusError>> = OnceLock::new();
434 let cached = CACHE.get_or_init(|| {
435 serde_json::from_str::<Index>(INDEX_JSON)
436 .map_err(|e| CorpusError::InvalidJson(e.to_string()))
437 });
438 match cached {
439 Ok(index) => Ok(index),
440 Err(e) => Err(e.clone()),
441 }
442}
443
444#[cfg(test)]
445mod tests {
446 use super::*;
447
448 /// The teksilo checkout this crate lives in, when there is one.
449 ///
450 /// `Some` in the repository, `None` for a `cargo package` verification
451 /// build or anyone testing the published crate from a registry copy —
452 /// the corpus no longer carries the sources, so a test that wants to
453 /// open one has to say what it does without them. Identified by the two
454 /// directories the corpus is generated from rather than by
455 /// `.git`/`Cargo.toml`, which a vendored copy could also have.
456 fn repo_root() -> Option<std::path::PathBuf> {
457 let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
458 .parent()?
459 .parent()?
460 .to_path_buf();
461 (root.join("docs").is_dir() && root.join("examples").is_dir()).then_some(root)
462 }
463
464 #[test]
465 fn index_parses() {
466 index().expect("embedded index.json should parse");
467 }
468
469 #[test]
470 fn chunk_count_matches_chunks_len() {
471 let idx = index().unwrap();
472 assert_eq!(idx.chunk_count, idx.chunks.len());
473 }
474
475 #[test]
476 fn every_chunk_path_is_a_repo_path_and_really_exists_in_a_checkout() {
477 // `path` is printed as a citation, so the claim to test is that it
478 // is a path someone can open. Two tiers, because the corpus no
479 // longer ships the file: the shape check runs everywhere (including
480 // from a registry copy), and the existence check — the one that
481 // would actually catch a generator writing `guides/…` again — runs
482 // whenever the checkout is there to check against.
483 let idx = index().unwrap();
484 let root = repo_root();
485 for chunk in &idx.chunks {
486 let valid_source = match chunk.kind.as_str() {
487 "guide" => {
488 chunk.path.starts_with("docs/")
489 || chunk.path == "crates/teksilo/src/app_guide.md"
490 }
491 "footer" => chunk.path.starts_with("docs/"),
492 "example" => chunk.path.starts_with("examples/"),
493 other => panic!("chunk {} has an unknown kind {other:?}", chunk.id),
494 };
495 assert!(
496 valid_source,
497 "chunk {} is a {} but its path is {:?}",
498 chunk.id, chunk.kind, chunk.path
499 );
500 assert!(
501 !chunk.path.contains("..") && !chunk.path.starts_with('/'),
502 "chunk {} path must be repo-relative, got {:?}",
503 chunk.id,
504 chunk.path
505 );
506 if let Some(root) = &root {
507 assert!(
508 root.join(&chunk.path).is_file(),
509 "chunk {} cites {}, which does not exist in this checkout",
510 chunk.id,
511 chunk.path
512 );
513 }
514 }
515 }
516
517 #[test]
518 fn df_is_non_empty() {
519 let idx = index().unwrap();
520 assert!(!idx.df.is_empty());
521 }
522
523 #[test]
524 fn no_chunk_has_empty_text() {
525 let idx = index().unwrap();
526 for chunk in &idx.chunks {
527 assert!(
528 !chunk.text().trim().is_empty(),
529 "chunk {} has empty text",
530 chunk.id
531 );
532 }
533 }
534
535 #[test]
536 fn chunk_text_looks_up_by_id() {
537 let idx = index().unwrap();
538 let first = &idx.chunks[0];
539 assert_eq!(idx.chunk_text(first.id), Some(first.text()));
540 assert_eq!(idx.chunk_text(usize::MAX), None);
541 assert!(idx.chunk(usize::MAX).is_none());
542 }
543
544 #[test]
545 fn every_chunk_line_range_is_ordered() {
546 // The half of the provenance claim that needs no checkout, and so no
547 // freshly regenerated corpus: it runs on every test run, unlike the
548 // checkout half below.
549 let idx = index().unwrap();
550 for chunk in &idx.chunks {
551 assert!(
552 chunk.line_start <= chunk.line_end,
553 "chunk {} has an inverted range {}..={}",
554 chunk.id,
555 chunk.line_start,
556 chunk.line_end
557 );
558 }
559 }
560
561 // Ignored on an ordinary test run; the release runs it.
562 //
563 // Its oracle is the checkout's own sources, and the corpus is regenerated
564 // only in the release commit (see the `pre-release-hook` in this crate's
565 // Cargo.toml). Between releases, every edit to an indexed document makes
566 // it fail, so on a pull request it reported the staleness the next release
567 // fixes rather than a fault in the change under review.
568 //
569 // The pre-release hook runs it by name, with `--ignored`, right after it
570 // regenerates the corpus: the one moment the claim must hold, and so the
571 // moment a failure means a real fault in the generator's line ranges.
572 // `build_corpus.py --check` does not cover it: that compares the corpus
573 // with the generator's output, never a chunk's range with the lines a
574 // reader would open. Renaming this test means renaming it in the hook,
575 // which otherwise matches nothing and passes.
576 #[test]
577 #[ignore = "needs a corpus regenerated from this checkout; the pre-release hook runs it"]
578 fn every_chunk_line_range_reproduces_its_own_text_in_a_checkout() {
579 // The provenance claim, checked the way a reader would check it:
580 // open `path`, take `line_start..=line_end`, expect this chunk. The
581 // range is no longer how the text is *stored*, which makes it
582 // exactly the kind of field that can rot unnoticed — so it gets a
583 // test rather than an assumption.
584 let idx = index().unwrap();
585 let Some(root) = repo_root() else {
586 return;
587 };
588 let mut sources: std::collections::HashMap<&str, String> = std::collections::HashMap::new();
589 for chunk in &idx.chunks {
590 let contents = sources
591 .entry(chunk.path.as_str())
592 .or_insert_with(|| std::fs::read_to_string(root.join(&chunk.path)).unwrap());
593 let count = chunk.line_end - chunk.line_start + 1;
594 let sliced: Vec<&str> = contents
595 .lines()
596 .skip(chunk.line_start)
597 .take(count)
598 .collect();
599 assert_eq!(
600 sliced.len(),
601 count,
602 "chunk {} ({}) claims lines {}..={} the file does not have",
603 chunk.id,
604 chunk.path,
605 chunk.line_start,
606 chunk.line_end
607 );
608 assert_eq!(
609 sliced.join("\n"),
610 chunk.text(),
611 "chunk {} ({}) lines {}..={} do not reproduce its stored text",
612 chunk.id,
613 chunk.path,
614 chunk.line_start,
615 chunk.line_end
616 );
617 }
618 }
619
620 #[test]
621 fn chunk_text_round_trips_a_guide_headings_own_line() {
622 // A guide chunk's range must start exactly at the ATX heading that
623 // named it. That pins the line range to the right place in the
624 // file, which re-slicing with the same code could not.
625 let idx = index().unwrap();
626 let mut checked = 0usize;
627 for chunk in &idx.chunks {
628 let Some(title) = chunk.heading_path.last() else {
629 continue;
630 };
631 if chunk.kind != "guide" {
632 continue;
633 }
634 let first = chunk.text().lines().next().expect("non-empty chunk");
635 assert!(
636 first.starts_with('#'),
637 "chunk {} ({}) should start at its heading, got {first:?}",
638 chunk.id,
639 chunk.path
640 );
641 assert!(
642 first.contains(title.as_str()),
643 "chunk {} ({}) heading line {first:?} does not contain {title:?}",
644 chunk.id,
645 chunk.path
646 );
647 checked += 1;
648 }
649 assert!(
650 checked > 100,
651 "expected many guide heading chunks, saw {checked}"
652 );
653 }
654
655 #[test]
656 fn terms_is_sorted_and_non_empty() {
657 let idx = index().unwrap();
658 assert!(!idx.terms.is_empty());
659 assert!(
660 idx.terms.windows(2).all(|w| w[0] < w[1]),
661 "terms must be sorted and duplicate-free"
662 );
663 // …and therefore binary-searchable.
664 let first = idx.terms.first().unwrap().clone();
665 assert_eq!(idx.term_index(&first), Some(0));
666 assert_eq!(idx.term_index("\u{1}not a corpus term"), None);
667 }
668
669 #[test]
670 fn every_token_key_indexes_a_real_term() {
671 let idx = index().unwrap();
672 for &term_idx in idx.df.keys() {
673 assert!(
674 idx.term(term_idx).is_some(),
675 "df key {term_idx} does not index a term (terms.len() = {})",
676 idx.terms.len()
677 );
678 }
679 for chunk in &idx.chunks {
680 for &term_idx in chunk.tokens.keys() {
681 assert!(
682 idx.term(term_idx).is_some(),
683 "chunk {} token key {term_idx} does not index a term",
684 chunk.id
685 );
686 }
687 assert_eq!(
688 idx.chunk_terms(chunk).len(),
689 chunk.tokens.len(),
690 "chunk {} lost terms resolving them to strings",
691 chunk.id
692 );
693 }
694 }
695
696 #[test]
697 fn base64_round_trips_every_byte_value() {
698 // 256 distinct bytes exercises all four residues of len % 3 as the
699 // slices below get truncated, and every symbol in the alphabet.
700 let all: Vec<u8> = (0..=255u8).collect();
701 for cut in [0usize, 1, 2, 3, 255, 256] {
702 let slice = &all[..cut];
703 let encoded = base64_i8::encode(slice);
704 assert_eq!(
705 encoded.len() % 4,
706 0,
707 "base64 must be padded to a multiple of 4"
708 );
709 assert_eq!(
710 base64_i8::decode(&encoded).unwrap(),
711 slice,
712 "round trip at len {cut}"
713 );
714 }
715 }
716
717 #[test]
718 fn base64_rejects_what_is_not_base64() {
719 assert!(base64_i8::decode("AAA").is_err(), "unpadded length");
720 assert!(base64_i8::decode("A!==").is_err(), "invalid character");
721 assert!(base64_i8::decode("A===").is_err(), "over-padded");
722 }
723
724 #[test]
725 fn an_embedding_round_trips_through_json_by_value() {
726 let original = ChunkEmbedding {
727 scale: 0.007_812_5,
728 values: vec![-128, -1, 0, 1, 127],
729 };
730 let json = serde_json::to_string(&original).unwrap();
731 assert!(
732 json.contains("\"q\":\""),
733 "values must serialize under `q`, got {json}"
734 );
735 assert!(
736 !json.contains('['),
737 "values must not serialize as an array, got {json}"
738 );
739 let back: ChunkEmbedding = serde_json::from_str(&json).unwrap();
740 assert_eq!(back, original);
741 assert_eq!(back.dim(), 5);
742 assert_eq!(back.dequantised()[4], 127.0 * 0.007_812_5);
743 }
744
745 #[test]
746 fn vectors_are_all_present_or_all_absent_and_agree_with_the_header() {
747 // Half a corpus of vectors would silently rank the vectorless half
748 // last forever, so the invariant is all-or-nothing.
749 let idx = index().unwrap();
750 let with_vectors = idx.chunks.iter().filter(|c| c.embedding.is_some()).count();
751 match (&idx.encoder, idx.encoder_dim) {
752 (None, None) => assert_eq!(with_vectors, 0, "vectors present with no encoder named"),
753 (Some(name), Some(dim)) => {
754 assert!(!name.is_empty());
755 assert_eq!(
756 with_vectors,
757 idx.chunks.len(),
758 "every chunk must carry a vector"
759 );
760 for chunk in &idx.chunks {
761 let e = chunk.embedding.as_ref().unwrap();
762 assert_eq!(
763 e.dim(),
764 dim as usize,
765 "chunk {} has the wrong dimension",
766 chunk.id
767 );
768 assert!(e.scale > 0.0, "chunk {} has a non-positive scale", chunk.id);
769 }
770 }
771 other => panic!("encoder and encoder_dim must be set together, got {other:?}"),
772 }
773 }
774
775 #[test]
776 fn document_frequency_resolves_by_string() {
777 let idx = index().unwrap();
778 // Pick a term the corpus certainly has, via the table itself, so
779 // this does not depend on any particular doc surviving an edit.
780 let sample = idx.terms.iter().max_by_key(|t| t.len()).unwrap();
781 let sample_idx = idx.term_index(sample).expect("term is in terms");
782 assert_eq!(idx.document_frequency(sample), idx.df[&sample_idx]);
783 assert!(idx.document_frequency(sample) > 0);
784 assert_eq!(idx.document_frequency("\u{1}not a corpus term"), 0);
785 }
786}