ingraph 0.1.0

A declaration-driven indexing framework: records and nodes are written as Rust types, and the storage rows, the dataflow between them and the GraphQL query surface are emitted from the declaration.
Documentation
use super::{ScriptRun, script_runs};

use crate::WritingSystem;

/// The runs of `text`, as `(script, offset, text)` — the shape every assertion below reads.
fn cut(text: &str) -> Vec<(WritingSystem, usize, &str)> {
  script_runs(text).map(ScriptRun::into_components).collect()
}

/// **THE WORKED EXAMPLE**: the digits and both spaces are Common, so they ride the Latin word that
/// precedes them, and the cut lands exactly where the writing system changes.
#[test]
fn the_worked_example_cuts_where_the_script_changes() {
  assert_eq!(
    cut("Rust 2024 年鉴"),
    vec![
      (WritingSystem::Latn, 0, "Rust 2024 "),
      (WritingSystem::Hani, 10, "年鉴"),
    ]
  );
}

/// **RULE 1**: a contextual character continues the run it is in, so a trailing space belongs to the
/// word before it and not to the one after.
#[test]
fn preceding_wins_over_following() {
  assert_eq!(
    cut("日本 word"),
    vec![
      (WritingSystem::Hani, 0, "日本 "),
      (WritingSystem::Latn, 7, "word")
    ]
  );
  assert_eq!(
    cut("word 日本"),
    vec![
      (WritingSystem::Latn, 0, "word "),
      (WritingSystem::Hani, 5, "日本")
    ]
  );
}

/// **RULE 2**: at the head there is nothing to continue, so the leading contextual stretch joins
/// what follows rather than minting a run of its own.
#[test]
fn a_leading_contextual_stretch_joins_what_follows() {
  assert_eq!(cut(" 年鉴"), vec![(WritingSystem::Hani, 0, " 年鉴")]);
  assert_eq!(
    cut("  \t\"hello\""),
    vec![(WritingSystem::Latn, 0, "  \t\"hello\"")]
  );
  assert_eq!(
    cut("«Привет» hello"),
    vec![
      (WritingSystem::Cyrl, 0, "«Привет» "),
      (WritingSystem::Latn, 17, "hello"),
    ]
  );
}

/// **RULE 3**: a text with no concrete script anywhere is one run in the fallback bucket.
#[test]
fn a_wholly_contextual_text_is_one_fallback_run() {
  assert_eq!(cut("123 !!!"), vec![(WritingSystem::Zzzz, 0, "123 !!!")]);
  assert_eq!(cut("   "), vec![(WritingSystem::Zzzz, 0, "   ")]);
  assert_eq!(cut("+-*/"), vec![(WritingSystem::Zzzz, 0, "+-*/")]);
  // A lone combining mark has nothing to inherit from, and lands in the same bucket.
  assert_eq!(cut("\u{0301}"), vec![(WritingSystem::Zzzz, 0, "\u{0301}")]);
}

/// **AN UNROSTERED SCRIPT CUTS ITS OWN RUN** rather than being swallowed — the property the whole
/// table's shape exists for.
#[test]
fn an_unrostered_script_is_never_swallowed() {
  assert_eq!(
    cut("word ᚠᚢᚦ word"),
    vec![
      (WritingSystem::Latn, 0, "word "),
      (WritingSystem::Zzzz, 5, "ᚠᚢᚦ "),
      (WritingSystem::Latn, 15, "word"),
    ]
  );
  // Armenian is a ROSTERED script since the sweep, so this pair names it rather than the bucket.
  // The property is the same one notch along: a script the roster does not carry cuts its own run,
  // and a script it DOES carry cuts its own run under its own name.
  assert_eq!(
    cut("中文 Հայերեն"),
    vec![
      (WritingSystem::Hani, 0, "中文 "),
      (WritingSystem::Armn, 7, "Հայերեն"),
    ]
  );
}

/// A combining mark is INHERITED, so an accented word is one Latin run and not three.
#[test]
fn combining_marks_do_not_cut() {
  let decomposed = "cafe\u{0301}";
  assert_eq!(cut(decomposed), vec![(WritingSystem::Latn, 0, decomposed)]);
  // The composed spelling is one run too, which is the same answer by a different road.
  assert_eq!(cut("café"), vec![(WritingSystem::Latn, 0, "café")]);
}

/// **JAPANESE CUTS INTO THREE**, faithfully — putting them back together is a router's policy and
/// not a segmenter's answer.
#[test]
fn japanese_cuts_into_its_three_scripts() {
  assert_eq!(
    cut("日本語のテキスト"),
    vec![
      (WritingSystem::Hani, 0, "日本語"),
      (WritingSystem::Hira, 9, "の"),
      (WritingSystem::Kana, 12, "テキスト"),
    ]
  );
}

/// **EMPTY IN, EMPTY OUT** — no run is ever zero bytes, so there is no empty run to represent it.
#[test]
fn an_empty_text_has_no_runs() {
  assert_eq!(cut(""), vec![]);
  assert_eq!(script_runs("").count(), 0);
}

/// The three promises, over a corpus that exercises every rule at once.
#[test]
fn the_runs_are_total_contiguous_and_non_empty() {
  for text in CORPUS {
    let runs: Vec<_> = script_runs(text).collect();
    let mut at = 0;

    for run in &runs {
      assert_eq!(run.at(), at, "a gap or an overlap in {text:?}");
      assert!(!run.text().is_empty(), "an empty run in {text:?}");
      assert_eq!(run.text(), &text[run.at()..run.end()], "in {text:?}");
      at = run.end();
    }

    assert_eq!(at, text.len(), "the tail of {text:?} is uncovered");
    let rebuilt: String = runs.iter().map(|run| run.text()).collect();
    assert_eq!(rebuilt, *text, "the runs do not rebuild {text:?}");
  }
}

/// **DETERMINISM**: the same text cuts the same way, every time.
#[test]
fn the_same_text_cuts_the_same_way() {
  for text in CORPUS {
    let once: Vec<_> = script_runs(text).collect();
    let twice: Vec<_> = script_runs(text).collect();
    assert_eq!(once, twice, "at {text:?}");
  }
}

/// **A RUN IS MAXIMAL**: no two neighbours share a script, or the cut between them would be one this
/// function invented.
#[test]
fn neighbouring_runs_never_share_a_script() {
  for text in CORPUS {
    let runs: Vec<_> = script_runs(text).collect();
    for pair in runs.windows(2) {
      assert_ne!(
        pair[0].script(),
        pair[1].script(),
        "an unnecessary cut in {text:?}"
      );
    }
  }
}

/// The iterator is FUSED, and spent means spent.
#[test]
fn a_spent_iterator_stays_spent() {
  let mut runs = script_runs("hi");
  assert!(runs.next().is_some());
  assert!(runs.next().is_none());
  assert!(runs.next().is_none());
}

/// The hint is a bound rather than a promise, and it bounds what is actually produced.
#[test]
fn the_size_hint_bounds_the_answer() {
  for text in CORPUS {
    let (low, high) = script_runs(text).size_hint();
    let produced = script_runs(text).count();
    assert!(produced >= low, "at {text:?}");
    assert!(
      produced <= high.expect("the hint has an upper bound"),
      "at {text:?}"
    );
  }
}

/// A run states its own span, and the two ends agree with the slice between them.
#[test]
fn a_run_states_its_own_span() {
  let run = ScriptRun::new(WritingSystem::Hani, 10, "年鉴");
  assert_eq!(run.script(), &WritingSystem::Hani);
  assert_eq!(run.at(), 10);
  assert_eq!(run.text(), "年鉴");
  assert_eq!(run.end(), 16);
  assert_eq!(
    run.clone().into_components(),
    (WritingSystem::Hani, 10, "年鉴")
  );
}

/// **CONCATENATION DEGRADES SANELY**: cutting `a` then `b` and cutting `a + b` differ only where the
/// join itself is a boundary — the run count never grows, and every byte is still covered once.
#[test]
fn a_concatenation_covers_what_its_halves_did() {
  for left in CORPUS {
    for right in CORPUS {
      let joined = format!("{left}{right}");
      let apart = script_runs(left).count() + script_runs(right).count();
      let together = script_runs(&joined).count();

      assert!(
        together <= apart,
        "joining {left:?} and {right:?} invented a boundary"
      );
      let rebuilt: String = script_runs(&joined)
        .map(|run| run.text().to_owned())
        .collect();
      assert_eq!(rebuilt, joined);
    }
  }
}

/// The texts every property above is measured over — one per edge the segmenter has.
const CORPUS: &[&str] = &[
  "",
  " ",
  "a",
  "hello world",
  "Rust 2024 年鉴",
  "123 !!!",
  "日本語のテキスト",
  "中文 English 混排",
  " 年鉴",
  "cafe\u{0301} 中文",
  "Привет, мир! Hello, world!",
  "مرحبا بالعالم hello",
  "שלום עולם",
  "Ελληνικά κείμενο",
  "हिन्दी पाठ",
  "ภาษาไทย",
  "한국어 텍스트",
  "word ᚠᚢᚦ word",
  "🎉 emoji 🎉 中文 🎉",
  "\u{0301}",
  "アイウ ABC",
  "a\u{200D}b",
  "𠀀𠀁 surrogate-free astral han",
  // One row per class the sweep added: no-space, Brahmic cluster, own-casing, own-joining.
  "ភាសាខ្មែរ ພາສາລາວ ဗမာစာ བོད་སྐད།",
  "বাংলা ਪੰਜਾਬੀ ગુજરાતી ଓଡ଼ିଆ தமிழ் తెలుగు ಕನ್ನಡ മലയാളം සිංහල",
  "Հայերեն ქართული ܣܘܪܝܝܐ ދިވެހި ᠮᠣᠩᠭᠣᠯ ᐃᓄᒃᑎᑐᑦ አማርኛ ㄅㄆㄇ",
];