ingraph 0.1.0

A declaration-driven indexing framework: records and nodes are written as Rust types, and the storage rows, the dataflow between them and the GraphQL query surface are emitted from the declaration.
Documentation
//! THE WHOLE PIPELINE, ON A REAL INDEX — one round trip through the shape the engine cutover left,
//! with nothing stubbed and nothing shortened.
//!
//! Every other test in this directory exercises a piece: the codec on its own, the faces on their
//! own, the vocabulary against the phases. This one runs the pieces IN ORDER, on a live tantivy
//! directory, and asserts the two things only the whole path can say:
//!
//! ```text
//! IN   Field::write ─► WriteBuffer ─► finalize ─► Address ─► TantivyDoc ─► FulltextWrite::upsert
//! OUT  FulltextRead::search ─► Address ─► open ─► ReadBuffer ─► Field::read ─► finish
//! ```
//!
//! - **THE SAME-BYTES LAW.** The address the write walk sealed and the address the hit arrived under
//!   are the same bytes for the same key. Everything the by-key half of a retrieve stands on is that
//!   sentence, and no unit test can make it: the write phase and the read phase are joined by a
//!   tantivy segment in between.
//! - **THE KEY COMES BACK.** A key with THREE columns of three different kinds — text, integer,
//!   uuid — is reported in, and the same value is taken back off the hit. A row that landed the
//!   wrong kind, a phase that framed a width differently, or a search that handed back another
//!   entry's stored bytes all fail here.
//!
//! The faces are the BLOCKING pair, which is the honest mode for this backend and the one that makes
//! the test a straight line: no runtime, no `block_on`, no future constructed anywhere.

use core::num::NonZeroUsize;

use uuid::Uuid;

use super::{
  ColumnReader, ColumnWriter, Connected, Field, FulltextEngine, FulltextRead, FulltextStore,
  FulltextSubset, FulltextWrite, TantivyDoc, TantivyFulltext, TantivyStore,
};

/// The slot this file's whole corpus lives in.
const SLOT: &str = "captions";

/// The source word every entry enters under — the identity's first column.
const SOURCE: &str = "captions";

/// Wider than the corpus, so what an assertion counts is the pipeline's doing rather than a ceiling.
const ROOMY: NonZeroUsize = NonZeroUsize::new(10).expect("10 is not zero");

/// A key with ONE COLUMN OF EACH SHAPE the language draws a real distinction between: a
/// variable-width text, a native integer, and a uuid the engine has its own column for.
///
/// Three kinds rather than three of one, because what a round trip has to survive is the SEQUENCE:
/// a phase that framed the uuid at the text's width, or a row that read them back out of order,
/// produces a key that decodes to something — and this is where that something is caught.
#[derive(Debug, Clone, PartialEq, Eq)]
struct CaptionKey {
  media: String,
  frame: i64,
  run: Uuid,
}

impl Field for CaptionKey {
  fn write(&self, w: &mut impl ColumnWriter) {
    self.media.write(w);
    self.frame.write(w);
    self.run.write(w);
  }

  fn read<R: ColumnReader>(r: &mut R) -> Result<Self, R::Error> {
    Ok(Self {
      media: String::read(r)?,
      frame: i64::read(r)?,
      run: Uuid::read(r)?,
    })
  }
}

fn key(media: &str, frame: i64) -> CaptionKey {
  CaptionKey {
    media: media.to_owned(),
    frame,
    run: Uuid::from_u128(0x9f1c_2d3e_4a5b_6c7d_8e9f_a0b1_c2d3_e4f5),
  }
}

/// THE WRITE WALK, spelled out — the three phases the framework drives, run by hand so the test
/// names each one.
///
/// It is what `TantivyDoc::with_address` does inside itself, and
/// [`the_builder_seals_the_same_address_the_walk_does`] is what holds the two together.
fn sealed(key: &CaptionKey) -> <TantivyFulltext as FulltextEngine>::Address {
  let mut buf = TantivyFulltext::buffer();
  key.write(&mut buf);
  TantivyFulltext::finalize(buf)
}

/// THE READ WALK, spelled out — `open` spends the address, the key takes its columns, and `finish`
/// asks whether it took all of them.
fn taken(
  address: <TantivyFulltext as FulltextEngine>::Address,
) -> Result<CaptionKey, super::TantivyFulltextError> {
  let mut phase = TantivyFulltext::open(address);
  let key = CaptionKey::read(&mut phase)?;
  phase.finish()?;
  Ok(key)
}

/// One entry, as the EMISSION builds it: the source word, the id into the identity, the key into the
/// stored address, the text into the body.
fn entry(key: &CaptionKey, body: &str) -> TantivyDoc {
  TantivyDoc::new(SOURCE)
    .with_identity(&key.frame)
    .with_address(key)
    .with_body(body)
}

/// A store under a temporary root with `corpus` committed into `SLOT`, its reader reloaded.
///
/// The `TempDir` must OUTLIVE the store — its drop removes the on-disk index — so it comes back with
/// it.
fn indexed(corpus: &[(CaptionKey, &str)]) -> (tempfile::TempDir, TantivyStore, TantivyFulltext) {
  let dir = tempfile::tempdir().expect("a temporary directory");
  let store = TantivyStore::open(dir.path()).expect("the store opens");
  let face = FulltextStore::fulltext(&store, SLOT);
  {
    let mut writer = store
      .writer(SLOT, TantivyStore::MIN_WRITER_HEAP_BYTES)
      .expect("the slot's writer opens");
    for (key, body) in corpus {
      // THE WRITE FACE, blocking: no future is constructed here at all.
      FulltextWrite::upsert(&face, Connected::new(&store, &mut writer), entry(key, body))
        .expect("the entry queues");
    }
    writer.commit().expect("the batch commits");
  }
  store.reload(SLOT).expect("the slot's reader reloads");
  (dir, store, face)
}

/// THE ROUND TRIP: a key reported in through the column language comes back out of a live search as
/// itself, at the same bytes.
#[test]
fn a_key_reported_in_columns_comes_back_off_a_live_search() {
  let fox = key("m1", 1);
  let heron = key("m2", 2);
  let (_dir, store, face) = indexed(&[
    (fox.clone(), "the quick brown fox"),
    (heron.clone(), "a heron over still water"),
  ]);

  let hits = FulltextRead::search(
    &face,
    Connected::new(&store, ()),
    FulltextSubset::All,
    "fox",
    ROOMY,
    None,
  )
  .expect("the search runs");

  assert_eq!(hits.len(), 1, "{hits:?}");
  // THE SAME-BYTES LAW: what the write walk sealed is what the hit arrived under.
  assert_eq!(
    hits[0].data_ref(),
    &sealed(&fox),
    "the stored address is not the one the write walk sealed",
  );
  // AND THE KEY COMES BACK — all three columns, in order, off the engine's own read phase.
  assert_eq!(
    taken(hits[0].data_ref().clone()).expect("the address reads back"),
    fox
  );
  // BM25 is natively higher-is-better, so a match scores above zero with no negation on the path.
  assert!(hits[0].score() > 0.0, "{hits:?}");

  // The other entry is reachable too, which is what keeps the assertion above from passing on an
  // index that holds one document however many were written.
  let hits = FulltextRead::search(
    &face,
    Connected::new(&store, ()),
    FulltextSubset::All,
    "heron",
    ROOMY,
    None,
  )
  .expect("the search runs");
  assert_eq!(hits.len(), 1, "{hits:?}");
  assert_eq!(
    taken(hits[0].data_ref().clone()).expect("the address reads back"),
    heron
  );
}

/// The ENTRY BUILDER and the framework's write walk seal the SAME address — which is what lets a
/// caller build a `TantivyDoc` and a retrieve read its hits back with no third party joining them.
///
/// It is the one place the builder's own `buffer` → `write` is checked against the three functions
/// spelled out, so a builder that started appending to something other than a fresh phase — or that
/// re-encoded on the way in — is caught here rather than at a search that finds nothing.
#[test]
fn the_builder_seals_the_same_address_the_walk_does() {
  let key = key("m1", 7);
  assert_eq!(
    entry(&key, "a red car").address().as_slice(),
    sealed(&key).as_slice()
  );
}

/// A KEY THAT READS FEWER COLUMNS THAN THE ADDRESS HOLDS is refused at `finish`, on a stored address
/// that came off a real search rather than one assembled in a test.
///
/// This is the surplus check earning its place end to end: the address below is a live hit's, and
/// the key reading it is a shorter one from another era of the same declaration.
#[test]
fn a_stored_address_the_key_does_not_finish_is_refused() {
  /// The same key with its last column dropped — a schema that lost a member.
  #[derive(Debug, PartialEq, Eq)]
  struct Shorter {
    media: String,
    frame: i64,
  }

  impl Field for Shorter {
    fn write(&self, w: &mut impl ColumnWriter) {
      self.media.write(w);
      self.frame.write(w);
    }

    fn read<R: ColumnReader>(r: &mut R) -> Result<Self, R::Error> {
      Ok(Self {
        media: String::read(r)?,
        frame: i64::read(r)?,
      })
    }
  }

  let (_dir, store, face) = indexed(&[(key("m1", 1), "the quick brown fox")]);
  let hits = FulltextRead::search(
    &face,
    Connected::new(&store, ()),
    FulltextSubset::All,
    "fox",
    ROOMY,
    None,
  )
  .expect("the search runs");

  let mut phase = TantivyFulltext::open(hits[0].data_ref().clone());
  assert_eq!(
    Shorter::read(&mut phase).expect("the two columns it does read are there"),
    Shorter {
      media: "m1".to_owned(),
      frame: 1,
    }
  );
  assert!(
    matches!(
      phase.finish(),
      Err(super::TantivyFulltextError::Key(
        super::KeyDecodeError::Trailing
      ))
    ),
    "a key that read short must be refused rather than served",
  );
}