ingraph 0.1.0

A declaration-driven indexing framework: records and nodes are written as Rust types, and the storage rows, the dataflow between them and the GraphQL query surface are emitted from the declaration.
Documentation
//! THE FOUR FULL-TEXT FACES ON A REAL INDEX — a round trip through the blocking pair, and both modes
//! answering one question alike.
//!
//! Beside [`tests`](super::tests) and [`subset_tests`](super::subset_tests) rather than inside
//! either, on this module's own precedent: one file per thing being proved. What a face MEANS is the
//! seam's own doc; here there is a real directory, a real writer and a real searcher, so here is
//! where *the blocking face is the act and the async face is a wrapper over it* can be read rather
//! than claimed.
//!
//! # The both-faces-agree pair is the load-bearing one
//!
//! The async faces DELEGATE, so today the two roads share a body and cannot disagree. That is
//! exactly why the assertion is worth writing: an async face that grew a body of its own — a
//! `spawn_blocking`, a second parse, a differently-clamped ceiling — would still compile, still
//! type-check, and would answer a caller who chose the other mode differently, with nothing anywhere
//! to say so. One question asked down both roads is what makes that observable.

use core::{
  future::Future,
  num::NonZeroUsize,
  pin::pin,
  task::{Context, Poll, Waker},
};

use super::{
  AsyncFulltextRead, AsyncFulltextWrite, ColumnReader, Connected, FulltextEngine, FulltextRead,
  FulltextStore, FulltextSubset, FulltextWrite, Hits, TantivyAddress, TantivyDoc, TantivyFulltext,
  TantivyStore,
};

/// Drive a `ready` future to its value without a runtime — [`tests`](super::tests)' own, and for its
/// reason: every future this backend returns is already ready.
fn block_on<F: Future>(future: F) -> F::Output {
  let mut future = pin!(future);
  let mut cx = Context::from_waker(Waker::noop());
  loop {
    if let Poll::Ready(value) = future.as_mut().poll(&mut cx) {
      return value;
    }
  }
}

/// The arena every writer here is minted with — the engine's floor, which is all a test batch needs.
const HEAP: usize = TantivyStore::MIN_WRITER_HEAP_BYTES;

/// A `k` wider than anything these fixtures index, so what an assertion counts is the FACE's doing
/// and never the ceiling's.
const ROOMY: NonZeroUsize = NonZeroUsize::new(10).expect("10 is not zero");

/// The slot every search below runs against — a NAME rather than a declaration, this file having no
/// record to derive one from.
const SLOT: &str = "face_captions";

/// The source word every entry below enters under. One source into one slot, which is the shape the
/// identity's framing exists to keep injective when a second one arrives.
const SOURCE: &str = "captions";

/// A fresh store under a temporary root. The `TempDir` must OUTLIVE the store — its drop removes the
/// on-disk indices.
fn open_store() -> (tempfile::TempDir, TantivyStore) {
  let dir = tempfile::tempdir().expect("a temporary directory");
  let store = TantivyStore::open(dir.path()).expect("the store opens");
  (dir, store)
}

/// THE EMISSION's half, by hand: an id into the identity, a key into the stored address, a body to
/// tokenize. Both values are shipped rows, so this file declares no record faces at all.
fn entry(id: i64, body: &str) -> TantivyDoc {
  TantivyDoc::new(SOURCE)
    .with_identity(&id)
    .with_address(&id)
    .with_body(body)
}

/// Queue `entries` through the BLOCKING write face, commit, release the lock, then reload the slot's
/// reader — a batch's whole shape, collapsed. The writer is dropped BEFORE the reload so the slot's
/// lockfile is free for the next call.
fn commit_through_the_blocking_face(
  store: &TantivyStore,
  face: &TantivyFulltext,
  entries: Vec<TantivyDoc>,
) {
  {
    let mut writer = store.writer(SLOT, HEAP).expect("the slot's writer opens");
    for doc in entries {
      FulltextWrite::upsert(face, Connected::new(store, &mut writer), doc)
        .expect("the document queues");
    }
    writer.commit().expect("the batch commits");
  }
  store.reload(SLOT).expect("the slot's reader reloads");
}

/// How many documents the slot's committed snapshot holds — reached through the private cache
/// because a document COUNT is not the seam's business, and an assertion about a keyed replace has
/// nothing else to look at.
fn live_documents(store: &TantivyStore) -> u64 {
  store
    .slot_index(SLOT, false)
    .expect("the slot opens")
    .reader
    .searcher()
    .num_docs()
}

/// The stored address a hit came back under, read as the `i64` it was written from — the read
/// direction of the vocabulary, spelled here so an assertion names the entity rather than its bytes.
///
/// It drives the engine's own READ PHASE, which is what a walk does: `open` spends the address,
/// the column comes off the cursor, and `finish` says the key took all of it.
fn addressed(hits: &Hits<TantivyAddress, f32>) -> Vec<i64> {
  hits
    .iter()
    .map(|hit| {
      let mut read = TantivyFulltext::open(hit.data_ref().clone());
      let id = read.i64().expect("the stored address decodes");
      read.finish().expect("the key took every column");
      id
    })
    .collect()
}

/// THE ROUND TRIP, entirely on the blocking pair: an entry queued through [`FulltextWrite`] comes
/// back out of [`FulltextRead`] as the address it was written under, with the engine's BM25 score on
/// it.
///
/// No future is constructed anywhere in this test, which is the whole claim the blocking faces make:
/// the act is complete on this thread.
#[test]
fn a_document_written_through_the_blocking_face_is_found_by_it() {
  let (_dir, store) = open_store();
  let face = store.fulltext(SLOT);

  commit_through_the_blocking_face(
    &store,
    &face,
    vec![
      entry(1, "the quick brown fox"),
      entry(2, "a heron over still water"),
    ],
  );

  let hits = FulltextRead::search(
    &face,
    Connected::new(&store, ()),
    FulltextSubset::All,
    "fox",
    ROOMY,
    None,
  )
  .expect("the search runs");

  assert_eq!(addressed(&hits), vec![1]);
  // BM25 is natively higher-is-better, so a match scores ABOVE zero with no negation on the path —
  // which is the whole of what this face's score contract says, and the face did not change it.
  assert!(hits[0].score() > 0.0, "{hits:?}");
}

/// The keyed replace survives the face: two entries at ONE identity leave ONE document.
///
/// The write face forwards to the same `upsert_entry` the base verb does, and this is what holds
/// that in place — a face that queued an APPEND instead would pass every search assertion above and
/// only fail here.
#[test]
fn the_blocking_write_face_replaces_rather_than_appends() {
  let (_dir, store) = open_store();
  let face = store.fulltext(SLOT);

  commit_through_the_blocking_face(&store, &face, vec![entry(1, "a red car")]);
  commit_through_the_blocking_face(&store, &face, vec![entry(1, "a blue car")]);

  assert_eq!(live_documents(&store), 1);

  let hits = FulltextRead::search(
    &face,
    Connected::new(&store, ()),
    FulltextSubset::All,
    "blue",
    ROOMY,
    None,
  )
  .expect("the search runs");
  assert_eq!(addressed(&hits), vec![1]);
}

/// BOTH READ FACES ANSWER ONE QUESTION ALIKE — the same query, down both roads, hit for hit and
/// score for score.
#[test]
fn both_read_faces_answer_one_question_alike() {
  let (_dir, store) = open_store();
  let face = store.fulltext(SLOT);

  commit_through_the_blocking_face(
    &store,
    &face,
    vec![
      entry(1, "a red car on a red road"),
      entry(2, "a red car"),
      entry(3, "a heron over still water"),
    ],
  );

  let blocking = FulltextRead::search(
    &face,
    Connected::new(&store, ()),
    FulltextSubset::All,
    "red car",
    ROOMY,
    None,
  )
  .expect("the blocking search runs");

  let asynchronous = block_on(AsyncFulltextRead::search(
    &face,
    Connected::new(&store, ()),
    FulltextSubset::All,
    "red car",
    ROOMY,
    None,
  ))
  .expect("the async search runs");

  // A NON-EMPTY answer first: two faces agreeing on nothing would satisfy the equality below while
  // proving neither ran. WHICH entries matched is asserted as a set — the ranking between two
  // documents that both hold both terms is BM25's own arithmetic, and pinning it here would be this
  // test claiming something about the engine rather than about the faces.
  let mut matched = addressed(&blocking);
  matched.sort_unstable();
  assert_eq!(matched, vec![1, 2]);
  // ORDER-SENSITIVE between the two faces, which is the point: one query, two roads, the same hits
  // in the same order at the same scores.
  assert_eq!(blocking, asynchronous);
}

/// Both WRITE faces queue one entry alike: the async face's document is the blocking face's, so
/// writing the same identity down each road still leaves ONE document, and the last write is the one
/// that stands.
#[test]
fn both_write_faces_queue_one_entry_alike() {
  let (_dir, store) = open_store();
  let face = store.fulltext(SLOT);

  commit_through_the_blocking_face(&store, &face, vec![entry(7, "a red car")]);
  {
    let mut writer = store.writer(SLOT, HEAP).expect("the slot's writer opens");
    block_on(AsyncFulltextWrite::upsert(
      &face,
      Connected::new(&store, &mut writer),
      entry(7, "a blue car"),
    ))
    .expect("the document queues");
    writer.commit().expect("the batch commits");
  }
  store.reload(SLOT).expect("the slot's reader reloads");

  assert_eq!(live_documents(&store), 1);

  let hits = FulltextRead::search(
    &face,
    Connected::new(&store, ()),
    FulltextSubset::All,
    "blue",
    ROOMY,
    None,
  )
  .expect("the search runs");
  assert_eq!(addressed(&hits), vec![7]);

  // The replaced text is GONE from the index, not merely outranked — a keyed replace that had
  // appended would still answer this query with the stale document.
  let stale = FulltextRead::search(
    &face,
    Connected::new(&store, ()),
    FulltextSubset::All,
    "red",
    ROOMY,
    None,
  )
  .expect("the search runs");
  assert!(stale.is_empty(), "{stale:?}");
}