ingraph-derive 0.1.0

The attribute and derive macros for ingraph: the words a record, node, key, filter and vocabulary declaration is written in.
Documentation
//! THE FULL-TEXT CONVERSION GRAMMAR, read off an engine block — the five words, the road each one
//! settles, and every refusal they carry.
//!
//! Beside [`engines`](super::engines) rather than inside it, on that file's own precedent: one file
//! per thing being proved, so a reader looking for what `join = ", "` MEANS finds it whole. The
//! block's other half — which engines a field named, and in what order — stays there.
//!
//! # What is asserted here, and what is asserted elsewhere
//!
//! Here: the ROAD a declaration settles on, and the SENTENCE a refusal carries. What the road turns
//! into — the `FulltextRoad` impls, the demand that lands on a field's own type — is
//! `ingraph-codegen`'s `index/roads.rs`. Whether the two builtins actually segment differently, and
//! whether `stored` reaches tantivy's flag, is `ingraph`'s, where there is a sink and a live index
//! to read back.
//!
//! # The partition is stated in BOTH directions, every time
//!
//! The block is now half read and half carried, so a test that only checked *the framework word was
//! read* would pass for a walk that had swallowed the engine's knobs with it — and one that only
//! checked *the knobs are still carried* would pass for a walk that read nothing. Every accepting
//! test below therefore says what the read half took AND what the carried half kept.

use ingraph_codegen::{
  engine::Road,
  index::IndexModel,
  model::{Conversion, NodeKind},
};

use super::tests::{fulltext, refused};

/// A full-text document whose two fields wear the given marker lists verbatim.
fn document(title: &str, body: &str) -> syn::DeriveInput {
  let title: proc_macro2::TokenStream = title.parse().expect("a marker list");
  let body: proc_macro2::TokenStream = body.parse().expect("a marker list");
  syn::parse_quote! {
    #[ingraph(upstream(node = "asr::Segment", title = "speaker", body = "text"), dedup(hash))]
    pub struct SearchDocument {
      #[ingraph(#title)]
      title: Vec<String>,
      #[ingraph(#body)]
      body: String,
    }
  }
}

/// The `title` field's FIRST engine block, which every test here writes on `title`.
fn block(model: &IndexModel) -> &ingraph_codegen::engine::Engine {
  &model.engines[0].engines[0]
}

/// That block's carried half, as a string a test can look for a knob in.
fn carried(model: &IndexModel) -> String {
  block(model).options.to_string()
}

#[test]
fn a_block_that_declares_no_road_takes_the_identity_one() {
  let model = fulltext(&document("searchable, lexical", "searchable")).expect("accepted");
  assert_eq!(block(&model).road, Road::Identity);
  assert!(
    !block(&model).road.declared(),
    "saying nothing is not declaring the identity road, it is taking it"
  );
}

/// THE DEFAULT IS NOT-STORED, which is the ruled flip and the one this whole word exists to make
/// visible.
#[test]
fn a_block_keeps_nothing_unless_it_says_so() {
  let model = fulltext(&document("searchable, lexical", "searchable")).expect("accepted");
  assert!(!block(&model).stored);
}

#[test]
fn the_stored_marker_is_read_and_is_not_carried_to_the_engine() {
  let model = fulltext(&document(
    "searchable, lexical(stored, ef = 200)",
    "searchable",
  ))
  .expect("accepted");
  assert!(block(&model).stored, "the framework's word was read");
  assert!(
    !carried(&model).contains("stored"),
    "and taken out of what the engine is handed: {}",
    carried(&model)
  );
  assert!(
    carried(&model).contains("ef") && carried(&model).contains("200"),
    "while the engine's own knob rode through untouched: {}",
    carried(&model)
  );
}

#[test]
fn the_join_shorthand_carries_the_separator_the_declaration_chose() {
  let model = fulltext(&document(
    "searchable, lexical(join = \" / \", m = 16)",
    "searchable",
  ))
  .expect("accepted");
  match &block(&model).road {
    Road::Join(separator) => assert_eq!(separator.value(), " / "),
    other => panic!("expected the join road, got {other:?}"),
  }
  assert!(
    !carried(&model).contains("join"),
    "read, not carried: {}",
    carried(&model)
  );
  assert!(
    carried(&model).contains("16"),
    "and the engine's knob survived beside it: {}",
    carried(&model)
  );
}

#[test]
fn the_multi_shorthand_is_a_bare_word() {
  let model = fulltext(&document("searchable, lexical(multi)", "searchable")).expect("accepted");
  assert_eq!(block(&model).road, Road::Multi);
  assert!(
    carried(&model).is_empty(),
    "a block holding only framework words hands the engine nothing: {}",
    carried(&model)
  );
}

/// The two shorthands are ONE question's two answers, so the model must be able to tell them apart —
/// stated as an inequality, because a walk that read both as "a shorthand" would pass every test
/// above.
#[test]
fn the_two_shorthands_are_different_roads() {
  let joined = fulltext(&document(
    "searchable, lexical(join = \", \")",
    "searchable",
  ))
  .expect("accepted");
  let each = fulltext(&document("searchable, lexical(multi)", "searchable")).expect("accepted");
  assert_ne!(block(&joined).road, block(&each).road);
}

#[test]
fn a_declared_from_is_the_shipped_infallible_conversion() {
  let model = fulltext(&document(
    "searchable, lexical(from = \"text::render\")",
    "searchable",
  ))
  .expect("accepted");
  match &block(&model).road {
    Road::Declared(Conversion::Infallible(path)) => {
      assert!(quote::quote!(#path).to_string().contains("render"));
    }
    other => panic!("expected a declared infallible road, got {other:?}"),
  }
  assert!(!block(&model).road.fallible());
}

#[test]
fn a_declared_try_from_is_the_shipped_fallible_conversion() {
  let model = fulltext(&document(
    "searchable, lexical(try_from = \"text::render\")",
    "searchable",
  ))
  .expect("accepted");
  assert!(
    matches!(&block(&model).road, Road::Declared(Conversion::Fallible(_))),
    "got {:?}",
    block(&model).road
  );
  assert!(block(&model).road.fallible());
}

/// TWO ENGINES, TWO ROADS on ONE field — the case the per-block placement exists for, and the one a
/// field-level road could not express.
#[test]
fn one_field_may_take_a_different_road_into_each_engine() {
  let model = fulltext(&document(
    "searchable, lexical(multi), fts5(join = \", \")",
    "searchable",
  ))
  .expect("accepted");
  assert_eq!(model.engines[0].engines[0].road, Road::Multi);
  assert!(matches!(
    model.engines[0].engines[1].road,
    Road::Join(ref separator) if separator.value() == ", "
  ));
}

/// The words are the FRAMEWORK's wherever they are written, so a free engine word's block reads them
/// exactly as a shipped one's does — the open half of the vocabulary does not get a second grammar.
#[test]
fn a_free_engine_s_block_takes_the_same_five_words() {
  let model = fulltext(&document(
    "searchable, sonic(stored, multi, tuning = 3)",
    "searchable",
  ))
  .expect("accepted");
  assert!(block(&model).stored);
  assert_eq!(block(&model).road, Road::Multi);
  assert!(carried(&model).contains("tuning"));
}

#[test]
fn the_stored_marker_takes_no_block() {
  let refusal = refused(
    document("searchable, lexical(stored(compressed))", "searchable"),
    NodeKind::Fulltext,
  );
  assert!(
    refusal.contains("is a BARE marker"),
    "the refusal says which shape it takes: {refusal}"
  );
}

#[test]
fn the_multi_shorthand_takes_no_value() {
  let refusal = refused(
    document("searchable, lexical(multi = \", \")", "searchable"),
    NodeKind::Fulltext,
  );
  assert!(
    refusal.contains("is a BARE marker"),
    "the refusal says which shape it takes: {refusal}"
  );
  assert!(
    refusal.contains("`join = \"<separator>\"`"),
    "and points at the road that DOES take one, which is the mistake actually being made: {refusal}"
  );
}

/// `join` BARE says nothing — a separator has no default, because whichever one the framework picked
/// would be a silent choice about what a phrase can match across.
#[test]
fn the_join_shorthand_needs_the_separator_it_joins_with() {
  let refusal = refused(
    document("searchable, lexical(join)", "searchable"),
    NodeKind::Fulltext,
  );
  assert!(
    refusal.contains("takes the SEPARATOR"),
    "the refusal names what is missing: {refusal}"
  );
}

#[test]
fn a_block_declares_one_road_and_the_two_shorthands_are_two() {
  let refusal = refused(
    document("searchable, lexical(join = \", \", multi)", "searchable"),
    NodeKind::Fulltext,
  );
  assert!(
    refusal.contains("two answers to ONE question"),
    "the refusal says why they cannot both be written: {refusal}"
  );
  assert!(
    refusal.contains("ACROSS"),
    "and what the question is: {refusal}"
  );
}

#[test]
fn a_shorthand_and_a_declared_conversion_are_two_roads_as_well() {
  let refusal = refused(
    document(
      "searchable, lexical(multi, from = \"text::render\")",
      "searchable",
    ),
    NodeKind::Fulltext,
  );
  assert!(
    refusal.contains("a slot takes ONE road"),
    "the refusal states the rule: {refusal}"
  );
  assert!(
    refusal.contains("a conversion the framework wrote out"),
    "and says what a shorthand IS, which is why the pair is redundant rather than complementary: \
     {refusal}"
  );
}

/// THE LEG THIS FACE DOES NOT HAVE. Refused by name rather than as an unknown entry, for the
/// courtesy every misplaced word in this pass gets: the author had a coherent idea about a
/// projection that is read back, and a document is not one.
#[test]
fn a_document_has_no_into_leg() {
  for leg in ["into", "try_into"] {
    let refusal = refused(
      document(
        &format!("searchable, lexical({leg} = \"text::render\")"),
        "searchable",
      ),
      NodeKind::Fulltext,
    );
    assert!(
      refusal.contains("never left"),
      "the refusal says why there is no leg: {refusal}"
    );
    assert!(
      refusal.contains("`from = \"path::to::fn\"`"),
      "and offers the one direction there is: {refusal}"
    );
  }
}

/// `from` AND `try_from` in one block is the SHIPPED refusal, reached through the shipped reader —
/// which is the whole point of not writing a second one.
#[test]
fn a_road_either_can_fail_or_it_cannot() {
  let refusal = refused(
    document(
      "searchable, lexical(from = \"text::render\", try_from = \"text::try_render\")",
      "searchable",
    ),
    NodeKind::Fulltext,
  );
  assert!(
    refusal.contains("there is no third answer"),
    "the shipped sentence, unrewritten: {refusal}"
  );
}

/// THE WORDS ARE FULL-TEXT'S. On the vector kind they are refused BY NAME, because an author who
/// wrote one had a coherent idea about the wrong kind of index.
#[test]
fn the_conversion_words_belong_to_the_full_text_kind() {
  for spelling in ["stored", "multi", "join = \", \"", "from = \"f\""] {
    let written: proc_macro2::TokenStream = spelling.parse().expect("a block entry");
    let refusal = refused(
      syn::parse_quote! {
        #[ingraph(upstream(node = "asr::Segment", image = "coords"), dedup(hash))]
        pub struct Frames {
          #[ingraph(embedding, knn, lance(#written))]
          image: Embedding<f32, 4>,
        }
      },
      NodeKind::Vector,
    );
    assert!(
      refusal.contains("FULL-TEXT conversion grammar"),
      "the refusal names the kind the words serve: {refusal}"
    );
    assert!(
      refusal.contains("coordinate"),
      "and why this kind has no use for them: {refusal}"
    );
  }
}

/// The refusal above must not have been bought by refusing the engine's OWN knobs on the vector
/// kind — the open half is untouched there, which is what this states.
#[test]
fn a_vector_engine_still_carries_its_own_knobs() {
  let model = {
    let input: syn::DeriveInput = syn::parse_quote! {
      #[ingraph(upstream(node = "asr::Segment", image = "coords"), dedup(hash))]
      pub struct Frames {
        #[ingraph(embedding, knn, hnsw(ef = 200, m = 16))]
        image: Embedding<f32, 4>,
      }
    };
    let (host, arguments) = crate::host::split(&input);
    super::super::index::parse(&host, &arguments, NodeKind::Vector).expect("accepted")
  };
  let options = model.engines[0].engines[0].options.to_string();
  assert!(
    options.contains("ef") && options.contains("200"),
    "{options}"
  );
  assert_eq!(model.engines[0].engines[0].road, Road::Identity);
}