Struct VectorStoreWriter 

pub struct VectorStoreWriter { /* private fields */ }

Stages appends, deletes, and compactions of one store, and publishes them together on commit.

Nothing a reader sees changes before a commit. Appending a document that is already present replaces it; deleting one that is not present does nothing. A commit with nothing staged still advances the time readers report as fresh, so an idle writer can keep a store within a reader’s allowed lag. One writer writes a store at a time.

Implementations

impl VectorStoreWriter

pub fn create( path: impl AsRef<Path>, encoding: Encoding, corpus: impl Into<String>, representation: Representation, ) -> Result<Self>

Writes an empty store at path, which must not exist.

Examples found in repository

examples/stored_maxsim.rs (line 84)

fn main() -> Result<()> {
    let representation = Representation::new("toy-encoder", "1", DIMENSION, true)?;
    let directory = std::env::temp_dir().join(format!("lateweave-example-{}", std::process::id()));
    let path = directory.join("vectors");

    // The indexing side: one token per word, the axis standing in for what
    // a real encoder produces.
    let mut writer =
        VectorStoreWriter::create(&path, Encoding::Float32, "fruit", representation.clone())?;
    let embeddings = [unit(0), unit(2), unit(1), unit(2), unit(0), unit(3)].concat();
    writer.append(["a", "b", "c"], &embeddings, DIMENSION, &[2, 2, 2], None)?;
    writer.commit()?;

    // The serving side.
    let gatherer = TermGatherer {
        requires: BTreeMap::new(),
        documents: vec![("a", "red apple"), ("b", "green apple"), ("c", "red car")],
        indexed_at: SystemTime::now(),
    };
    let store = Arc::new(VectorStore::open(&path)?) as Arc<dyn MultiVectorSource>;
    let reranker = MaxSimReranker::new([store], DEFAULT_FEATURE)?;
    let pipeline = SearchPipeline::new(Arc::new(gatherer), Some(Arc::new(reranker)));

    let query = Query::new("apple").with_feature(
        DEFAULT_FEATURE,
        Feature::lazy(representation, || {
            TokenMatrix::new(unit(1).to_vec(), DIMENSION)
        }),
    );
    let request = SearchRequest::new(10, 2).with_budget(ResourceBudget::new(4096, 64, Some(1))?);

    let print = |label: &str| -> Result<()> {
        let result = pipeline.search(&query, &request)?;
        println!("{label}:");
        for document in &result.documents {
            println!(
                "  #{} {:?} score {}",
                document.rank, document.key, document.score
            );
        }
        println!("  {:?}", result.diagnostics);
        Ok(())
    };
    print("before the delete")?;

    // The indexing side deletes "b" from the vectors; the gatherer still
    // finds it, and the pipeline drops it.
    writer.delete(["b"]);
    writer.commit()?;
    print("after the delete")?;

    std::fs::remove_dir_all(directory)?;
    Ok(())
}

pub fn open(path: impl AsRef<Path>) -> Result<Self>

pub fn path(&self) -> &Path

pub fn corpus(&self) -> &str

pub fn encoding(&self) -> Encoding

pub fn representation(&self) -> &Representation

pub fn append<I>( &mut self, document_ids: I, embeddings: &[f32], dimension: usize, lengths: &[usize], threads: Option<usize>, ) -> Result<()>

where I: IntoIterator, I::Item: Into<Arc<str>>,

Stages documents as a new segment. embeddings is a row-major float32 [sum(lengths), dimension] matrix packed by document in the order of document_ids.

Examples found in repository

examples/stored_maxsim.rs (line 86)

fn main() -> Result<()> {
    let representation = Representation::new("toy-encoder", "1", DIMENSION, true)?;
    let directory = std::env::temp_dir().join(format!("lateweave-example-{}", std::process::id()));
    let path = directory.join("vectors");

    // The indexing side: one token per word, the axis standing in for what
    // a real encoder produces.
    let mut writer =
        VectorStoreWriter::create(&path, Encoding::Float32, "fruit", representation.clone())?;
    let embeddings = [unit(0), unit(2), unit(1), unit(2), unit(0), unit(3)].concat();
    writer.append(["a", "b", "c"], &embeddings, DIMENSION, &[2, 2, 2], None)?;
    writer.commit()?;

    // The serving side.
    let gatherer = TermGatherer {
        requires: BTreeMap::new(),
        documents: vec![("a", "red apple"), ("b", "green apple"), ("c", "red car")],
        indexed_at: SystemTime::now(),
    };
    let store = Arc::new(VectorStore::open(&path)?) as Arc<dyn MultiVectorSource>;
    let reranker = MaxSimReranker::new([store], DEFAULT_FEATURE)?;
    let pipeline = SearchPipeline::new(Arc::new(gatherer), Some(Arc::new(reranker)));

    let query = Query::new("apple").with_feature(
        DEFAULT_FEATURE,
        Feature::lazy(representation, || {
            TokenMatrix::new(unit(1).to_vec(), DIMENSION)
        }),
    );
    let request = SearchRequest::new(10, 2).with_budget(ResourceBudget::new(4096, 64, Some(1))?);

    let print = |label: &str| -> Result<()> {
        let result = pipeline.search(&query, &request)?;
        println!("{label}:");
        for document in &result.documents {
            println!(
                "  #{} {:?} score {}",
                document.rank, document.key, document.score
            );
        }
        println!("  {:?}", result.diagnostics);
        Ok(())
    };
    print("before the delete")?;

    // The indexing side deletes "b" from the vectors; the gatherer still
    // finds it, and the pipeline drops it.
    writer.delete(["b"]);
    writer.commit()?;
    print("after the delete")?;

    std::fs::remove_dir_all(directory)?;
    Ok(())
}

pub fn delete<I>(&mut self, document_ids: I) -> usize

where I: IntoIterator, I::Item: AsRef<str>,

Stages the deletion of every present document among document_ids, returning how many there were.

Examples found in repository

examples/stored_maxsim.rs (line 123)

fn main() -> Result<()> {
    let representation = Representation::new("toy-encoder", "1", DIMENSION, true)?;
    let directory = std::env::temp_dir().join(format!("lateweave-example-{}", std::process::id()));
    let path = directory.join("vectors");

    // The indexing side: one token per word, the axis standing in for what
    // a real encoder produces.
    let mut writer =
        VectorStoreWriter::create(&path, Encoding::Float32, "fruit", representation.clone())?;
    let embeddings = [unit(0), unit(2), unit(1), unit(2), unit(0), unit(3)].concat();
    writer.append(["a", "b", "c"], &embeddings, DIMENSION, &[2, 2, 2], None)?;
    writer.commit()?;

    // The serving side.
    let gatherer = TermGatherer {
        requires: BTreeMap::new(),
        documents: vec![("a", "red apple"), ("b", "green apple"), ("c", "red car")],
        indexed_at: SystemTime::now(),
    };
    let store = Arc::new(VectorStore::open(&path)?) as Arc<dyn MultiVectorSource>;
    let reranker = MaxSimReranker::new([store], DEFAULT_FEATURE)?;
    let pipeline = SearchPipeline::new(Arc::new(gatherer), Some(Arc::new(reranker)));

    let query = Query::new("apple").with_feature(
        DEFAULT_FEATURE,
        Feature::lazy(representation, || {
            TokenMatrix::new(unit(1).to_vec(), DIMENSION)
        }),
    );
    let request = SearchRequest::new(10, 2).with_budget(ResourceBudget::new(4096, 64, Some(1))?);

    let print = |label: &str| -> Result<()> {
        let result = pipeline.search(&query, &request)?;
        println!("{label}:");
        for document in &result.documents {
            println!(
                "  #{} {:?} score {}",
                document.rank, document.key, document.score
            );
        }
        println!("  {:?}", result.diagnostics);
        Ok(())
    };
    print("before the delete")?;

    // The indexing side deletes "b" from the vectors; the gatherer still
    // finds it, and the pipeline drops it.
    writer.delete(["b"]);
    writer.commit()?;
    print("after the delete")?;

    std::fs::remove_dir_all(directory)?;
    Ok(())
}

pub fn compact(&mut self) -> Result<()>

Stages the store’s present documents as one segment, dropping deleted and replaced rows.

pub fn commit(&mut self) -> Result<SystemTime>

Publishes everything staged, and returns the commit time readers report for it. Files no longer named are then removed.

Examples found in repository

examples/stored_maxsim.rs (line 87)

fn main() -> Result<()> {
    let representation = Representation::new("toy-encoder", "1", DIMENSION, true)?;
    let directory = std::env::temp_dir().join(format!("lateweave-example-{}", std::process::id()));
    let path = directory.join("vectors");

    // The indexing side: one token per word, the axis standing in for what
    // a real encoder produces.
    let mut writer =
        VectorStoreWriter::create(&path, Encoding::Float32, "fruit", representation.clone())?;
    let embeddings = [unit(0), unit(2), unit(1), unit(2), unit(0), unit(3)].concat();
    writer.append(["a", "b", "c"], &embeddings, DIMENSION, &[2, 2, 2], None)?;
    writer.commit()?;

    // The serving side.
    let gatherer = TermGatherer {
        requires: BTreeMap::new(),
        documents: vec![("a", "red apple"), ("b", "green apple"), ("c", "red car")],
        indexed_at: SystemTime::now(),
    };
    let store = Arc::new(VectorStore::open(&path)?) as Arc<dyn MultiVectorSource>;
    let reranker = MaxSimReranker::new([store], DEFAULT_FEATURE)?;
    let pipeline = SearchPipeline::new(Arc::new(gatherer), Some(Arc::new(reranker)));

    let query = Query::new("apple").with_feature(
        DEFAULT_FEATURE,
        Feature::lazy(representation, || {
            TokenMatrix::new(unit(1).to_vec(), DIMENSION)
        }),
    );
    let request = SearchRequest::new(10, 2).with_budget(ResourceBudget::new(4096, 64, Some(1))?);

    let print = |label: &str| -> Result<()> {
        let result = pipeline.search(&query, &request)?;
        println!("{label}:");
        for document in &result.documents {
            println!(
                "  #{} {:?} score {}",
                document.rank, document.key, document.score
            );
        }
        println!("  {:?}", result.diagnostics);
        Ok(())
    };
    print("before the delete")?;

    // The indexing side deletes "b" from the vectors; the gatherer still
    // finds it, and the pipeline drops it.
    writer.delete(["b"]);
    writer.commit()?;
    print("after the delete")?;

    std::fs::remove_dir_all(directory)?;
    Ok(())
}