swiftide_indexing/lib.rs
1//! Streaming indexing pipelines for Swiftide.
2//!
3//! This crate provides the indexing [`Pipeline`], the [`loaders::FileLoader`], built-in
4//! [`transformers`] (chunking, embedding, LLM generated metadata) and the in-memory
5//! [`persist::MemoryStorage`]. The traits these build on live in `swiftide-core`; storage,
6//! models and code-aware transformers come from `swiftide-integrations`.
7//!
8//! The [`swiftide`](https://docs.rs/swiftide) facade re-exports this crate together with the core
9//! indexing types as `swiftide::indexing`. Prefer that unless you want a minimal dependency.
10//!
11//! Guide: <https://swiftide.rs/indexing/>
12//!
13//! ```no_run
14//! use swiftide_indexing::{
15//! Pipeline, loaders::FileLoader, persist::MemoryStorage, transformers::ChunkMarkdown,
16//! };
17//!
18//! # async fn run() -> anyhow::Result<()> {
19//! Pipeline::from_loader(FileLoader::new("README.md").with_extensions(&["md"]))
20//! .then_chunk(ChunkMarkdown::from_chunk_range(20..2048))
21//! .then_store_with(MemoryStorage::default())
22//! .run()
23//! .await?;
24//! # Ok(())
25//! # }
26//! ```
27// show feature flags in the generated documentation
28// https://doc.rust-lang.org/rustdoc/unstable-features.html#extensions-to-the-doc-attribute
29#![cfg_attr(docsrs, feature(doc_cfg))]
30#![cfg_attr(docsrs, doc(auto_cfg))]
31#![doc(html_logo_url = "https://github.com/bosun-ai/swiftide/raw/master/images/logo.png")]
32
33pub mod loaders;
34pub mod persist;
35pub mod transformers;
36
37mod pipeline;
38pub use pipeline::Pipeline;