swiftide/lib.rs
1// show feature flags in the generated documentation
2// https://doc.rust-lang.org/rustdoc/unstable-features.html#extensions-to-the-doc-attribute
3#![cfg_attr(docsrs, feature(doc_cfg))]
4#![cfg_attr(docsrs, doc(auto_cfg))]
5#![doc(html_logo_url = "https://github.com/bosun-ai/swiftide/raw/master/images/logo.png")]
6#![allow(unused_imports, reason = "that is what we do here")]
7#![allow(clippy::doc_markdown, reason = "the readme is invalid and that is ok")]
8#![doc = include_str!(env!("DOC_README"))]
9#![doc = document_features::document_features!()]
10
11#[doc(inline)]
12pub use swiftide_core::prompt;
13#[doc(inline)]
14pub use swiftide_core::type_aliases::*;
15
16#[cfg(feature = "swiftide-agents")]
17#[doc(inline)]
18pub use swiftide_agents as agents;
19
20/// Common traits for common behaviour, re-exported from indexing and query
21pub mod traits {
22 #[doc(inline)]
23 pub use swiftide_core::agent_traits::*;
24 #[doc(inline)]
25 pub use swiftide_core::chat_completion::traits::*;
26 #[doc(inline)]
27 pub use swiftide_core::indexing_traits::*;
28 #[doc(inline)]
29 pub use swiftide_core::query_traits::*;
30 #[doc(inline)]
31 pub use swiftide_core::tokenizer::*;
32}
33
34/// Abstractions for chat completions and LLM interactions.
35#[doc(inline)]
36pub use swiftide_core::chat_completion;
37
38/// Integrations with various platforms and external services.
39pub mod integrations {
40 #[doc(inline)]
41 pub use swiftide_integrations::*;
42}
43
44/// This module serves as the main entry point for indexing in Swiftide.
45///
46/// The indexing system in Swiftide is designed to handle the asynchronous processing of large
47/// volumes of data, including loading, transforming, and storing data chunks.
48pub mod indexing {
49 #[doc(inline)]
50 pub use swiftide_core::indexing::*;
51 #[doc(inline)]
52 pub use swiftide_indexing::*;
53
54 pub mod transformers {
55 #[cfg(feature = "tree-sitter")]
56 #[doc(inline)]
57 pub use swiftide_integrations::treesitter::transformers::*;
58
59 pub use swiftide_indexing::transformers::*;
60 }
61}
62
63#[cfg(feature = "macros")]
64#[doc(inline)]
65pub use swiftide_macros::*;
66/// # Querying pipelines
67///
68/// Swiftide allows you to define sophisticated query pipelines.
69///
70/// Consider the following code that uses Swiftide to load some markdown text, chunk it, embed it,
71/// and store it in a Qdrant index:
72///
73/// ```no_run
74/// use swiftide::{
75/// indexing::{
76/// self,
77/// loaders::FileLoader,
78/// transformers::{ChunkMarkdown, Embed, MetadataQAText},
79/// },
80/// integrations::{self, qdrant::Qdrant},
81/// integrations::openai::OpenAI,
82/// query::{self, answers, query_transformers, response_transformers},
83/// };
84///
85/// async fn index() -> Result<(), Box<dyn std::error::Error>> {
86/// let openai_client = OpenAI::builder()
87/// .default_embed_model("text-embedding-3-large")
88/// .default_prompt_model("gpt-4o")
89/// .build()?;
90///
91/// let qdrant = Qdrant::builder()
92/// .batch_size(50)
93/// .vector_size(3072)
94/// .collection_name("swiftide-examples")
95/// .build()?;
96///
97/// indexing::Pipeline::from_loader(FileLoader::new("README.md"))
98/// .then_chunk(ChunkMarkdown::from_chunk_range(10..2048))
99/// .then(MetadataQAText::new(openai_client.clone()))
100/// .then_in_batch(Embed::new(openai_client.clone()).with_batch_size(10))
101/// .then_store_with(qdrant.clone())
102/// .run()
103/// .await?;
104///
105/// Ok(())
106/// }
107/// ```
108///
109/// We could then define a query pipeline that uses the Qdrant index to answer questions:
110///
111/// ```no_run
112/// # use swiftide::{
113/// # indexing::{
114/// # self,
115/// # loaders::FileLoader,
116/// # transformers::{ChunkMarkdown, Embed, MetadataQAText},
117/// # },
118/// # integrations::{self, qdrant::Qdrant},
119/// # query::{self, answers, query_transformers, response_transformers},
120/// # integrations::openai::OpenAI,
121/// # };
122/// # async fn query() -> Result<(), Box<dyn std::error::Error>> {
123/// # let openai_client = OpenAI::builder()
124/// # .default_embed_model("text-embedding-3-large")
125/// # .default_prompt_model("gpt-4o")
126/// # .build()?;
127/// # let qdrant = Qdrant::builder()
128/// # .batch_size(50)
129/// # .vector_size(3072)
130/// # .collection_name("swiftide-examples")
131/// # .build()?;
132/// // By default the search strategy is SimilaritySingleEmbedding
133/// // which takes the latest query, embeds it, and does a similarity search
134/// let pipeline = query::Pipeline::default()
135/// .then_transform_query(query_transformers::GenerateSubquestions::from_client(
136/// openai_client.clone(),
137/// ))
138/// .then_transform_query(query_transformers::Embed::from_client(
139/// openai_client.clone(),
140/// ))
141/// .then_retrieve(qdrant.clone())
142/// .then_transform_response(response_transformers::Summary::from_client(
143/// openai_client.clone(),
144/// ))
145/// .then_answer(answers::Simple::from_client(openai_client.clone()));
146///
147/// let result = pipeline
148/// .query("What is swiftide? Please provide an elaborate explanation")
149/// .await?;
150///
151/// println!("{:?}", result.answer());
152/// # Ok(())
153/// # }
154/// ```
155///
156/// By using a query pipeline to transform queries, we can improve the quality of the answers we get
157/// from our index. In this example, we used an LLM to generate subquestions, embedding those and
158/// then using them to search the index. Finally, we summarize the results and combine them together
159/// into a single answer.
160pub mod query {
161 #[doc(inline)]
162 pub use swiftide_core::querying::*;
163 #[doc(inline)]
164 pub use swiftide_query::*;
165}
166
167#[cfg(feature = "langfuse")]
168#[doc(inline)]
169pub use swiftide_langfuse as langfuse;
170
171/// Re-exports for macros
172#[doc(hidden)]
173pub mod reexports {
174 pub use ::anyhow;
175 pub use ::async_trait;
176 pub use ::schemars;
177 pub use ::serde;
178 pub use ::serde_json;
179}