summa_core/
lib.rs

1// clippy 1.98 added `chunks_exact_to_as_chunks`, which fires on ~40 call sites
2// here. Migrating them is a real (if mechanical) improvement — a const-generic
3// chunk width lets LLVM drop the per-chunk length check — but most of the sites
4// are inside vector / ScaNN / fast-field wire-format parsers, and rewriting those
5// belongs in its own reviewed change rather than riding along with a toolchain
6// bump. Tracked as follow-up; see docs/algebraic-float-reductions.md.
7#![allow(clippy::chunks_exact_to_as_chunks)]
8#![deny(
9    rustdoc::bare_urls,
10    rustdoc::broken_intra_doc_links,
11    rustdoc::invalid_html_tags,
12    rustdoc::private_intra_doc_links
13)]
14
15//! Summa - A minimal async search engine library
16//!
17//! Features:
18//! - Fully async IO with Directory abstraction for network/local/memory storage
19//! - SSTable-based term dictionary with hot cache and lazy loading
20//! - Bitpacked posting lists with block-level skip info
21//! - Document store with Zstd compression
22//! - Multiple segments with merge support
23//! - Text and numeric field support
24//! - Term, boolean, and boost queries
25//! - MaxScore / block-max pruning query optimizations
26
27pub mod compression;
28pub mod directories;
29pub mod dsl;
30pub mod error;
31pub mod index;
32pub mod merge;
33pub(crate) mod observe;
34pub mod query;
35#[cfg(feature = "query-diagnostics")]
36pub mod search_diagnostics;
37pub mod segment;
38pub mod structures;
39pub mod tokenizer;
40
41// Re-exports from dsl
42pub use dsl::{
43    BinaryDenseVectorConfig, Document, Field, FieldDef, FieldEntry, FieldType, FieldValue,
44    IndexDef, IvfRoutingMode, QueryLanguageParser, Schema, SchemaBuilder, SdlParser, parse_sdl,
45    parse_single_index,
46};
47
48// Re-exports from structures
49pub use structures::{
50    AsyncSSTableReader, BlockPostingList, HorizontalBP128Iterator, HorizontalBP128PostingList,
51    PostingList, PostingListIterator, SSTableValue, TERMINATED, TermInfo,
52};
53
54// Re-exports from directories
55#[cfg(feature = "native")]
56pub use directories::FsDirectory;
57#[cfg(feature = "http")]
58pub use directories::HttpDirectory;
59#[cfg(feature = "native")]
60pub use directories::MmapDirectory;
61pub use directories::{
62    CachingDirectory, Directory, DirectoryWriter, FileHandle, OwnedBytes, RamDirectory,
63    SliceCacheStats, SliceCachingDirectory,
64};
65
66/// Default directory type for native builds - uses memory-mapped files for efficient access
67#[cfg(feature = "native")]
68pub type DefaultDirectory = MmapDirectory;
69
70// Re-exports from segment
71pub use segment::{AsyncStoreReader, FieldStats, SegmentId, SegmentMeta, SegmentReader};
72#[cfg(any(feature = "native", feature = "wasm"))]
73pub use segment::{SegmentBuilder, SegmentBuilderConfig, SegmentBuilderStats};
74
75// Re-exports from query
76pub use query::{
77    BinaryDenseVectorQuery, Bm25Params, BooleanQuery, BoostQuery, MaxScoreExecutor, PhraseQuery,
78    PrefixQuery, Query, RegexQuery, ScoredDoc, Scorer, SearchHit, SearchResponse, SearchResult,
79    TermQuery, TopKCollector, WildcardQuery,
80};
81
82// Re-exports from tokenizer
83pub use tokenizer::{
84    BoxedTokenizer, Language, LanguageAwareTokenizer, LexOptions, LexTokenizer,
85    MultiLanguageStemmer, RawCiTokenizer, RawTokenizer, Script, SimpleTokenizer, StemmerTokenizer,
86    Token, Tokenizer, TokenizerRegistry, TokenizerSpec, language_code, parse_language,
87    parse_language_opt,
88};
89
90// Re-exports from other modules
91pub use directories::SLICE_CACHE_EXTENSION;
92pub use error::{Error, Result};
93pub use index::Searcher;
94#[cfg(all(feature = "wasm", not(feature = "native")))]
95pub use index::WasmIndexWriter;
96#[cfg(feature = "native")]
97pub use index::{Index, IndexReader, IndexWriter};
98pub use index::{IndexConfig, IndexMetadata, SLICE_CACHE_FILENAME};
99#[cfg(feature = "native")]
100pub use index::{
101    IndexingStats, SchemaConfig, SchemaFieldConfig, create_index_at_path, create_index_from_sdl,
102    index_documents_from_reader, index_json_document, parse_schema,
103};
104
105// Re-exports from merge
106#[cfg(feature = "native")]
107pub use merge::SegmentManager;
108pub use merge::{MergeCandidate, MergePolicy, NoMergePolicy, SegmentInfo, TieredMergePolicy};
109
110pub type DocId = u32;
111pub type TermFreq = u32;
112pub type Score = f32;
113
114/// Format a byte count with IEC binary units.
115///
116/// All Summa user-facing size logs use this formatter so a `MiB` always
117/// means 1,048,576 bytes and the precision is consistent across subsystems.
118pub fn format_bytes(bytes: u64) -> String {
119    const KIB: u64 = 1024;
120    const MIB: u64 = 1024 * KIB;
121    const GIB: u64 = 1024 * MIB;
122    const TIB: u64 = 1024 * GIB;
123
124    if bytes >= TIB {
125        format!("{:.2} TiB", bytes as f64 / TIB as f64)
126    } else if bytes >= GIB {
127        format!("{:.2} GiB", bytes as f64 / GIB as f64)
128    } else if bytes >= MIB {
129        format!("{:.2} MiB", bytes as f64 / MIB as f64)
130    } else if bytes >= KIB {
131        format!("{:.2} KiB", bytes as f64 / KIB as f64)
132    } else {
133        format!("{bytes} B")
134    }
135}
136
137/// Default number of indexing threads (cpu / 4, minimum 1).
138/// Centralized so all configs share one definition.
139#[cfg(feature = "native")]
140pub fn default_indexing_threads() -> usize {
141    default_quarter_cpu_threads()
142}
143
144/// Default width of the process-wide search CPU pool (cpu / 4, minimum 1).
145#[cfg(feature = "native")]
146pub fn default_search_threads() -> usize {
147    default_quarter_cpu_threads()
148}
149
150/// Default width of the process-wide document-store compression pool
151/// (cpu / 4, minimum 1).
152#[cfg(feature = "native")]
153pub fn default_compression_threads() -> usize {
154    default_quarter_cpu_threads()
155}
156
157#[cfg(feature = "native")]
158fn default_quarter_cpu_threads() -> usize {
159    (num_cpus::get() / 4).max(1)
160}
161
162#[cfg(test)]
163mod tests {
164    #[test]
165    fn format_bytes_uses_iec_units() {
166        assert_eq!(super::format_bytes(0), "0 B");
167        assert_eq!(super::format_bytes(1023), "1023 B");
168        assert_eq!(super::format_bytes(1024), "1.00 KiB");
169        assert_eq!(super::format_bytes(1024 * 1024), "1.00 MiB");
170        assert_eq!(super::format_bytes(3 * 1024 * 1024 * 1024), "3.00 GiB");
171        assert_eq!(
172            super::format_bytes(2 * 1024 * 1024 * 1024 * 1024),
173            "2.00 TiB"
174        );
175    }
176
177    #[cfg(feature = "native")]
178    #[test]
179    fn cpu_bound_defaults_share_one_policy() {
180        let expected = (num_cpus::get() / 4).max(1);
181
182        assert_eq!(super::default_indexing_threads(), expected);
183        assert_eq!(super::default_search_threads(), expected);
184        assert_eq!(super::default_compression_threads(), expected);
185    }
186}
summa_core/lib.rs

summa_core/
lib.rs
