hermes_core/
lib.rs

1// clippy 1.98 added `chunks_exact_to_as_chunks`, which fires on ~40 call sites
2// here. Migrating them is a real (if mechanical) improvement — a const-generic
3// chunk width lets LLVM drop the per-chunk length check — but most of the sites
4// are inside BMP / ScaNN / fast-field wire-format parsers, and rewriting those
5// belongs in its own reviewed change rather than riding along with a toolchain
6// bump. Tracked as follow-up; see docs/algebraic-float-reductions.md.
7#![allow(clippy::chunks_exact_to_as_chunks)]
8#![deny(
9    rustdoc::bare_urls,
10    rustdoc::broken_intra_doc_links,
11    rustdoc::invalid_html_tags,
12    rustdoc::private_intra_doc_links
13)]
14
15//! Hermes - A minimal async search engine library
16//!
17//! Features:
18//! - Fully async IO with Directory abstraction for network/local/memory storage
19//! - SSTable-based term dictionary with hot cache and lazy loading
20//! - Bitpacked posting lists with block-level skip info
21//! - Document store with Zstd compression
22//! - Multiple segments with merge support
23//! - Text and numeric field support
24//! - Term, boolean, and boost queries
25//! - MaxScore / block-max pruning query optimizations
26
27pub mod compression;
28pub mod directories;
29pub mod dsl;
30pub mod error;
31pub mod index;
32pub mod merge;
33pub(crate) mod observe;
34pub mod query;
35pub mod segment;
36pub mod structures;
37pub mod tokenizer;
38
39// Re-exports from dsl
40pub use dsl::{
41    BinaryDenseVectorConfig, Document, Field, FieldDef, FieldEntry, FieldType, FieldValue,
42    IndexDef, IvfRoutingMode, QueryLanguageParser, Schema, SchemaBuilder, SdlParser, parse_sdl,
43    parse_single_index,
44};
45
46// Re-exports from structures
47pub use structures::{
48    AsyncSSTableReader, BlockPostingList, HorizontalBP128Iterator, HorizontalBP128PostingList,
49    PostingList, PostingListIterator, SSTableValue, TERMINATED, TermInfo,
50};
51
52// Re-exports from directories
53#[cfg(feature = "native")]
54pub use directories::FsDirectory;
55#[cfg(feature = "http")]
56pub use directories::HttpDirectory;
57#[cfg(feature = "native")]
58pub use directories::MmapDirectory;
59pub use directories::{
60    CachingDirectory, Directory, DirectoryWriter, FileHandle, OwnedBytes, RamDirectory,
61    SliceCacheStats, SliceCachingDirectory,
62};
63
64/// Default directory type for native builds - uses memory-mapped files for efficient access
65#[cfg(feature = "native")]
66pub type DefaultDirectory = MmapDirectory;
67
68// Re-exports from segment
69pub use segment::{AsyncStoreReader, FieldStats, SegmentId, SegmentMeta, SegmentReader};
70#[cfg(any(feature = "native", feature = "wasm"))]
71pub use segment::{SegmentBuilder, SegmentBuilderConfig, SegmentBuilderStats};
72
73// Re-exports from query
74pub use query::{
75    BinaryDenseVectorQuery, Bm25Params, BooleanQuery, BoostQuery, MaxScoreExecutor, PhraseQuery,
76    PrefixQuery, Query, ScoredDoc, Scorer, SearchHit, SearchResponse, SearchResult, TermQuery,
77    TopKCollector,
78};
79
80// Re-exports from tokenizer
81pub use tokenizer::{
82    BoxedTokenizer, DynamicStemmer, Language, LanguageAwareTokenizer, MultiLanguageStemmer,
83    RawCiTokenizer, RawTokenizer, Script, SimpleTokenizer, StemmerTokenizer, Token, Tokenizer,
84    TokenizerRegistry, TokenizerSpec, language_code, parse_language, parse_language_opt,
85};
86
87// Re-exports from other modules
88pub use directories::SLICE_CACHE_EXTENSION;
89pub use error::{Error, Result};
90pub use index::Searcher;
91#[cfg(all(feature = "wasm", not(feature = "native")))]
92pub use index::WasmIndexWriter;
93#[cfg(feature = "native")]
94pub use index::{Index, IndexReader, IndexWriter};
95pub use index::{IndexConfig, IndexMetadata, SLICE_CACHE_FILENAME};
96#[cfg(feature = "native")]
97pub use index::{
98    IndexingStats, SchemaConfig, SchemaFieldConfig, create_index_at_path, create_index_from_sdl,
99    index_documents_from_reader, index_json_document, parse_schema,
100};
101
102// Re-exports from merge
103#[cfg(feature = "native")]
104pub use merge::SegmentManager;
105pub use merge::{MergeCandidate, MergePolicy, NoMergePolicy, SegmentInfo, TieredMergePolicy};
106
107pub type DocId = u32;
108pub type TermFreq = u32;
109pub type Score = f32;
110
111/// Format a byte count with IEC binary units.
112///
113/// All Hermes user-facing size logs use this formatter so a `MiB` always
114/// means 1,048,576 bytes and the precision is consistent across subsystems.
115pub fn format_bytes(bytes: u64) -> String {
116    const KIB: u64 = 1024;
117    const MIB: u64 = 1024 * KIB;
118    const GIB: u64 = 1024 * MIB;
119    const TIB: u64 = 1024 * GIB;
120
121    if bytes >= TIB {
122        format!("{:.2} TiB", bytes as f64 / TIB as f64)
123    } else if bytes >= GIB {
124        format!("{:.2} GiB", bytes as f64 / GIB as f64)
125    } else if bytes >= MIB {
126        format!("{:.2} MiB", bytes as f64 / MIB as f64)
127    } else if bytes >= KIB {
128        format!("{:.2} KiB", bytes as f64 / KIB as f64)
129    } else {
130        format!("{bytes} B")
131    }
132}
133
134/// Default number of indexing threads (cpu / 4, minimum 1).
135/// Centralized so all configs share one definition.
136#[cfg(feature = "native")]
137pub fn default_indexing_threads() -> usize {
138    default_quarter_cpu_threads()
139}
140
141/// Default width of the process-wide search CPU pool (cpu / 4, minimum 1).
142#[cfg(feature = "native")]
143pub fn default_search_threads() -> usize {
144    default_quarter_cpu_threads()
145}
146
147/// Default width of the process-wide document-store compression pool
148/// (cpu / 4, minimum 1).
149#[cfg(feature = "native")]
150pub fn default_compression_threads() -> usize {
151    default_quarter_cpu_threads()
152}
153
154#[cfg(feature = "native")]
155fn default_quarter_cpu_threads() -> usize {
156    (num_cpus::get() / 4).max(1)
157}
158
159#[cfg(test)]
160mod tests {
161    #[test]
162    fn format_bytes_uses_iec_units() {
163        assert_eq!(super::format_bytes(0), "0 B");
164        assert_eq!(super::format_bytes(1023), "1023 B");
165        assert_eq!(super::format_bytes(1024), "1.00 KiB");
166        assert_eq!(super::format_bytes(1024 * 1024), "1.00 MiB");
167        assert_eq!(super::format_bytes(3 * 1024 * 1024 * 1024), "3.00 GiB");
168        assert_eq!(
169            super::format_bytes(2 * 1024 * 1024 * 1024 * 1024),
170            "2.00 TiB"
171        );
172    }
173
174    #[cfg(feature = "native")]
175    #[test]
176    fn cpu_bound_defaults_share_one_policy() {
177        let expected = (num_cpus::get() / 4).max(1);
178
179        assert_eq!(super::default_indexing_threads(), expected);
180        assert_eq!(super::default_search_threads(), expected);
181        assert_eq!(super::default_compression_threads(), expected);
182    }
183}
hermes_core/lib.rs

hermes_core/
lib.rs
