hermes_core/
lib.rs

1// clippy 1.98 added `chunks_exact_to_as_chunks`, which fires on ~40 call sites
2// here. Migrating them is a real (if mechanical) improvement — a const-generic
3// chunk width lets LLVM drop the per-chunk length check — but most of the sites
4// are inside BMP / ScaNN / fast-field wire-format parsers, and rewriting those
5// belongs in its own reviewed change rather than riding along with a toolchain
6// bump. Tracked as follow-up; see docs/algebraic-float-reductions.md.
7#![allow(clippy::chunks_exact_to_as_chunks)]
8#![deny(
9    rustdoc::bare_urls,
10    rustdoc::broken_intra_doc_links,
11    rustdoc::invalid_html_tags,
12    rustdoc::private_intra_doc_links
13)]
14
15//! Hermes - A minimal async search engine library
16//!
17//! Features:
18//! - Fully async IO with Directory abstraction for network/local/memory storage
19//! - SSTable-based term dictionary with hot cache and lazy loading
20//! - Bitpacked posting lists with block-level skip info
21//! - Document store with Zstd compression
22//! - Multiple segments with merge support
23//! - Text and numeric field support
24//! - Term, boolean, and boost queries
25//! - MaxScore / block-max pruning query optimizations
26
27pub mod compression;
28pub mod directories;
29pub mod dsl;
30pub mod error;
31pub mod index;
32pub mod merge;
33pub(crate) mod observe;
34pub mod query;
35pub mod segment;
36pub mod structures;
37pub mod tokenizer;
38
39// Re-exports from dsl
40pub use dsl::{
41    BinaryDenseVectorConfig, Document, Field, FieldDef, FieldEntry, FieldType, FieldValue,
42    IndexDef, IvfRoutingMode, QueryLanguageParser, Schema, SchemaBuilder, SdlParser, parse_sdl,
43    parse_single_index,
44};
45
46// Re-exports from structures
47pub use structures::{
48    AsyncSSTableReader, BlockPostingList, HorizontalBP128Iterator, HorizontalBP128PostingList,
49    PostingList, PostingListIterator, SSTableValue, TERMINATED, TermInfo,
50};
51
52// Re-exports from directories
53#[cfg(feature = "native")]
54pub use directories::FsDirectory;
55#[cfg(feature = "http")]
56pub use directories::HttpDirectory;
57#[cfg(feature = "native")]
58pub use directories::MmapDirectory;
59pub use directories::{
60    CachingDirectory, Directory, DirectoryWriter, FileHandle, OwnedBytes, RamDirectory,
61    SliceCacheStats, SliceCachingDirectory,
62};
63
64/// Default directory type for native builds - uses memory-mapped files for efficient access
65#[cfg(feature = "native")]
66pub type DefaultDirectory = MmapDirectory;
67
68// Re-exports from segment
69pub use segment::{AsyncStoreReader, FieldStats, SegmentId, SegmentMeta, SegmentReader};
70#[cfg(any(feature = "native", feature = "wasm"))]
71pub use segment::{SegmentBuilder, SegmentBuilderConfig, SegmentBuilderStats};
72
73// Re-exports from query
74pub use query::{
75    BinaryDenseVectorQuery, Bm25Params, BooleanQuery, BoostQuery, MaxScoreExecutor, PhraseQuery,
76    PrefixQuery, Query, ScoredDoc, Scorer, SearchHit, SearchResponse, SearchResult, TermQuery,
77    TopKCollector,
78};
79
80// Re-exports from tokenizer
81pub use tokenizer::{
82    BoxedTokenizer, Language, LanguageAwareTokenizer, LexOptions, LexTokenizer,
83    MultiLanguageStemmer, RawCiTokenizer, RawTokenizer, Script, SimpleTokenizer, StemmerTokenizer,
84    Token, Tokenizer, TokenizerRegistry, TokenizerSpec, language_code, parse_language,
85    parse_language_opt,
86};
87
88// Re-exports from other modules
89pub use directories::SLICE_CACHE_EXTENSION;
90pub use error::{Error, Result};
91pub use index::Searcher;
92#[cfg(all(feature = "wasm", not(feature = "native")))]
93pub use index::WasmIndexWriter;
94#[cfg(feature = "native")]
95pub use index::{Index, IndexReader, IndexWriter};
96pub use index::{IndexConfig, IndexMetadata, SLICE_CACHE_FILENAME};
97#[cfg(feature = "native")]
98pub use index::{
99    IndexingStats, SchemaConfig, SchemaFieldConfig, create_index_at_path, create_index_from_sdl,
100    index_documents_from_reader, index_json_document, parse_schema,
101};
102
103// Re-exports from merge
104#[cfg(feature = "native")]
105pub use merge::SegmentManager;
106pub use merge::{MergeCandidate, MergePolicy, NoMergePolicy, SegmentInfo, TieredMergePolicy};
107
108pub type DocId = u32;
109pub type TermFreq = u32;
110pub type Score = f32;
111
112/// Format a byte count with IEC binary units.
113///
114/// All Hermes user-facing size logs use this formatter so a `MiB` always
115/// means 1,048,576 bytes and the precision is consistent across subsystems.
116pub fn format_bytes(bytes: u64) -> String {
117    const KIB: u64 = 1024;
118    const MIB: u64 = 1024 * KIB;
119    const GIB: u64 = 1024 * MIB;
120    const TIB: u64 = 1024 * GIB;
121
122    if bytes >= TIB {
123        format!("{:.2} TiB", bytes as f64 / TIB as f64)
124    } else if bytes >= GIB {
125        format!("{:.2} GiB", bytes as f64 / GIB as f64)
126    } else if bytes >= MIB {
127        format!("{:.2} MiB", bytes as f64 / MIB as f64)
128    } else if bytes >= KIB {
129        format!("{:.2} KiB", bytes as f64 / KIB as f64)
130    } else {
131        format!("{bytes} B")
132    }
133}
134
135/// Default number of indexing threads (cpu / 4, minimum 1).
136/// Centralized so all configs share one definition.
137#[cfg(feature = "native")]
138pub fn default_indexing_threads() -> usize {
139    default_quarter_cpu_threads()
140}
141
142/// Default width of the process-wide search CPU pool (cpu / 4, minimum 1).
143#[cfg(feature = "native")]
144pub fn default_search_threads() -> usize {
145    default_quarter_cpu_threads()
146}
147
148/// Default width of the process-wide document-store compression pool
149/// (cpu / 4, minimum 1).
150#[cfg(feature = "native")]
151pub fn default_compression_threads() -> usize {
152    default_quarter_cpu_threads()
153}
154
155#[cfg(feature = "native")]
156fn default_quarter_cpu_threads() -> usize {
157    (num_cpus::get() / 4).max(1)
158}
159
160#[cfg(test)]
161mod tests {
162    #[test]
163    fn format_bytes_uses_iec_units() {
164        assert_eq!(super::format_bytes(0), "0 B");
165        assert_eq!(super::format_bytes(1023), "1023 B");
166        assert_eq!(super::format_bytes(1024), "1.00 KiB");
167        assert_eq!(super::format_bytes(1024 * 1024), "1.00 MiB");
168        assert_eq!(super::format_bytes(3 * 1024 * 1024 * 1024), "3.00 GiB");
169        assert_eq!(
170            super::format_bytes(2 * 1024 * 1024 * 1024 * 1024),
171            "2.00 TiB"
172        );
173    }
174
175    #[cfg(feature = "native")]
176    #[test]
177    fn cpu_bound_defaults_share_one_policy() {
178        let expected = (num_cpus::get() / 4).max(1);
179
180        assert_eq!(super::default_indexing_threads(), expected);
181        assert_eq!(super::default_search_threads(), expected);
182        assert_eq!(super::default_compression_threads(), expected);
183    }
184}
hermes_core/lib.rs

hermes_core/
lib.rs
