hermes_core/
lib.rs

1#![deny(
2    rustdoc::bare_urls,
3    rustdoc::broken_intra_doc_links,
4    rustdoc::invalid_html_tags,
5    rustdoc::private_intra_doc_links
6)]
7
8//! Hermes - A minimal async search engine library
9//!
10//! Features:
11//! - Fully async IO with Directory abstraction for network/local/memory storage
12//! - SSTable-based term dictionary with hot cache and lazy loading
13//! - Bitpacked posting lists with block-level skip info
14//! - Document store with Zstd compression
15//! - Multiple segments with merge support
16//! - Text and numeric field support
17//! - Term, boolean, and boost queries
18//! - MaxScore / block-max pruning query optimizations
19
20pub mod compression;
21pub mod directories;
22pub mod dsl;
23pub mod error;
24pub mod index;
25pub mod merge;
26pub(crate) mod observe;
27pub mod query;
28pub mod segment;
29pub mod structures;
30pub mod tokenizer;
31
32// Re-exports from dsl
33pub use dsl::{
34    BinaryDenseVectorConfig, Document, Field, FieldDef, FieldEntry, FieldType, FieldValue,
35    IndexDef, IvfRoutingMode, QueryLanguageParser, Schema, SchemaBuilder, SdlParser, parse_sdl,
36    parse_single_index,
37};
38
39// Re-exports from structures
40pub use structures::{
41    AsyncSSTableReader, BlockPostingList, HorizontalBP128Iterator, HorizontalBP128PostingList,
42    PostingList, PostingListIterator, SSTableValue, TERMINATED, TermInfo,
43};
44
45// Re-exports from directories
46#[cfg(feature = "native")]
47pub use directories::FsDirectory;
48#[cfg(feature = "http")]
49pub use directories::HttpDirectory;
50#[cfg(feature = "native")]
51pub use directories::MmapDirectory;
52pub use directories::{
53    CachingDirectory, Directory, DirectoryWriter, FileHandle, OwnedBytes, RamDirectory,
54    SliceCacheStats, SliceCachingDirectory,
55};
56
57/// Default directory type for native builds - uses memory-mapped files for efficient access
58#[cfg(feature = "native")]
59pub type DefaultDirectory = MmapDirectory;
60
61// Re-exports from segment
62pub use segment::{AsyncStoreReader, FieldStats, SegmentId, SegmentMeta, SegmentReader};
63#[cfg(any(feature = "native", feature = "wasm"))]
64pub use segment::{SegmentBuilder, SegmentBuilderConfig, SegmentBuilderStats};
65
66// Re-exports from query
67pub use query::{
68    BinaryDenseVectorQuery, Bm25Params, BooleanQuery, BoostQuery, MaxScoreExecutor, PrefixQuery,
69    Query, ScoredDoc, Scorer, SearchHit, SearchResponse, SearchResult, TermQuery, TopKCollector,
70};
71
72// Re-exports from tokenizer
73pub use tokenizer::{
74    BoxedTokenizer, Language, LanguageAwareTokenizer, MultiLanguageStemmer, RawCiTokenizer,
75    RawTokenizer, SimpleTokenizer, StemmerTokenizer, Token, Tokenizer, TokenizerRegistry,
76    parse_language,
77};
78
79// Re-exports from other modules
80pub use directories::SLICE_CACHE_EXTENSION;
81pub use error::{Error, Result};
82pub use index::Searcher;
83#[cfg(all(feature = "wasm", not(feature = "native")))]
84pub use index::WasmIndexWriter;
85#[cfg(feature = "native")]
86pub use index::{Index, IndexReader, IndexWriter};
87pub use index::{IndexConfig, IndexMetadata, SLICE_CACHE_FILENAME};
88#[cfg(feature = "native")]
89pub use index::{
90    IndexingStats, SchemaConfig, SchemaFieldConfig, create_index_at_path, create_index_from_sdl,
91    index_documents_from_reader, index_json_document, parse_schema,
92};
93
94// Re-exports from merge
95#[cfg(feature = "native")]
96pub use merge::SegmentManager;
97pub use merge::{MergeCandidate, MergePolicy, NoMergePolicy, SegmentInfo, TieredMergePolicy};
98
99pub type DocId = u32;
100pub type TermFreq = u32;
101pub type Score = f32;
102
103/// Format a byte count with IEC binary units.
104///
105/// All Hermes user-facing size logs use this formatter so a `MiB` always
106/// means 1,048,576 bytes and the precision is consistent across subsystems.
107pub fn format_bytes(bytes: u64) -> String {
108    const KIB: u64 = 1024;
109    const MIB: u64 = 1024 * KIB;
110    const GIB: u64 = 1024 * MIB;
111    const TIB: u64 = 1024 * GIB;
112
113    if bytes >= TIB {
114        format!("{:.2} TiB", bytes as f64 / TIB as f64)
115    } else if bytes >= GIB {
116        format!("{:.2} GiB", bytes as f64 / GIB as f64)
117    } else if bytes >= MIB {
118        format!("{:.2} MiB", bytes as f64 / MIB as f64)
119    } else if bytes >= KIB {
120        format!("{:.2} KiB", bytes as f64 / KIB as f64)
121    } else {
122        format!("{bytes} B")
123    }
124}
125
126/// Default number of indexing threads (cpu / 4, minimum 1).
127/// Centralized so all configs share one definition.
128#[cfg(feature = "native")]
129pub fn default_indexing_threads() -> usize {
130    default_quarter_cpu_threads()
131}
132
133/// Default width of the process-wide search CPU pool (cpu / 4, minimum 1).
134#[cfg(feature = "native")]
135pub fn default_search_threads() -> usize {
136    default_quarter_cpu_threads()
137}
138
139/// Default width of the process-wide document-store compression pool
140/// (cpu / 4, minimum 1).
141#[cfg(feature = "native")]
142pub fn default_compression_threads() -> usize {
143    default_quarter_cpu_threads()
144}
145
146#[cfg(feature = "native")]
147fn default_quarter_cpu_threads() -> usize {
148    (num_cpus::get() / 4).max(1)
149}
150
151#[cfg(test)]
152mod tests {
153    #[test]
154    fn format_bytes_uses_iec_units() {
155        assert_eq!(super::format_bytes(0), "0 B");
156        assert_eq!(super::format_bytes(1023), "1023 B");
157        assert_eq!(super::format_bytes(1024), "1.00 KiB");
158        assert_eq!(super::format_bytes(1024 * 1024), "1.00 MiB");
159        assert_eq!(super::format_bytes(3 * 1024 * 1024 * 1024), "3.00 GiB");
160        assert_eq!(
161            super::format_bytes(2 * 1024 * 1024 * 1024 * 1024),
162            "2.00 TiB"
163        );
164    }
165
166    #[cfg(feature = "native")]
167    #[test]
168    fn cpu_bound_defaults_share_one_policy() {
169        let expected = (num_cpus::get() / 4).max(1);
170
171        assert_eq!(super::default_indexing_threads(), expected);
172        assert_eq!(super::default_search_threads(), expected);
173        assert_eq!(super::default_compression_threads(), expected);
174    }
175}
hermes_core/lib.rs

hermes_core/
lib.rs
