hermes_core/
lib.rs

1//! Hermes - A minimal async search engine library
2//!
3//! Features:
4//! - Fully async IO with Directory abstraction for network/local/memory storage
5//! - SSTable-based term dictionary with hot cache and lazy loading
6//! - Bitpacked posting lists with block-level skip info
7//! - Document store with Zstd compression
8//! - Multiple segments with merge support
9//! - Text and numeric field support
10//! - Term, boolean, and boost queries
11//! - MaxScore / block-max pruning query optimizations
12
13pub mod compression;
14pub mod directories;
15pub mod dsl;
16pub mod error;
17pub mod index;
18pub mod merge;
19pub(crate) mod observe;
20pub mod query;
21pub mod segment;
22pub mod structures;
23pub mod tokenizer;
24
25// Re-exports from dsl
26pub use dsl::{
27    BinaryDenseVectorConfig, Document, Field, FieldDef, FieldEntry, FieldType, FieldValue,
28    IndexDef, IvfRoutingMode, QueryLanguageParser, Schema, SchemaBuilder, SdlParser, parse_sdl,
29    parse_single_index,
30};
31
32// Re-exports from structures
33pub use structures::{
34    AsyncSSTableReader, BlockPostingList, HorizontalBP128Iterator, HorizontalBP128PostingList,
35    PostingList, PostingListIterator, SSTableValue, TERMINATED, TermInfo,
36};
37
38// Re-exports from directories
39#[cfg(feature = "native")]
40pub use directories::FsDirectory;
41#[cfg(feature = "http")]
42pub use directories::HttpDirectory;
43#[cfg(feature = "native")]
44pub use directories::MmapDirectory;
45pub use directories::{
46    CachingDirectory, Directory, DirectoryWriter, FileHandle, OwnedBytes, RamDirectory,
47    SliceCacheStats, SliceCachingDirectory,
48};
49
50/// Default directory type for native builds - uses memory-mapped files for efficient access
51#[cfg(feature = "native")]
52pub type DefaultDirectory = MmapDirectory;
53
54// Re-exports from segment
55pub use segment::{AsyncStoreReader, FieldStats, SegmentId, SegmentMeta, SegmentReader};
56#[cfg(any(feature = "native", feature = "wasm"))]
57pub use segment::{SegmentBuilder, SegmentBuilderConfig, SegmentBuilderStats};
58
59// Re-exports from query
60pub use query::{
61    BinaryDenseVectorQuery, Bm25Params, BooleanQuery, BoostQuery, MaxScoreExecutor, PrefixQuery,
62    Query, ScoredDoc, Scorer, SearchHit, SearchResponse, SearchResult, TermQuery, TopKCollector,
63};
64
65// Re-exports from tokenizer
66pub use tokenizer::{
67    BoxedTokenizer, Language, LanguageAwareTokenizer, MultiLanguageStemmer, RawCiTokenizer,
68    RawTokenizer, SimpleTokenizer, StemmerTokenizer, Token, Tokenizer, TokenizerRegistry,
69    parse_language,
70};
71
72// Re-exports from other modules
73pub use directories::SLICE_CACHE_EXTENSION;
74pub use error::{Error, Result};
75pub use index::Searcher;
76#[cfg(all(feature = "wasm", not(feature = "native")))]
77pub use index::WasmIndexWriter;
78#[cfg(feature = "native")]
79pub use index::{Index, IndexReader, IndexWriter};
80pub use index::{IndexConfig, IndexMetadata, SLICE_CACHE_FILENAME};
81#[cfg(feature = "native")]
82pub use index::{
83    IndexingStats, SchemaConfig, SchemaFieldConfig, create_index_at_path, create_index_from_sdl,
84    index_documents_from_reader, index_json_document, parse_schema,
85};
86
87// Re-exports from merge
88#[cfg(feature = "native")]
89pub use merge::SegmentManager;
90pub use merge::{MergeCandidate, MergePolicy, NoMergePolicy, SegmentInfo, TieredMergePolicy};
91
92pub type DocId = u32;
93pub type TermFreq = u32;
94pub type Score = f32;
95
96/// Format a byte count with IEC binary units.
97///
98/// All Hermes user-facing size logs use this formatter so a `MiB` always
99/// means 1,048,576 bytes and the precision is consistent across subsystems.
100pub fn format_bytes(bytes: u64) -> String {
101    const KIB: u64 = 1024;
102    const MIB: u64 = 1024 * KIB;
103    const GIB: u64 = 1024 * MIB;
104    const TIB: u64 = 1024 * GIB;
105
106    if bytes >= TIB {
107        format!("{:.2} TiB", bytes as f64 / TIB as f64)
108    } else if bytes >= GIB {
109        format!("{:.2} GiB", bytes as f64 / GIB as f64)
110    } else if bytes >= MIB {
111        format!("{:.2} MiB", bytes as f64 / MIB as f64)
112    } else if bytes >= KIB {
113        format!("{:.2} KiB", bytes as f64 / KIB as f64)
114    } else {
115        format!("{bytes} B")
116    }
117}
118
119/// Default number of indexing threads (cpu / 4, minimum 1).
120/// Centralized so all configs share one definition.
121#[cfg(feature = "native")]
122pub fn default_indexing_threads() -> usize {
123    (num_cpus::get() / 4).max(1)
124}
125
126/// Default width of the process-wide search CPU pool (cpu / 4, minimum 1).
127#[cfg(feature = "native")]
128pub fn default_search_threads() -> usize {
129    (num_cpus::get() / 4).max(1)
130}
131
132/// Default number of compression threads (cpu / 4, minimum 1).
133/// Centralized so all configs share one definition.
134#[cfg(feature = "native")]
135pub fn default_compression_threads() -> usize {
136    (num_cpus::get() / 4).max(1)
137}
138
139#[cfg(test)]
140mod tests {
141    #[test]
142    fn format_bytes_uses_iec_units() {
143        assert_eq!(super::format_bytes(0), "0 B");
144        assert_eq!(super::format_bytes(1023), "1023 B");
145        assert_eq!(super::format_bytes(1024), "1.00 KiB");
146        assert_eq!(super::format_bytes(1024 * 1024), "1.00 MiB");
147        assert_eq!(super::format_bytes(3 * 1024 * 1024 * 1024), "3.00 GiB");
148        assert_eq!(
149            super::format_bytes(2 * 1024 * 1024 * 1024 * 1024),
150            "2.00 TiB"
151        );
152    }
153}
hermes_core/lib.rs

hermes_core/
lib.rs
