hermes_core/
lib.rs

1//! Hermes - A minimal async search engine library
2//!
3//! Features:
4//! - Fully async IO with Directory abstraction for network/local/memory storage
5//! - SSTable-based term dictionary with hot cache and lazy loading
6//! - Bitpacked posting lists with block-level skip info
7//! - Document store with Zstd compression
8//! - Multiple segments with merge support
9//! - Text and numeric field support
10//! - Term, boolean, and boost queries
11//! - MaxScore / block-max pruning query optimizations
12
13pub mod compression;
14pub mod directories;
15pub mod dsl;
16pub mod error;
17pub mod index;
18pub mod merge;
19pub(crate) mod observe;
20pub mod query;
21pub mod segment;
22pub mod structures;
23pub mod tokenizer;
24
25// Re-exports from dsl
26pub use dsl::{
27    BinaryDenseVectorConfig, Document, Field, FieldDef, FieldEntry, FieldType, FieldValue,
28    IndexDef, QueryLanguageParser, Schema, SchemaBuilder, SdlParser, parse_sdl, parse_single_index,
29};
30
31// Re-exports from structures
32pub use structures::{
33    AsyncSSTableReader, BlockPostingList, HorizontalBP128Iterator, HorizontalBP128PostingList,
34    PostingList, PostingListIterator, SSTableValue, TERMINATED, TermInfo,
35};
36
37// Re-exports from directories
38#[cfg(feature = "native")]
39pub use directories::FsDirectory;
40#[cfg(feature = "http")]
41pub use directories::HttpDirectory;
42#[cfg(feature = "native")]
43pub use directories::MmapDirectory;
44pub use directories::{
45    CachingDirectory, Directory, DirectoryWriter, FileHandle, OwnedBytes, RamDirectory,
46    SliceCacheStats, SliceCachingDirectory,
47};
48
49/// Default directory type for native builds - uses memory-mapped files for efficient access
50#[cfg(feature = "native")]
51pub type DefaultDirectory = MmapDirectory;
52
53// Re-exports from segment
54pub use segment::{AsyncStoreReader, FieldStats, SegmentId, SegmentMeta, SegmentReader};
55#[cfg(any(feature = "native", feature = "wasm"))]
56pub use segment::{SegmentBuilder, SegmentBuilderConfig, SegmentBuilderStats};
57
58// Re-exports from query
59pub use query::{
60    BinaryDenseVectorQuery, Bm25Params, BooleanQuery, BoostQuery, MaxScoreExecutor, PrefixQuery,
61    Query, ScoredDoc, Scorer, SearchHit, SearchResponse, SearchResult, TermQuery, TopKCollector,
62};
63
64// Re-exports from tokenizer
65pub use tokenizer::{
66    BoxedTokenizer, Language, LanguageAwareTokenizer, MultiLanguageStemmer, RawCiTokenizer,
67    RawTokenizer, SimpleTokenizer, StemmerTokenizer, Token, Tokenizer, TokenizerRegistry,
68    parse_language,
69};
70
71// Re-exports from other modules
72pub use directories::SLICE_CACHE_EXTENSION;
73pub use error::{Error, Result};
74pub use index::Searcher;
75#[cfg(all(feature = "wasm", not(feature = "native")))]
76pub use index::WasmIndexWriter;
77#[cfg(feature = "native")]
78pub use index::{Index, IndexReader, IndexWriter};
79pub use index::{IndexConfig, IndexMetadata, SLICE_CACHE_FILENAME};
80#[cfg(feature = "native")]
81pub use index::{
82    IndexingStats, SchemaConfig, SchemaFieldConfig, create_index_at_path, create_index_from_sdl,
83    index_documents_from_reader, index_json_document, parse_schema,
84};
85
86// Re-exports from merge
87#[cfg(feature = "native")]
88pub use merge::SegmentManager;
89pub use merge::{MergeCandidate, MergePolicy, NoMergePolicy, SegmentInfo, TieredMergePolicy};
90
91pub type DocId = u32;
92pub type TermFreq = u32;
93pub type Score = f32;
94
95/// Default number of indexing threads (cpu / 4, minimum 1).
96/// Centralized so all configs share one definition.
97#[cfg(feature = "native")]
98pub fn default_indexing_threads() -> usize {
99    (num_cpus::get() / 4).max(1)
100}
101
102/// Default number of compression threads (cpu / 4, minimum 1).
103/// Centralized so all configs share one definition.
104#[cfg(feature = "native")]
105pub fn default_compression_threads() -> usize {
106    (num_cpus::get() / 4).max(1)
107}
hermes_core/lib.rs

hermes_core/
lib.rs
