sochdb_core/
lib.rs

1// SPDX-License-Identifier: AGPL-3.0-or-later
2// SochDB - LLM-Optimized Embedded Database
3// Copyright (C) 2026 Sushanth Reddy Vanagala (https://github.com/sushanthpy)
4//
5// This program is free software: you can redistribute it and/or modify
6// it under the terms of the GNU Affero General Public License as published by
7// the Free Software Foundation, either version 3 of the License, or
8// (at your option) any later version.
9//
10// This program is distributed in the hope that it will be useful,
11// but WITHOUT ANY WARRANTY; without even the implied warranty of
12// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13// GNU Affero General Public License for more details.
14//
15// You should have received a copy of the GNU Affero General Public License
16// along with this program. If not, see <https://www.gnu.org/licenses/>.
17
18//! SochDB Core
19//!
20//! TOON-native ACID database - fundamental types and data structures.
21//!
22//! SochDB is to TOON what MongoDB is to JSON - a database that natively
23//! understands and stores TOON (Tabular Object-Oriented Notation) documents.
24//!
25//! # Core Components
26//!
27//! - **TOON Format**: Native data format with schema support
28//! - **Transaction Manager**: ACID transaction support via WAL
29//! - **Virtual Filesystem**: POSIX-like VFS backed by WAL
30//! - **Schema Catalog**: Table and index metadata management
31//!
32//! # Memory Allocation
33//!
34//! **Recommended**: Enable jemalloc for production workloads:
35//!
36//! ```toml
37//! sochdb-core = { version = "...", features = ["jemalloc"] }
38//! ```
39//!
40//! jemalloc provides:
41//! - Thread-local caching (no lock contention)
42//! - Superior fragmentation handling
43//! - Automatic memory return to OS
44//! - Battle-tested in production (Firefox, Redis, RocksDB)
45//!
46//! **Note**: The `buddy_allocator` module is deprecated. See its documentation
47//! for migration guidance.
48//!
49//! # Features
50//!
51//! - `jemalloc` - Use jemalloc as the global allocator for better performance
52//!
53//! # Example
54//!
55//! ```rust,ignore
56//! use sochdb_core::soch::{SochSchema, SochType, SochTable, SochRow, SochValue};
57//!
58//! // Define a schema
59//! let schema = SochSchema::new("users")
60//!     .field("id", SochType::UInt)
61//!     .field("name", SochType::Text)
62//!     .field("email", SochType::Text)
63//!     .primary_key("id");
64//!
65//! // Create a table
66//! let mut table = SochTable::new(schema);
67//! table.push(SochRow::new(vec![
68//!     SochValue::UInt(1),
69//!     SochValue::Text("Alice".into()),
70//!     SochValue::Text("alice@example.com".into()),
71//! ]));
72//!
73//! // Format as TOON
74//! println!("{}", table.format());
75//! // Output: users[1]{id,name,email}:
76//! //         1,Alice,alice@example.com
77//! ```
78
79// Use jemalloc as global allocator when feature is enabled
80// This provides better performance for allocation-heavy workloads
81#[cfg(feature = "jemalloc")]
82#[global_allocator]
83static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
84
85pub mod block_storage;
86#[deprecated(
87    since = "0.2.0",
88    note = "Use jemalloc feature instead. See buddy_allocator module docs for migration."
89)]
90#[allow(deprecated)]
91pub mod buddy_allocator;
92pub mod catalog;
93pub mod columnar; // True Columnar Storage with Arrow-compatible layout (mm.md Task 1)
94pub mod concurrency; // Hierarchical Lock Architecture (mm.md Task 2)
95pub mod epoch_gc;
96pub mod error;
97pub mod format_migration;
98pub mod edge_encoding; // Binary edge key encoding for graph overlay (Phase 1b)
99pub mod key;
100pub mod learned_index;
101pub mod record_id; // RecordId: universal table:id identifier with binary key encoding (Phase 1b)
102pub mod lockfree_interner; // Lock-free string interner (mm.md Task 6)
103pub mod memory_schema; // Canonical Episode/Entity/Event schema
104pub mod path_trie;
105pub mod predefined_views;
106pub mod reclamation;
107pub mod schema_bridge;
108pub mod schema_evolution;
109pub mod sharded_block_store;
110pub mod string_interner; // String interning for path segments
111pub mod tbp; // TOON Binary Protocol for zero-copy wire format (mm.md Task 3.1)
112pub mod knowledge_object; // Knowledge Fabric: Object-centric data model (content-addressed, bitemporal, multi-space)
113pub mod soch;
114pub mod soch_codec;
115pub mod sochfs_metadata;
116pub mod transaction_typestate; // Type-State Transaction API (compile-time safe transaction lifecycle)
117pub mod txn;
118pub mod version_chain; // Unified MVCC version chain trait
119pub mod vfs;
120pub mod zero_copy; // Predefined SochQL views (Task 7)
121
122// Analytics - anonymous usage tracking (disabled with SOCHDB_DISABLE_ANALYTICS=true)
123#[cfg(feature = "analytics")]
124pub mod analytics;
125
126// Re-export core types
127pub use block_storage::{
128    BlockCompression, BlockRef, BlockStore, BlockStoreStats, FileBlockManager,
129};
130pub use catalog::{Catalog, CatalogEntry, CatalogEntryType, McpToolDescriptor, OperationImpl};
131pub use columnar::{
132    ColumnChunk, ColumnStats, ColumnType as ColumnarColumnType, ColumnValue as ColumnarColumnValue,
133    ColumnarStore, ColumnarTable, MemoryComparison, TypedColumn, ValidityBitmap,
134};
135pub use error::{Result, SochDBError};
136pub use key::{CausalKey, TemporalKey};
137pub use learned_index::LearnedSparseIndex;
138pub use record_id::{IdValue, RecordId};
139pub use lockfree_interner::{InternerStats, LockFreeInterner, Symbol};
140pub use memory_schema::{
141    Entity, EntityFacts, EntityKind, EntitySearchResult, Episode, EpisodeSearchResult, EpisodeType,
142    Event, EventMetrics, EventRole, MemoryStore, TableRole, TableSemanticMetadata,
143};
144pub use path_trie::{ColumnGroupAffinity, ColumnType as PathTrieColumnType, PathTrie, TrieNode};
145pub use predefined_views::{
146    ViewDefinition, build_view_map, get_predefined_views, get_view, naming,
147};
148pub use knowledge_object::{
149    BitemporalCoord, CompressionMode, Edge, EdgeKind, EmbeddingSpace, KnowledgeObject,
150    KnowledgeObjectBuilder, KnowledgeObjectError, ObjectId, ObjectIdError,
151    ObjectKind, Provenance,
152};
153pub use soch::{SochField, SochIndex, SochRow, SochSchema, SochTable, SochType, SochValue};
154pub use soch_codec::{
155    SochDbBinaryCodec, SochDocument, SochParseError, SochTextEncoder, SochTextParser,
156    SochTokenCounter,
157};
158pub use sochfs_metadata::{DirEntryRow, FsMetadataStore, FsWalOp, InodeRow, SochFS};
159pub use txn::{
160    AriesCheckpointData, AriesDirtyPageEntry, AriesTransactionEntry, IsolationLevel, Lsn, PageId,
161    Transaction, TransactionManager, TxnId, TxnState, TxnStats, TxnWalEntry, TxnWrite,
162    WalRecordType,
163};
164pub use transaction_typestate::{
165    Transaction as TypestateTransaction, Active, Committed, Aborted,
166    ReadOnly, ReadWrite, WriteOnly, TransactionStorage, TransactionMode,
167};
168pub use version_chain::{
169    BinarySearchChain, ChainEntry,
170    ConcurrencyPolicy, ExternalLock, InternalRwLock, LockFreeAtomic,
171    MvccGcStats, MvccStore, MvccStoreError,
172    MvccVersionChain, MvccVersionChainMut, VersionMeta, VisibilityContext, WriteConflictDetection,
173};
174pub use vfs::{
175    BlockId, DirEntry, Directory, FileStat, FileType, Inode, InodeId, Permissions, Superblock,
176    VfsOp,
177};
178
179/// Database version
180pub const SOCHDB_VERSION: &str = env!("CARGO_PKG_VERSION");
181
182/// Magic bytes for SochDB files
183pub const SOCHDB_MAGIC: [u8; 4] = *b"TOON";
184
185/// Current schema version
186pub const SCHEMA_VERSION: u32 = 1;
187
188#[cfg(test)]
189mod tests {
190    use super::*;
191
192    #[test]
193    fn test_soch_roundtrip() {
194        let schema = SochSchema::new("test")
195            .field("id", SochType::UInt)
196            .field("value", SochType::Text);
197
198        let mut table = SochTable::new(schema);
199        table.push(SochRow::new(vec![
200            SochValue::UInt(1),
201            SochValue::Text("hello".into()),
202        ]));
203
204        let formatted = table.format();
205        let parsed = SochTable::parse(&formatted).unwrap();
206
207        assert_eq!(parsed.schema.name, "test");
208        assert_eq!(parsed.rows.len(), 1);
209    }
210}
sochdb_core/lib.rs

sochdb_core/
lib.rs
