sochdb_core/
lib.rs

1// SPDX-License-Identifier: AGPL-3.0-or-later
2// SochDB - LLM-Optimized Embedded Database
3// Copyright (C) 2026 Sushanth Reddy Vanagala (https://github.com/sushanthpy)
4//
5// This program is free software: you can redistribute it and/or modify
6// it under the terms of the GNU Affero General Public License as published by
7// the Free Software Foundation, either version 3 of the License, or
8// (at your option) any later version.
9//
10// This program is distributed in the hope that it will be useful,
11// but WITHOUT ANY WARRANTY; without even the implied warranty of
12// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13// GNU Affero General Public License for more details.
14//
15// You should have received a copy of the GNU Affero General Public License
16// along with this program. If not, see <https://www.gnu.org/licenses/>.
17
18//! SochDB Core
19//!
20//! TOON-native ACID database - fundamental types and data structures.
21//!
22//! SochDB is to TOON what MongoDB is to JSON - a database that natively
23//! understands and stores TOON (Tabular Object-Oriented Notation) documents.
24//!
25//! # Core Components
26//!
27//! - **TOON Format**: Native data format with schema support
28//! - **Transaction Manager**: ACID transaction support via WAL
29//! - **Virtual Filesystem**: POSIX-like VFS backed by WAL
30//! - **Schema Catalog**: Table and index metadata management
31//!
32//! # Memory Allocation
33//!
34//! **Recommended**: Enable jemalloc for production workloads:
35//!
36//! ```toml
37//! sochdb-core = { version = "...", features = ["jemalloc"] }
38//! ```
39//!
40//! jemalloc provides:
41//! - Thread-local caching (no lock contention)
42//! - Superior fragmentation handling
43//! - Automatic memory return to OS
44//! - Battle-tested in production (Firefox, Redis, RocksDB)
45//!
46//! **Note**: The `buddy_allocator` module is deprecated. See its documentation
47//! for migration guidance.
48//!
49//! # Features
50//!
51//! - `jemalloc` - Use jemalloc as the global allocator for better performance
52//!
53//! # Example
54//!
55//! ```rust,ignore
56//! use sochdb_core::soch::{SochSchema, SochType, SochTable, SochRow, SochValue};
57//!
58//! // Define a schema
59//! let schema = SochSchema::new("users")
60//!     .field("id", SochType::UInt)
61//!     .field("name", SochType::Text)
62//!     .field("email", SochType::Text)
63//!     .primary_key("id");
64//!
65//! // Create a table
66//! let mut table = SochTable::new(schema);
67//! table.push(SochRow::new(vec![
68//!     SochValue::UInt(1),
69//!     SochValue::Text("Alice".into()),
70//!     SochValue::Text("alice@example.com".into()),
71//! ]));
72//!
73//! // Format as TOON
74//! println!("{}", table.format());
75//! // Output: users[1]{id,name,email}:
76//! //         1,Alice,alice@example.com
77//! ```
78
79// Use jemalloc as global allocator when feature is enabled
80// This provides better performance for allocation-heavy workloads
81#[cfg(feature = "jemalloc")]
82#[global_allocator]
83static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
84
85pub mod block_storage;
86#[deprecated(
87    since = "0.2.0",
88    note = "Use jemalloc feature instead. See buddy_allocator module docs for migration."
89)]
90#[allow(deprecated)]
91pub mod buddy_allocator;
92pub mod catalog;
93pub mod columnar; // True Columnar Storage with Arrow-compatible layout (mm.md Task 1)
94pub mod concurrency; // Hierarchical Lock Architecture (mm.md Task 2)
95pub mod epoch_gc;
96pub mod error;
97pub mod format_migration;
98pub mod key;
99pub mod learned_index;
100pub mod lockfree_interner; // Lock-free string interner (mm.md Task 6)
101pub mod memory_schema; // Canonical Episode/Entity/Event schema
102pub mod path_trie;
103pub mod predefined_views;
104pub mod reclamation;
105pub mod schema_bridge;
106pub mod schema_evolution;
107pub mod sharded_block_store;
108pub mod string_interner; // String interning for path segments
109pub mod tbp; // TOON Binary Protocol for zero-copy wire format (mm.md Task 3.1)
110pub mod knowledge_object; // Knowledge Fabric: Object-centric data model (content-addressed, bitemporal, multi-space)
111pub mod soch;
112pub mod soch_codec;
113pub mod sochfs_metadata;
114pub mod transaction_typestate; // Type-State Transaction API (compile-time safe transaction lifecycle)
115pub mod txn;
116pub mod version_chain; // Unified MVCC version chain trait
117pub mod vfs;
118pub mod zero_copy; // Predefined SochQL views (Task 7)
119
120// Analytics - anonymous usage tracking (disabled with SOCHDB_DISABLE_ANALYTICS=true)
121#[cfg(feature = "analytics")]
122pub mod analytics;
123
124// Re-export core types
125pub use block_storage::{
126    BlockCompression, BlockRef, BlockStore, BlockStoreStats, FileBlockManager,
127};
128pub use catalog::{Catalog, CatalogEntry, CatalogEntryType, McpToolDescriptor, OperationImpl};
129pub use columnar::{
130    ColumnChunk, ColumnStats, ColumnType as ColumnarColumnType, ColumnValue as ColumnarColumnValue,
131    ColumnarStore, ColumnarTable, MemoryComparison, TypedColumn, ValidityBitmap,
132};
133pub use error::{Result, SochDBError};
134pub use key::{CausalKey, TemporalKey};
135pub use learned_index::LearnedSparseIndex;
136pub use lockfree_interner::{InternerStats, LockFreeInterner, Symbol};
137pub use memory_schema::{
138    Entity, EntityFacts, EntityKind, EntitySearchResult, Episode, EpisodeSearchResult, EpisodeType,
139    Event, EventMetrics, EventRole, MemoryStore, TableRole, TableSemanticMetadata,
140};
141pub use path_trie::{ColumnGroupAffinity, ColumnType as PathTrieColumnType, PathTrie, TrieNode};
142pub use predefined_views::{
143    ViewDefinition, build_view_map, get_predefined_views, get_view, naming,
144};
145pub use knowledge_object::{
146    BitemporalCoord, CompressionMode, Edge, EdgeKind, EmbeddingSpace, KnowledgeObject,
147    KnowledgeObjectBuilder, KnowledgeObjectError, ObjectId, ObjectIdError,
148    ObjectKind, Provenance,
149};
150pub use soch::{SochField, SochIndex, SochRow, SochSchema, SochTable, SochType, SochValue};
151pub use soch_codec::{
152    SochDbBinaryCodec, SochDocument, SochParseError, SochTextEncoder, SochTextParser,
153    SochTokenCounter,
154};
155pub use sochfs_metadata::{DirEntryRow, FsMetadataStore, FsWalOp, InodeRow, SochFS};
156pub use txn::{
157    AriesCheckpointData, AriesDirtyPageEntry, AriesTransactionEntry, IsolationLevel, Lsn, PageId,
158    Transaction, TransactionManager, TxnId, TxnState, TxnStats, TxnWalEntry, TxnWrite,
159    WalRecordType,
160};
161pub use transaction_typestate::{
162    Transaction as TypestateTransaction, Active, Committed, Aborted,
163    ReadOnly, ReadWrite, WriteOnly, TransactionStorage, TransactionMode,
164};
165pub use version_chain::{
166    BinarySearchChain, ChainEntry,
167    ConcurrencyPolicy, ExternalLock, InternalRwLock, LockFreeAtomic,
168    MvccGcStats, MvccStore, MvccStoreError,
169    MvccVersionChain, MvccVersionChainMut, VersionMeta, VisibilityContext, WriteConflictDetection,
170};
171pub use vfs::{
172    BlockId, DirEntry, Directory, FileStat, FileType, Inode, InodeId, Permissions, Superblock,
173    VfsOp,
174};
175
176/// Database version
177pub const SOCHDB_VERSION: &str = env!("CARGO_PKG_VERSION");
178
179/// Magic bytes for SochDB files
180pub const SOCHDB_MAGIC: [u8; 4] = *b"TOON";
181
182/// Current schema version
183pub const SCHEMA_VERSION: u32 = 1;
184
185#[cfg(test)]
186mod tests {
187    use super::*;
188
189    #[test]
190    fn test_soch_roundtrip() {
191        let schema = SochSchema::new("test")
192            .field("id", SochType::UInt)
193            .field("value", SochType::Text);
194
195        let mut table = SochTable::new(schema);
196        table.push(SochRow::new(vec![
197            SochValue::UInt(1),
198            SochValue::Text("hello".into()),
199        ]));
200
201        let formatted = table.format();
202        let parsed = SochTable::parse(&formatted).unwrap();
203
204        assert_eq!(parsed.schema.name, "test");
205        assert_eq!(parsed.rows.len(), 1);
206    }
207}
sochdb_core/lib.rs

sochdb_core/
lib.rs
