haagenti_fragments/lib.rs
1//! Cross-Model Fragment Sharing Library
2//!
3//! This module implements a shared fragment library that enables:
4//! - Deduplication of similar tensor fragments across models
5//! - Single-load-multi-use for shared attention patterns
6//! - Significant storage reduction (30-50% for model families)
7//! - Faster model switching by reusing cached fragments
8//!
9//! # Key Concepts
10//!
11//! - **Fragment**: A compressed tensor chunk (typically 64KB-1MB)
12//! - **Signature**: Locality-sensitive hash for similarity detection
13//! - **Library**: Collection of unique fragments with reference tracking
14//! - **Manifest**: Per-model mapping from layer names to fragment refs
15//!
16//! # Architecture
17//!
18//! ```text
19//! ┌─────────────────────────────────────────────────────────────┐
20//! │ Fragment Library │
21//! ├─────────────────────────────────────────────────────────────┤
22//! │ FragmentId → Fragment (deduplicated storage) │
23//! │ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ │
24//! │ │ frag_0 │ │ frag_1 │ │ frag_2 │ │ frag_3 │ ... │
25//! │ └────────┘ └────────┘ └────────┘ └────────┘ │
26//! └─────────────────────────────────────────────────────────────┘
27//! ↑ ↑ ↑
28//! ┌──────┴───┐ ┌───┴───┐ ┌───┴───┐
29//! │ SDXL │ │ SD3.5 │ │ Flux │
30//! │ manifest │ │manifest│ │manifest│
31//! └──────────┘ └───────┘ └───────┘
32//! ```
33
34mod error;
35mod fragment;
36mod library;
37mod manifest;
38mod signature;
39mod similarity;
40
41pub use error::{FragmentError, Result};
42pub use fragment::{Fragment, FragmentId, FragmentMetadata, FragmentType};
43pub use library::{FragmentLibrary, LibraryConfig, LibraryStats};
44pub use manifest::{LayerMapping, ModelManifest, TensorRef};
45pub use signature::{FragmentSignature, SignatureConfig};
46pub use similarity::{SimilarityIndex, SimilarityMatch, SimilarityThreshold};
47
48/// Version of the fragment library format
49pub const LIBRARY_FORMAT_VERSION: u32 = 1;
50
51/// Default similarity threshold for fragment deduplication
52pub const DEFAULT_SIMILARITY_THRESHOLD: f32 = 0.995;
53
54/// Prelude for common imports
55pub mod prelude {
56 pub use super::{
57 Fragment, FragmentId, FragmentLibrary, FragmentSignature, LibraryConfig, ModelManifest,
58 Result, SimilarityIndex, TensorRef,
59 };
60}