apr_format/v2/
mod.rs

1//! APR Format Module (v2 `APR\0`) — sovereign leaf (issue #2231)
2//!
3//! Implements the APR v2 container format with:
4//! - 64-byte tensor alignment for zero-copy mmap
5//! - LZ4 block compression (64KB blocks)
6//! - JSON metadata section
7//! - Multi-file sharding for 10B+ parameter models
8//! - Single unified format (no versioning complexity)
9//!
10//! # Format Structure (APR)
11//!
12//! ```text
13//! ┌─────────────────────────────────────────────────────────────┐
14//! │ Header (64 bytes, 64-byte aligned)                          │
15//! │   - Magic: "APR\0" (4 bytes) - ONE format, no versioning    │
16//! │   - Version: major.minor (2 bytes)                          │
17//! │   - Flags (2 bytes)                                         │
18//! │   - Tensor count (4 bytes)                                  │
19//! │   - Metadata offset (8 bytes)                               │
20//! │   - Metadata size (4 bytes)                                 │
21//! │   - Tensor index offset (8 bytes)                           │
22//! │   - Data offset (8 bytes)                                   │
23//! │   - Checksum (4 bytes)                                      │
24//! │   - Reserved (20 bytes, zero-padded)                        │
25//! ├─────────────────────────────────────────────────────────────┤
26//! │ JSON Metadata (variable, padded to 64-byte boundary)        │
27//! ├─────────────────────────────────────────────────────────────┤
28//! │ Tensor Index (sorted by name, 64-byte aligned entries)      │
29//! ├─────────────────────────────────────────────────────────────┤
30//! │ Tensor Data (each tensor 64-byte aligned)                   │
31//! ├─────────────────────────────────────────────────────────────┤
32//! │ Footer Checksum (4 bytes)                                   │
33//! └─────────────────────────────────────────────────────────────┘
34//! ```
35//!
36//! # Example
37//!
38//! ```rust
39//! use apr_format::v2::{AprV2Header, AprV2Flags, MAGIC_V2, ALIGNMENT};
40//!
41//! let header = AprV2Header::new();
42//! assert_eq!(header.magic, MAGIC_V2);
43//! assert!(header.is_valid());
44//! ```
45//!
46//! # Sovereignty (issue #2231)
47//!
48//! This module contains ONLY the container I/O — pure bytes, shapes, and
49//! dtypes. It carries **no** ML/GPU/tokenizer dependency:
50//!   - CRC32 routes through the single [`crate::crc32::crc32`].
51//!   - f16 conversion routes through [`crate::f16`] (the IEEE-correct `half`
52//!     crate), NOT `trueno::f32_to_f16`. See the f16 note in `crate::f16`.
53//!   - The dequantizing `get_tensor_as_f32` accessor (which needs the GGUF
54//!     Q4_K/Q6_K dequant + f32 physics) is **severed** from the leaf reader and
55//!     re-attached in `aprender-core` as an extension trait (`AprV2DequantExt`).
56//!     The leaf exposes the raw bytes via [`AprV2Reader::get_tensor_data`] and
57//!     the typed-but-trivial [`AprV2Reader::get_f32_tensor`] (F32 dtype only).
58
59// ============================================================================
60// Constants
61// ============================================================================
62
63/// APR magic number: "APR\0" in ASCII (0x41505200)
64/// ONE format. No versioning. Period.
65pub const MAGIC_V2: [u8; 4] = [0x41, 0x50, 0x52, 0x00];
66
67/// Format version 2.0
68pub const VERSION_V2: (u8, u8) = (2, 0);
69
70/// Header size in bytes (64-byte aligned)
71pub const HEADER_SIZE_V2: usize = 64;
72
73/// Tensor alignment in bytes (for zero-copy mmap)
74pub const ALIGNMENT: usize = 64;
75
76/// LZ4 block size in bytes
77pub const LZ4_BLOCK_SIZE: usize = 64 * 1024; // 64KB
78
79/// Maximum metadata size (16MB)
80pub const MAX_METADATA_SIZE: usize = 16 * 1024 * 1024;
81
82/// Maximum tensor name length
83pub const MAX_TENSOR_NAME_LEN: usize = 256;
84
85// ============================================================================
86// Flags
87// ============================================================================
88
89/// APR v2 feature flags (16-bit for expanded feature set)
90#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
91pub struct AprV2Flags(u16);
92
93impl AprV2Flags {
94    /// Payload is compressed with LZ4
95    pub const LZ4_COMPRESSED: u16 = 0b0000_0000_0000_0001;
96    /// Payload is compressed with Zstd
97    pub const ZSTD_COMPRESSED: u16 = 0b0000_0000_0000_0010;
98    /// Payload is encrypted (AES-256-GCM)
99    pub const ENCRYPTED: u16 = 0b0000_0000_0000_0100;
100    /// Has digital signature (Ed25519)
101    pub const SIGNED: u16 = 0b0000_0000_0000_1000;
102    /// Model is sharded across multiple files
103    pub const SHARDED: u16 = 0b0000_0000_0001_0000;
104    /// Tensors are quantized
105    pub const QUANTIZED: u16 = 0b0000_0000_0010_0000;
106    /// Has embedded filterbank (for Whisper models)
107    pub const HAS_FILTERBANK: u16 = 0b0000_0000_0100_0000;
108    /// Has model card metadata
109    pub const HAS_MODEL_CARD: u16 = 0b0000_0000_1000_0000;
110    /// Supports streaming/chunked loading
111    pub const STREAMING: u16 = 0b0000_0001_0000_0000;
112    /// Contains vocabulary/tokenizer
113    pub const HAS_VOCAB: u16 = 0b0000_0010_0000_0000;
114
115    /// LAYOUT-002: Tensor layout is row-major (REQUIRED for valid APR files)
116    /// All APR files created after LAYOUT-002 must have this flag set.
117    /// Pre-LAYOUT-002 files without this flag are assumed row-major.
118    pub const LAYOUT_ROW_MAJOR: u16 = 0b0000_0100_0000_0000;
119
120    /// LAYOUT-002: Tensor layout is column-major (FORBIDDEN - Jidoka guard)
121    /// If this flag is set, the APR file is "dirty" and must be rejected.
122    /// This flag exists to catch improperly converted GGUF files.
123    pub const LAYOUT_COLUMN_MAJOR: u16 = 0b0000_1000_0000_0000;
124
125    /// Create new empty flags
126    #[must_use]
127    pub const fn new() -> Self {
128        Self(0)
129    }
130
131    /// Create from raw u16 value
132    #[must_use]
133    pub const fn from_bits(bits: u16) -> Self {
134        Self(bits)
135    }
136
137    /// Get raw bits
138    #[must_use]
139    pub const fn bits(self) -> u16 {
140        self.0
141    }
142
143    /// Check if flag is set
144    #[must_use]
145    pub const fn contains(self, flag: u16) -> bool {
146        (self.0 & flag) == flag
147    }
148
149    /// Set a flag
150    #[must_use]
151    pub const fn with(self, flag: u16) -> Self {
152        Self(self.0 | flag)
153    }
154
155    /// Clear a flag
156    #[must_use]
157    pub const fn without(self, flag: u16) -> Self {
158        Self(self.0 & !flag)
159    }
160
161    /// Check if LZ4 compressed
162    #[must_use]
163    pub const fn is_lz4_compressed(self) -> bool {
164        self.contains(Self::LZ4_COMPRESSED)
165    }
166
167    /// Check if Zstd compressed
168    #[must_use]
169    pub const fn is_zstd_compressed(self) -> bool {
170        self.contains(Self::ZSTD_COMPRESSED)
171    }
172
173    /// Check if encrypted
174    #[must_use]
175    pub const fn is_encrypted(self) -> bool {
176        self.contains(Self::ENCRYPTED)
177    }
178
179    /// Check if sharded
180    #[must_use]
181    pub const fn is_sharded(self) -> bool {
182        self.contains(Self::SHARDED)
183    }
184
185    /// Check if quantized
186    #[must_use]
187    pub const fn is_quantized(self) -> bool {
188        self.contains(Self::QUANTIZED)
189    }
190
191    /// LAYOUT-002: Check if row-major layout flag is set
192    #[must_use]
193    pub const fn is_row_major(self) -> bool {
194        self.contains(Self::LAYOUT_ROW_MAJOR)
195    }
196
197    /// LAYOUT-002: Check if column-major layout flag is set (should be rejected)
198    #[must_use]
199    pub const fn is_column_major(self) -> bool {
200        self.contains(Self::LAYOUT_COLUMN_MAJOR)
201    }
202
203    /// LAYOUT-002: Validate layout is safe for inference
204    /// Returns true if the file is row-major or pre-LAYOUT-002 (assumed row-major)
205    /// Returns false if explicitly marked as column-major (dirty APR file)
206    #[must_use]
207    pub const fn is_layout_valid(self) -> bool {
208        // Reject if explicitly marked as column-major
209        !self.is_column_major()
210    }
211}
212
213// ============================================================================
214// Header
215// ============================================================================
216
217/// APR file header (64 bytes)
218#[derive(Debug, Clone, Copy)]
219#[repr(C)]
220pub struct AprV2Header {
221    /// Magic number ("APR\0") - ONE format, no versioning
222    pub magic: [u8; 4],
223    /// Format version (major, minor)
224    pub version: (u8, u8),
225    /// Feature flags
226    pub flags: AprV2Flags,
227    /// Number of tensors
228    pub tensor_count: u32,
229    /// Offset to JSON metadata section
230    pub metadata_offset: u64,
231    /// Size of metadata in bytes
232    pub metadata_size: u32,
233    /// Offset to tensor index
234    pub tensor_index_offset: u64,
235    /// Offset to tensor data
236    pub data_offset: u64,
237    /// Header checksum (CRC32)
238    pub checksum: u32,
239    /// Reserved for future use (zero-padded)
240    pub reserved: [u8; 20],
241}
242
243impl Default for AprV2Header {
244    fn default() -> Self {
245        Self::new()
246    }
247}
248
249// --- Split modules (was include!(); now real `mod`s, issue #2231 Stage 2) ----
250// Each formerly-`include!`d file is a real module that re-derives its own `use`
251// block and reaches the parent-scope structs/consts via `super::`. The public
252// items are re-exported into the `v2` namespace below so the historical flat
253// paths (`aprender::format::v2::AprV2Reader`, …) keep resolving unchanged.
254mod header_impl;
255mod reader_impl;
256mod streaming_writer;
257mod tensor_index_impl;
258mod v2format_error;
259mod writer;
260
261pub use header_impl::{
262    AprV2Metadata, ChatSpecialTokens, QuantizationMetadata, ShardingMetadata, TensorIndexEntry,
263};
264pub use reader_impl::{AprV2Reader, AprV2ReaderRef, ShardInfo, ShardManifest};
265pub use streaming_writer::AprV2StreamingWriter;
266pub use tensor_index_impl::{align_64, align_up, is_aligned_64, padding_to_align, TensorDType};
267pub use v2format_error::V2FormatError;
268pub use writer::AprV2Writer;
269
270// Provenance stamping — SHIP-009 full-discharge enabler (task #141).
271// Lives as a real submodule (not `include!`) so its inline tests nest
272// cleanly under `v2::stamp::tests`.
273pub mod stamp;
274pub use stamp::{stamp_provenance_bytes, ProvenancePatch};
275
276#[cfg(test)]
277mod tests;
apr_format/v2/mod.rs

apr_format/v2/
mod.rs
