use std::{fs, path::Path};
use serde::{Deserialize, Serialize};
use crate::error::{RealizarError, Result};
pub const MAGIC: [u8; 4] = [0x41, 0x50, 0x52, 0x4E];
pub const FORMAT_VERSION: (u8, u8) = (1, 0);
pub const HEADER_SIZE: usize = 32;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[repr(u16)]
pub enum AprModelType {
LinearRegression = 0x0001,
LogisticRegression = 0x0002,
DecisionTree = 0x0003,
RandomForest = 0x0004,
GradientBoosting = 0x0005,
KMeans = 0x0006,
Pca = 0x0007,
NaiveBayes = 0x0008,
Knn = 0x0009,
Svm = 0x000A,
NgramLm = 0x0010,
Tfidf = 0x0011,
CountVectorizer = 0x0012,
NeuralSequential = 0x0020,
NeuralCustom = 0x0021,
ContentRecommender = 0x0030,
MixtureOfExperts = 0x0040,
TransformerLM = 0x0050,
Custom = 0x00FF,
}
impl AprModelType {
#[must_use]
pub fn from_u16(value: u16) -> Option<Self> {
match value {
0x0001 => Some(Self::LinearRegression),
0x0002 => Some(Self::LogisticRegression),
0x0003 => Some(Self::DecisionTree),
0x0004 => Some(Self::RandomForest),
0x0005 => Some(Self::GradientBoosting),
0x0006 => Some(Self::KMeans),
0x0007 => Some(Self::Pca),
0x0008 => Some(Self::NaiveBayes),
0x0009 => Some(Self::Knn),
0x000A => Some(Self::Svm),
0x0010 => Some(Self::NgramLm),
0x0011 => Some(Self::Tfidf),
0x0012 => Some(Self::CountVectorizer),
0x0020 => Some(Self::NeuralSequential),
0x0021 => Some(Self::NeuralCustom),
0x0030 => Some(Self::ContentRecommender),
0x0040 => Some(Self::MixtureOfExperts),
0x0050 => Some(Self::TransformerLM),
0x00FF => Some(Self::Custom),
_ => None,
}
}
#[must_use]
pub const fn as_u16(self) -> u16 {
self as u16
}
}
#[derive(Debug, Clone)]
pub struct AprHeader {
pub magic: [u8; 4],
pub version: (u8, u8),
pub flags: u8,
pub reserved: u8,
pub model_type: AprModelType,
pub metadata_len: u32,
pub payload_len: u32,
pub original_size: u32,
pub reserved2: [u8; 10],
}
impl AprHeader {
pub fn from_bytes(data: &[u8]) -> Result<Self> {
if data.len() < HEADER_SIZE {
return Err(RealizarError::FormatError {
reason: format!(
".apr header too small: {} bytes (need {})",
data.len(),
HEADER_SIZE
),
});
}
let magic: [u8; 4] = data[0..4]
.try_into()
.map_err(|_| RealizarError::FormatError {
reason: "Failed to read magic bytes".to_string(),
})?;
if magic != MAGIC {
return Err(RealizarError::FormatError {
reason: format!("Invalid .apr magic: expected {MAGIC:?}, got {magic:?}"),
});
}
let version = (data[4], data[5]);
let flags = data[6];
let reserved = data[7];
let model_type_raw = u16::from_le_bytes([data[8], data[9]]);
let model_type =
AprModelType::from_u16(model_type_raw).ok_or_else(|| RealizarError::FormatError {
reason: format!("Unknown model type: 0x{model_type_raw:04X}"),
})?;
let metadata_len = u32::from_le_bytes([data[10], data[11], data[12], data[13]]);
let payload_len = u32::from_le_bytes([data[14], data[15], data[16], data[17]]);
let original_size = u32::from_le_bytes([data[18], data[19], data[20], data[21]]);
let reserved2: [u8; 10] =
data[22..32]
.try_into()
.map_err(|_| RealizarError::FormatError {
reason: "Failed to read reserved bytes".to_string(),
})?;
Ok(Self {
magic,
version,
flags,
reserved,
model_type,
metadata_len,
payload_len,
original_size,
reserved2,
})
}
#[must_use]
pub const fn is_compressed(&self) -> bool {
self.flags & 0x01 != 0
}
#[must_use]
pub const fn is_encrypted(&self) -> bool {
self.flags & 0x02 != 0
}
#[must_use]
pub const fn is_signed(&self) -> bool {
self.flags & 0x04 != 0
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelWeights {
pub weights: Vec<Vec<f32>>,
pub biases: Vec<Vec<f32>>,
pub dimensions: Vec<usize>,
}
#[derive(Debug)]
pub struct AprModel {
model_type: AprModelType,
weights: ModelWeights,
metadata: AprMetadata,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct AprMetadata {
pub name: Option<String>,
pub description: Option<String>,
pub trained_at: Option<String>,
pub framework_version: Option<String>,
pub custom: std::collections::HashMap<String, String>,
}
impl AprModel {
pub fn load<P: AsRef<Path>>(path: P) -> Result<Self> {
let data = fs::read(path.as_ref()).map_err(|e| RealizarError::IoError {
message: format!("Failed to read .apr file: {e}"),
})?;
Self::from_bytes(&data)
}
pub fn from_bytes(data: &[u8]) -> Result<Self> {
let header = AprHeader::from_bytes(data)?;
if header.version.0 > FORMAT_VERSION.0 {
return Err(RealizarError::FormatError {
reason: format!(
".apr version {}.{} not supported (max {}.{})",
header.version.0, header.version.1, FORMAT_VERSION.0, FORMAT_VERSION.1
),
});
}
if header.is_encrypted() {
return Err(RealizarError::FormatError {
reason: "Encrypted .apr files require aprender crate for decryption".to_string(),
});
}
let metadata_start = HEADER_SIZE;
let metadata_end = metadata_start + header.metadata_len as usize;
let payload_start = metadata_end;
let payload_end = payload_start + header.payload_len as usize;
if data.len() < payload_end {
return Err(RealizarError::FormatError {
reason: format!(
".apr file truncated: expected {} bytes, got {}",
payload_end,
data.len()
),
});
}
let metadata_bytes = &data[metadata_start..metadata_end];
let payload_bytes = &data[payload_start..payload_end];
let metadata: AprMetadata = if metadata_bytes.is_empty() {
AprMetadata::default()
} else {
serde_json::from_slice(metadata_bytes).unwrap_or_default()
};
let decompressed = if header.is_compressed() {
return Err(RealizarError::FormatError {
reason: "Compressed .apr files not yet supported in standalone mode".to_string(),
});
} else {
payload_bytes.to_vec()
};
let weights: ModelWeights =
serde_json::from_slice(&decompressed).map_err(|e| RealizarError::FormatError {
reason: format!("Failed to parse model weights: {e}"),
})?;
Ok(Self {
model_type: header.model_type,
weights,
metadata,
})
}
#[must_use]
pub const fn model_type(&self) -> AprModelType {
self.model_type
}
#[must_use]
pub const fn metadata(&self) -> &AprMetadata {
&self.metadata
}
#[must_use]
pub fn num_parameters(&self) -> usize {
let weight_params: usize = self.weights.weights.iter().map(Vec::len).sum();
let bias_params: usize = self.weights.biases.iter().map(Vec::len).sum();
weight_params + bias_params
}
pub fn predict(&self, input: &[f32]) -> Result<Vec<f32>> {
if self.weights.dimensions.is_empty() {
return Err(RealizarError::InvalidShape {
reason: "Model has no layers".to_string(),
});
}
let expected_input_dim = self.weights.dimensions[0];
if input.len() != expected_input_dim {
return Err(RealizarError::InvalidShape {
reason: format!(
"Input dimension mismatch: expected {}, got {}",
expected_input_dim,
input.len()
),
});
}
let mut current = input.to_vec();
for (i, (weights, biases)) in self
.weights
.weights
.iter()
.zip(self.weights.biases.iter())
.enumerate()
{
let in_dim = self.weights.dimensions[i];
let out_dim = self.weights.dimensions[i + 1];
let mut output = vec![0.0; out_dim];
for (j, out_val) in output.iter_mut().enumerate() {
let mut sum = biases.get(j).copied().unwrap_or(0.0);
for (k, &in_val) in current.iter().enumerate() {
let weight_idx = j * in_dim + k;
if let Some(&w) = weights.get(weight_idx) {
sum += in_val * w;
}
}
*out_val = sum;
}
if i < self.weights.weights.len() - 1 {
for val in &mut output {
*val = val.max(0.0); }
}
current = output;
}
Ok(current)
}
pub fn predict_batch(&self, inputs: &[Vec<f32>]) -> Result<Vec<Vec<f32>>> {
inputs.iter().map(|input| self.predict(input)).collect()
}
}
pub fn is_apr_file<P: AsRef<Path>>(path: P) -> bool {
fs::read(path.as_ref()).is_ok_and(|data| data.len() >= 4 && data[0..4] == MAGIC)
}
pub fn detect_format<P: AsRef<Path>>(path: P) -> &'static str {
let path = path.as_ref();
if let Some(ext) = path.extension() {
let ext = ext.to_string_lossy().to_lowercase();
match ext.as_str() {
"apr" => return "apr",
"gguf" => return "gguf",
"safetensors" => return "safetensors",
_ => {},
}
}
if let Ok(data) = fs::read(path) {
if data.len() >= 4 {
if data[0..4] == MAGIC {
return "apr";
}
if data[0..4] == [0x47, 0x47, 0x55, 0x46] {
return "gguf";
}
if data[0] == b'{' {
return "safetensors";
}
}
}
"unknown"
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_magic_constant() {
assert_eq!(MAGIC, [0x41, 0x50, 0x52, 0x4E]);
assert_eq!(&MAGIC, b"APRN");
}
#[test]
fn test_apr_model_type_from_u16() {
assert_eq!(
AprModelType::from_u16(0x0001),
Some(AprModelType::LinearRegression)
);
assert_eq!(
AprModelType::from_u16(0x0020),
Some(AprModelType::NeuralSequential)
);
assert_eq!(
AprModelType::from_u16(0x0040),
Some(AprModelType::MixtureOfExperts)
);
assert_eq!(
AprModelType::from_u16(0x0050),
Some(AprModelType::TransformerLM)
);
assert_eq!(AprModelType::from_u16(0xFFFF), None);
}
#[test]
fn test_apr_model_type_as_u16() {
assert_eq!(AprModelType::LinearRegression.as_u16(), 0x0001);
assert_eq!(AprModelType::NeuralSequential.as_u16(), 0x0020);
assert_eq!(AprModelType::MixtureOfExperts.as_u16(), 0x0040);
assert_eq!(AprModelType::TransformerLM.as_u16(), 0x0050);
}
#[test]
fn test_header_from_bytes_too_small() {
let data = vec![0u8; 10];
let result = AprHeader::from_bytes(&data);
assert!(result.is_err());
}
#[test]
fn test_header_from_bytes_invalid_magic() {
let mut data = vec![0u8; HEADER_SIZE];
data[0..4].copy_from_slice(b"GGUF"); let result = AprHeader::from_bytes(&data);
assert!(result.is_err());
}
#[test]
fn test_header_from_bytes_valid() {
let mut data = vec![0u8; HEADER_SIZE];
data[0..4].copy_from_slice(&MAGIC);
data[4] = 1; data[5] = 0; data[6] = 0x01; data[8..10].copy_from_slice(&0x0001u16.to_le_bytes());
let header = AprHeader::from_bytes(&data).expect("should parse");
assert_eq!(header.magic, MAGIC);
assert_eq!(header.version, (1, 0));
assert!(header.is_compressed());
assert!(!header.is_encrypted());
assert_eq!(header.model_type, AprModelType::LinearRegression);
}
#[test]
fn test_header_flags() {
let mut data = vec![0u8; HEADER_SIZE];
data[0..4].copy_from_slice(&MAGIC);
data[4] = 1;
data[5] = 0;
data[6] = 0x07; data[8..10].copy_from_slice(&0x0001u16.to_le_bytes());
let header = AprHeader::from_bytes(&data).expect("should parse");
assert!(header.is_compressed());
assert!(header.is_encrypted());
assert!(header.is_signed());
}
#[test]
fn test_model_weights_predict() {
let weights = ModelWeights {
weights: vec![vec![0.5, 0.5]], biases: vec![vec![0.0]],
dimensions: vec![2, 1],
};
let metadata = AprMetadata::default();
let model = AprModel {
model_type: AprModelType::LinearRegression,
weights,
metadata,
};
let input = vec![1.0, 1.0];
let output = model.predict(&input).expect("should predict");
assert_eq!(output.len(), 1);
assert!((output[0] - 1.0).abs() < 1e-6); }
#[test]
fn test_model_weights_predict_wrong_dim() {
let weights = ModelWeights {
weights: vec![vec![0.5, 0.5]],
biases: vec![vec![0.0]],
dimensions: vec![2, 1],
};
let model = AprModel {
model_type: AprModelType::LinearRegression,
weights,
metadata: AprMetadata::default(),
};
let input = vec![1.0]; let result = model.predict(&input);
assert!(result.is_err());
}
#[test]
fn test_detect_format_by_extension() {
assert_eq!(detect_format("/fake/model.apr"), "apr");
assert_eq!(detect_format("/fake/model.gguf"), "gguf");
assert_eq!(detect_format("/fake/model.safetensors"), "safetensors");
}
#[test]
fn test_num_parameters() {
let weights = ModelWeights {
weights: vec![vec![0.0; 6], vec![0.0; 3]], biases: vec![vec![0.0; 3], vec![0.0; 1]],
dimensions: vec![2, 3, 1],
};
let model = AprModel {
model_type: AprModelType::NeuralSequential,
weights,
metadata: AprMetadata::default(),
};
assert_eq!(model.num_parameters(), 6 + 3 + 3 + 1); }
#[test]
fn test_from_bytes_truncated() {
let mut data = vec![0u8; HEADER_SIZE];
data[0..4].copy_from_slice(&MAGIC);
data[4] = 1;
data[5] = 0;
data[8..10].copy_from_slice(&0x0001u16.to_le_bytes());
data[10..14].copy_from_slice(&100u32.to_le_bytes()); data[14..18].copy_from_slice(&100u32.to_le_bytes());
let result = AprModel::from_bytes(&data);
assert!(result.is_err());
}
struct ReproducibleRng {
state: u64,
}
impl ReproducibleRng {
fn new(seed: u64) -> Self {
Self { state: seed }
}
fn next_u64(&mut self) -> u64 {
self.state = self
.state
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1_442_695_040_888_963_407);
self.state
}
#[allow(clippy::cast_precision_loss)]
fn next_f32(&mut self, scale: f32) -> f32 {
let bits = self.next_u64();
let normalized = (bits >> 33) as f32 / (u32::MAX >> 1) as f32;
(normalized - 0.5) * 2.0 * scale
}
}
fn generate_test_weights(
input_dim: usize,
hidden_dim: usize,
output_dim: usize,
seed: u64,
) -> ModelWeights {
let mut rng = ReproducibleRng::new(seed);
let scale = 0.1;
let hidden_weights: Vec<f32> = (0..hidden_dim * input_dim)
.map(|_| rng.next_f32(scale))
.collect();
let hidden_biases: Vec<f32> = (0..hidden_dim).map(|_| rng.next_f32(scale * 0.1)).collect();
let output_weights: Vec<f32> = (0..output_dim * hidden_dim)
.map(|_| rng.next_f32(scale))
.collect();
let output_biases: Vec<f32> = (0..output_dim).map(|_| rng.next_f32(scale * 0.1)).collect();
ModelWeights {
weights: vec![hidden_weights, output_weights],
biases: vec![hidden_biases, output_biases],
dimensions: vec![input_dim, hidden_dim, output_dim],
}
}
fn create_test_apr_bytes(weights: &ModelWeights) -> Vec<u8> {
let payload = serde_json::to_vec(weights).expect("serialize");
let mut data = Vec::with_capacity(HEADER_SIZE + payload.len());
data.extend_from_slice(&MAGIC);
data.push(1); data.push(0); data.push(0); data.push(0); data.extend_from_slice(&AprModelType::NeuralSequential.as_u16().to_le_bytes());
data.extend_from_slice(&0u32.to_le_bytes()); #[allow(clippy::cast_possible_truncation)]
data.extend_from_slice(&(payload.len() as u32).to_le_bytes());
#[allow(clippy::cast_possible_truncation)]
data.extend_from_slice(&(payload.len() as u32).to_le_bytes()); data.extend_from_slice(&[0u8; 10]);
data.extend_from_slice(&payload);
data
}
const REPRODUCIBLE_SEED: u64 = 42;
#[test]
fn test_reproducibility_same_seed_same_weights() {
let weights1 = generate_test_weights(4, 8, 3, REPRODUCIBLE_SEED);
let weights2 = generate_test_weights(4, 8, 3, REPRODUCIBLE_SEED);
assert_eq!(weights1.weights, weights2.weights, "weights should match");
assert_eq!(weights1.biases, weights2.biases, "biases should match");
assert_eq!(
weights1.dimensions, weights2.dimensions,
"dimensions should match"
);
}
#[test]
fn test_reproducibility_different_seed_different_weights() {
let weights1 = generate_test_weights(4, 8, 3, 42);
let weights2 = generate_test_weights(4, 8, 3, 43);
assert_ne!(
weights1.weights, weights2.weights,
"different seeds should produce different weights"
);
}
#[test]
fn test_reproducibility_apr_bytes_identical() {
let weights = generate_test_weights(4, 8, 3, REPRODUCIBLE_SEED);
let bytes1 = create_test_apr_bytes(&weights);
let bytes2 = create_test_apr_bytes(&weights);
assert_eq!(bytes1, bytes2, "APR bytes should be identical");
}
#[test]
fn test_reproducibility_model_outputs_identical() {
let weights = generate_test_weights(4, 8, 3, REPRODUCIBLE_SEED);
let apr_bytes = create_test_apr_bytes(&weights);
let model1 = AprModel::from_bytes(&apr_bytes).expect("load model 1");
let model2 = AprModel::from_bytes(&apr_bytes).expect("load model 2");
let mut rng = ReproducibleRng::new(REPRODUCIBLE_SEED + 1000);
let input: Vec<f32> = (0..4).map(|_| rng.next_f32(1.0).abs()).collect();
let output1 = model1.predict(&input).expect("predict 1");
let output2 = model2.predict(&input).expect("predict 2");
assert_eq!(output1, output2, "model outputs should be identical");
}
#[test]
fn test_reproducibility_checksum_stable() {
let weights = generate_test_weights(4, 8, 3, REPRODUCIBLE_SEED);
let apr_bytes = create_test_apr_bytes(&weights);
let checksum: u64 = apr_bytes.iter().map(|&b| u64::from(b)).sum();
assert!(
checksum > 10000,
"checksum should be reasonable: {checksum}"
);
}
#[test]
fn test_reproducibility_batch_outputs_identical() {
let weights = generate_test_weights(4, 8, 3, REPRODUCIBLE_SEED);
let apr_bytes = create_test_apr_bytes(&weights);
let model = AprModel::from_bytes(&apr_bytes).expect("load model");
let batch: Vec<Vec<f32>> = (0..4)
.map(|i| {
let mut rng = ReproducibleRng::new(REPRODUCIBLE_SEED + 2000 + i);
(0..4).map(|_| rng.next_f32(1.0).abs()).collect()
})
.collect();
let outputs1 = model.predict_batch(&batch).expect("batch predict 1");
let outputs2 = model.predict_batch(&batch).expect("batch predict 2");
assert_eq!(outputs1, outputs2, "batch outputs should be identical");
}
}