hanzo_ai_format/
lib.rs

1//! Hanzo AI Format - Universal .ai file format for AI artifacts
2//!
3//! This crate provides a standard format for packaging, distributing, and sharing
4//! AI artifacts across the Hanzo/Zoo network. The .ai format supports:
5//!
6//! - Model weights and biases
7//! - Fine-tuning deltas (LoRA, QLoRA)
8//! - Quantized models (GGUF, AWQ, GPTQ)
9//! - Datasets and evaluation data
10//! - Embeddings and vector stores
11//! - Agent state and memory
12//! - Configuration and hyperparameters
13//!
14//! # File Structure
15//!
16//! ```text
17//! artifact.ai (ZIP archive with .ai extension)
18//! ├── manifest.json          # Artifact metadata and manifest
19//! ├── data/                   # Primary artifact data
20//! │   ├── weights/            # Model weights (safetensors, bin, etc.)
21//! │   ├── config/             # Model configuration
22//! │   └── tokenizer/          # Tokenizer files
23//! ├── delta/                  # Fine-tuning deltas (optional)
24//! ├── dataset/                # Training/eval datasets (optional)
25//! ├── embeddings/             # Pre-computed embeddings (optional)
26//! ├── state/                  # Agent state/memory (optional)
27//! └── signatures/             # Cryptographic signatures
28//! ```
29//!
30//! # Storage Backends
31//!
32//! The format supports multiple storage backends:
33//! - Local filesystem
34//! - HuggingFace Hub (fallback/mirror)
35//! - P2P swarm (BitTorrent-style distribution)
36//! - IPFS (content-addressed storage)
37//!
38//! # Example
39//!
40//! ```ignore
41//! use hanzo_ai_format::{AiArtifact, ArtifactType, Storage};
42//!
43//! // Create a new artifact
44//! let artifact = AiArtifact::builder()
45//!     .name("my-model")
46//!     .artifact_type(ArtifactType::Model)
47//!     .add_weights("weights.safetensors", weights_data)
48//!     .build()?;
49//!
50//! // Save to .ai file
51//! artifact.save("my-model.ai").await?;
52//!
53//! // Upload to storage
54//! let storage = Storage::new_with_hf_fallback();
55//! storage.upload(&artifact).await?;
56//! ```
57
58pub mod artifact;
59pub mod error;
60pub mod manifest;
61pub mod storage;
62
63pub use artifact::*;
64pub use error::*;
65pub use manifest::*;
66pub use storage::*;
67
68use chrono::{DateTime, Utc};
69use serde::{Deserialize, Serialize};
70use std::collections::HashMap;
71
72/// Magic bytes for .ai file format
73pub const AI_MAGIC: &[u8; 4] = b"HAIF"; // Hanzo AI Format
74
75/// Current format version
76pub const FORMAT_VERSION: u32 = 1;
77
78/// File extension for AI artifacts
79pub const AI_EXTENSION: &str = "ai";
80
81/// Artifact identifier (content-addressed)
82pub type ArtifactId = String;
83
84/// Unique identifier for artifacts
85#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
86pub struct ArtifactRef {
87    /// Content hash (Blake3)
88    pub hash: String,
89    /// Human-readable name
90    pub name: String,
91    /// Version
92    pub version: String,
93    /// Storage location hints
94    pub locations: Vec<StorageLocation>,
95}
96
97impl ArtifactRef {
98    pub fn new(hash: String, name: String, version: String) -> Self {
99        Self {
100            hash,
101            name,
102            version,
103            locations: Vec::new(),
104        }
105    }
106
107    pub fn id(&self) -> ArtifactId {
108        format!("{}@{}", self.name, self.hash[..8].to_string())
109    }
110
111    pub fn with_location(mut self, location: StorageLocation) -> Self {
112        self.locations.push(location);
113        self
114    }
115}
116
117/// Storage location for an artifact
118#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
119pub enum StorageLocation {
120    /// Local filesystem path
121    Local(String),
122    /// HuggingFace Hub repository
123    HuggingFace { repo_id: String, revision: Option<String> },
124    /// P2P swarm (BitTorrent-style)
125    Swarm { info_hash: String, peers: Vec<String> },
126    /// IPFS CID
127    Ipfs(String),
128    /// HTTP(S) URL
129    Http(String),
130    /// Node operator storage (peer ID + path)
131    NodeStorage { peer_id: String, path: String },
132}
133
134impl StorageLocation {
135    pub fn local(path: impl Into<String>) -> Self {
136        Self::Local(path.into())
137    }
138
139    pub fn huggingface(repo_id: impl Into<String>) -> Self {
140        Self::HuggingFace {
141            repo_id: repo_id.into(),
142            revision: None,
143        }
144    }
145
146    pub fn swarm(info_hash: impl Into<String>) -> Self {
147        Self::Swarm {
148            info_hash: info_hash.into(),
149            peers: Vec::new(),
150        }
151    }
152
153    pub fn ipfs(cid: impl Into<String>) -> Self {
154        Self::Ipfs(cid.into())
155    }
156
157    pub fn http(url: impl Into<String>) -> Self {
158        Self::Http(url.into())
159    }
160}
161
162/// Type of AI artifact
163#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
164pub enum ArtifactType {
165    /// Complete model (weights + config + tokenizer)
166    Model,
167    /// Model weights only
168    Weights,
169    /// Quantized model (GGUF, AWQ, GPTQ, etc.)
170    QuantizedModel {
171        format: QuantFormat,
172        bits: u8,
173    },
174    /// Fine-tuning delta (LoRA, QLoRA, etc.)
175    Delta {
176        base_model: String,
177        method: DeltaMethod,
178    },
179    /// Training or evaluation dataset
180    Dataset {
181        task: DatasetTask,
182        split: Option<String>,
183    },
184    /// Pre-computed embeddings
185    Embeddings {
186        model: String,
187        dimensions: usize,
188    },
189    /// Vector store / index
190    VectorStore {
191        index_type: String,
192        dimensions: usize,
193    },
194    /// Agent state / memory
195    AgentState {
196        agent_type: String,
197    },
198    /// Tokenizer only
199    Tokenizer,
200    /// Configuration only
201    Config,
202    /// Checkpoint during training
203    Checkpoint {
204        epoch: usize,
205        step: usize,
206    },
207    /// Custom artifact type
208    Custom(String),
209}
210
211/// Quantization format
212#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
213pub enum QuantFormat {
214    /// GGUF (llama.cpp format)
215    GGUF,
216    /// AWQ (Activation-aware Weight Quantization)
217    AWQ,
218    /// GPTQ
219    GPTQ,
220    /// ExLlamaV2
221    ExL2,
222    /// BitsAndBytes
223    BnB,
224    /// MLX quantization
225    MLX,
226    /// Custom format
227    Custom(String),
228}
229
230/// Fine-tuning delta method
231#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
232pub enum DeltaMethod {
233    /// Low-Rank Adaptation
234    LoRA { rank: usize, alpha: f32 },
235    /// Quantized LoRA
236    QLoRA { rank: usize, alpha: f32, bits: u8 },
237    /// Full fine-tuning diff
238    FullDiff,
239    /// Adapter layers
240    Adapter,
241    /// Prefix tuning
242    Prefix { num_tokens: usize },
243    /// Custom method
244    Custom(String),
245}
246
247/// Dataset task type
248#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
249pub enum DatasetTask {
250    TextGeneration,
251    Conversation,
252    Classification,
253    QuestionAnswering,
254    Summarization,
255    Translation,
256    CodeGeneration,
257    Embedding,
258    Custom(String),
259}
260
261/// License type for artifacts
262#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
263pub enum License {
264    MIT,
265    Apache2,
266    GPL3,
267    Llama2,
268    Llama3,
269    Qwen,
270    Gemma,
271    CcBy4,
272    CcByNc4,
273    CcBySa4,
274    Custom(String),
275    Proprietary,
276}
277
278impl Default for License {
279    fn default() -> Self {
280        Self::Apache2
281    }
282}
283
284/// Network where artifact is available
285#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
286pub enum Network {
287    HanzoMainnet,
288    HanzoTestnet,
289    ZooMainnet,
290    ZooTestnet,
291    All,
292}
293
294/// Compute requirements for using this artifact
295#[derive(Debug, Clone, Serialize, Deserialize, Default)]
296pub struct ComputeRequirements {
297    /// Minimum VRAM in MB
298    pub min_vram_mb: Option<u32>,
299    /// Minimum RAM in MB
300    pub min_ram_mb: Option<u32>,
301    /// GPU required
302    pub gpu_required: bool,
303    /// Supported backends
304    pub backends: Vec<String>,
305    /// Recommended batch size
306    pub recommended_batch_size: Option<usize>,
307}
308
309/// Metadata about the artifact
310#[derive(Debug, Clone, Serialize, Deserialize)]
311pub struct ArtifactMetadata {
312    /// Unique identifier
313    pub id: ArtifactId,
314    /// Human-readable name
315    pub name: String,
316    /// Version
317    pub version: String,
318    /// Description
319    pub description: Option<String>,
320    /// Artifact type
321    pub artifact_type: ArtifactType,
322    /// Author
323    pub author: Option<String>,
324    /// License
325    pub license: License,
326    /// Tags
327    pub tags: Vec<String>,
328    /// Creation timestamp
329    pub created_at: DateTime<Utc>,
330    /// Last modified timestamp
331    pub updated_at: DateTime<Utc>,
332    /// Total size in bytes
333    pub size_bytes: u64,
334    /// Content hash (Blake3)
335    pub content_hash: String,
336    /// Compute requirements
337    pub requirements: ComputeRequirements,
338    /// Networks where available
339    pub networks: Vec<Network>,
340    /// Storage locations
341    pub locations: Vec<StorageLocation>,
342    /// Dependencies (other artifacts)
343    pub dependencies: Vec<ArtifactRef>,
344    /// Custom metadata
345    pub custom: HashMap<String, serde_json::Value>,
346}
347
348impl ArtifactMetadata {
349    pub fn new(name: impl Into<String>, artifact_type: ArtifactType) -> Self {
350        let name = name.into();
351        let now = Utc::now();
352        Self {
353            id: uuid::Uuid::new_v4().to_string(),
354            name,
355            version: "1.0.0".to_string(),
356            description: None,
357            artifact_type,
358            author: None,
359            license: License::default(),
360            tags: Vec::new(),
361            created_at: now,
362            updated_at: now,
363            size_bytes: 0,
364            content_hash: String::new(),
365            requirements: ComputeRequirements::default(),
366            networks: vec![Network::All],
367            locations: Vec::new(),
368            dependencies: Vec::new(),
369            custom: HashMap::new(),
370        }
371    }
372
373    pub fn with_version(mut self, version: impl Into<String>) -> Self {
374        self.version = version.into();
375        self
376    }
377
378    pub fn with_description(mut self, desc: impl Into<String>) -> Self {
379        self.description = Some(desc.into());
380        self
381    }
382
383    pub fn with_author(mut self, author: impl Into<String>) -> Self {
384        self.author = Some(author.into());
385        self
386    }
387
388    pub fn with_license(mut self, license: License) -> Self {
389        self.license = license;
390        self
391    }
392
393    pub fn with_tag(mut self, tag: impl Into<String>) -> Self {
394        self.tags.push(tag.into());
395        self
396    }
397
398    pub fn with_network(mut self, network: Network) -> Self {
399        self.networks.push(network);
400        self
401    }
402
403    pub fn with_dependency(mut self, dep: ArtifactRef) -> Self {
404        self.dependencies.push(dep);
405        self
406    }
407}
408
409/// Statistics about artifact usage across the network
410#[derive(Debug, Clone, Default, Serialize, Deserialize)]
411pub struct ArtifactStats {
412    pub downloads: u64,
413    pub unique_users: u64,
414    pub compute_hours: f64,
415    pub peer_count: usize,
416    pub average_rating: Option<f32>,
417}
418
419#[cfg(test)]
420mod tests {
421    use super::*;
422
423    #[test]
424    fn test_artifact_ref() {
425        let artifact_ref = ArtifactRef::new(
426            "abc123def456".to_string(),
427            "my-model".to_string(),
428            "1.0.0".to_string(),
429        );
430        assert_eq!(artifact_ref.id(), "my-model@abc123de");
431    }
432
433    #[test]
434    fn test_storage_locations() {
435        let local = StorageLocation::local("/path/to/model");
436        assert!(matches!(local, StorageLocation::Local(_)));
437
438        let hf = StorageLocation::huggingface("hanzo-lm/Llama-3-8B");
439        assert!(matches!(hf, StorageLocation::HuggingFace { .. }));
440
441        let ipfs = StorageLocation::ipfs("QmXyz123");
442        assert!(matches!(ipfs, StorageLocation::Ipfs(_)));
443    }
444
445    #[test]
446    fn test_artifact_metadata() {
447        let metadata = ArtifactMetadata::new("test-model", ArtifactType::Model)
448            .with_version("2.0.0")
449            .with_author("hanzo")
450            .with_license(License::Apache2)
451            .with_tag("llm")
452            .with_tag("inference");
453
454        assert_eq!(metadata.name, "test-model");
455        assert_eq!(metadata.version, "2.0.0");
456        assert_eq!(metadata.author, Some("hanzo".to_string()));
457        assert_eq!(metadata.tags.len(), 2);
458    }
459
460    #[test]
461    fn test_artifact_types() {
462        let model = ArtifactType::Model;
463        let quant = ArtifactType::QuantizedModel {
464            format: QuantFormat::GGUF,
465            bits: 4,
466        };
467        let delta = ArtifactType::Delta {
468            base_model: "llama-3-8b".to_string(),
469            method: DeltaMethod::LoRA { rank: 64, alpha: 32.0 },
470        };
471
472        assert!(matches!(model, ArtifactType::Model));
473        assert!(matches!(quant, ArtifactType::QuantizedModel { .. }));
474        assert!(matches!(delta, ArtifactType::Delta { .. }));
475    }
476}
hanzo_ai_format/lib.rs

hanzo_ai_format/
lib.rs
