ipfrs_core/
lib.rs

1//! # IPFRS Core
2//!
3//! Core types and traits for the IPFRS (InterPlanetary File Replication System).
4//!
5//! This crate provides fundamental building blocks for content-addressed storage:
6//!
7//! - **[`Block`]** - Content-addressed data blocks with CID verification
8//! - **[`Cid`]** - Content Identifiers for unique data addressing
9//! - **[`Ipld`]** - InterPlanetary Linked Data for structured content
10//! - **Chunking** - Split large files into Merkle DAG structures
11//! - **Streaming** - Async readers for DAG traversal
12//!
13//! ## Quick Start
14//!
15//! ```rust
16//! use ipfrs_core::{Block, CidBuilder};
17//! use bytes::Bytes;
18//!
19//! // Create a block from data
20//! let block = Block::new(Bytes::from_static(b"Hello, IPFS!")).unwrap();
21//! println!("CID: {}", block.cid());
22//!
23//! // Generate a CID directly
24//! let cid = CidBuilder::new().build(b"some data").unwrap();
25//! println!("Generated CID: {}", cid);
26//! ```
27//!
28//! ## Chunking Large Files
29//!
30//! ```rust
31//! use ipfrs_core::{Chunker, ChunkingConfig};
32//!
33//! let data = vec![0u8; 1_000_000]; // 1MB of data
34//! let chunker = Chunker::new();
35//! let chunked = chunker.chunk(&data).unwrap();
36//!
37//! println!("Root CID: {}", chunked.root_cid);
38//! println!("Chunks: {}", chunked.chunk_count);
39//! ```
40//!
41//! ## IPLD Encoding
42//!
43//! ```rust
44//! use ipfrs_core::Ipld;
45//! use std::collections::BTreeMap;
46//!
47//! // Create structured data
48//! let mut map = BTreeMap::new();
49//! map.insert("name".to_string(), Ipld::String("example".to_string()));
50//! map.insert("version".to_string(), Ipld::Integer(1));
51//! let ipld = Ipld::Map(map);
52//!
53//! // Encode to DAG-CBOR
54//! let cbor = ipld.to_dag_cbor().unwrap();
55//!
56//! // Decode back
57//! let decoded = Ipld::from_dag_cbor(&cbor).unwrap();
58//! ```
59//!
60//! ## Features
61//!
62//! - **SHA2-256, SHA2-512, SHA3-256, SHA3-512, BLAKE2b, BLAKE2s, and BLAKE3** hash algorithms with SIMD acceleration
63//! - **CIDv0 and CIDv1** support with conversion
64//! - **Multibase encoding** (Base32, Base58btc, Base64)
65//! - **DAG-CBOR, DAG-JSON, and DAG-JOSE** codecs
66//! - **Pluggable codec registry** for custom encoding/decoding
67//! - **DAG traversal and analysis** utilities for Merkle DAGs
68//! - **CAR (Content Addressable aRchive)** format support for data portability
69//! - **Compression support** with Zstd and LZ4 algorithms for storage efficiency
70//! - **Streaming compression** for efficient compression/decompression of large files
71//! - **Async streaming** for large files
72//! - **LRU block cache** for fast repeated access to frequently used blocks
73//! - **Apache Arrow integration** for zero-copy tensor access
74//! - **Parallel batch processing** with Rayon for high performance
75//! - **Parallel chunking** for multi-core large file processing
76//! - **Content-defined chunking** with deduplication
77//! - **Production metrics** and observability with percentile tracking
78
79pub mod arrow;
80pub mod batch;
81pub mod block;
82pub mod block_cache;
83pub mod car;
84pub mod chunking;
85pub mod cid;
86pub mod codec_registry;
87pub mod compression;
88pub mod config;
89pub mod dag;
90pub mod error;
91pub mod hash;
92pub mod integration;
93pub mod ipld;
94pub mod jose;
95pub mod manifest;
96pub mod merkle_batch;
97pub mod metrics;
98pub mod parallel_chunking;
99pub mod pool;
100pub mod safetensors;
101pub mod streaming;
102pub mod streaming_compression;
103pub mod tensor;
104pub mod types;
105pub mod utils;
106pub mod wasm_compat;
107
108pub use self::arrow::{
109    arrow_dtype_to_tensor, arrow_to_tensor_block, tensor_dtype_to_arrow, TensorBlockArrowExt,
110};
111pub use self::batch::{BatchProcessor, BatchStats};
112pub use self::block::{Block, BlockBuilder, BlockMetadata, MAX_BLOCK_SIZE, MIN_BLOCK_SIZE};
113pub use self::block_cache::{BlockCache, CacheStats};
114pub use self::car::{CarCompressionStats, CarHeader, CarReader, CarWriter, CarWriterBuilder};
115pub use self::chunking::{
116    ChunkedFile, Chunker, ChunkingConfig, ChunkingConfigBuilder, ChunkingStrategy, DagBuilder,
117    DagLink, DagNode, DeduplicationStats,
118};
119pub use self::cid::{
120    codec, parse_cid, parse_cid_with_base, Cid, CidBuilder, CidExt, HashAlgorithm,
121    MultibaseEncoding,
122};
123pub use self::codec_registry::{
124    global_codec_registry, Codec, CodecRegistry, DagCborCodec, DagJsonCodec, RawCodec,
125};
126pub use self::compression::{compress, compression_ratio, decompress, CompressionAlgorithm};
127pub use self::config::{global_config, set_global_config, Config, ConfigBuilder};
128pub use self::dag::{
129    collect_all_links, collect_unique_links, count_links_by_depth, dag_fanout_by_level,
130    extract_links, filter_dag, find_paths_to_cid, is_dag, map_dag, subgraph_size, topological_sort,
131    traverse_bfs, traverse_dfs, DagMetrics, DagStats,
132};
133pub use self::error::{Error, Result};
134pub use self::hash::{
135    global_hash_registry, Blake2b256Engine, Blake2b512Engine, Blake2s256Engine, Blake3Engine,
136    CpuFeatures, HashEngine, HashRegistry, Sha256Engine, Sha3_256Engine, Sha3_512Engine,
137    Sha512Engine,
138};
139pub use self::integration::{
140    DeduplicationStats as TensorDeduplicationStats, TensorBatchProcessor, TensorDeduplicator,
141    TensorStore,
142};
143pub use self::ipld::Ipld;
144pub use self::jose::{JoseBuilder, JoseSignature};
145pub use self::manifest::{ContentManifest, ManifestDiff, ManifestEntry, MerkleTree};
146pub use self::metrics::{global_metrics, Metrics, MetricsSnapshot, PercentileStats, Timer};
147pub use self::parallel_chunking::{
148    ParallelChunker, ParallelChunkingConfig, ParallelChunkingResult, ParallelDeduplicator,
149};
150pub use self::pool::{
151    freeze_bytes, global_bytes_pool, global_cid_string_pool, BytesPool, CidStringPool, PoolStats,
152};
153pub use self::safetensors::{SafetensorInfo, SafetensorsFile};
154pub use self::streaming::{
155    read_chunked_file, AsyncBlockReader, BlockFetcher, BlockReader, DagChunkStream,
156    MemoryBlockFetcher,
157};
158pub use self::streaming_compression::{CompressingStream, DecompressingStream, StreamingStats};
159pub use self::tensor::{TensorBlock, TensorDtype, TensorMetadata, TensorShape};
160pub use self::types::{BlockSize, PeerId, Priority};
161pub use self::wasm_compat::{PlatformTime, TargetCapabilities, IS_WASM32};
ipfrs_core/lib.rs

ipfrs_core/
lib.rs
