ipfrs_interface/lib.rs
1//! IPFRS Interface - High-performance HTTP Gateway and API Layer
2//!
3//! This crate provides a comprehensive API layer for IPFRS (InterPlanetary File & Reasoning System),
4//! offering multiple interfaces for accessing distributed content and computation.
5//!
6//! # Features
7//!
8//! ## HTTP Gateway
9//! - **Kubo-compatible API** (`/api/v0/*`) - Full compatibility with IPFS Kubo clients
10//! - **High-speed v1 API** (`/v1/*`) - Optimized endpoints with batch operations
11//! - **Content gateway** (`/ipfs/{cid}`) - Direct content retrieval with range requests
12//! - **Multi-range support** - Efficient sparse downloads with HTTP 206
13//!
14//! ## gRPC Interface
15//! - **BlockService** - Raw block operations (Get, Put, Has, Delete, Batch)
16//! - **DagService** - DAG operations (Get, Put, Resolve, Traverse)
17//! - **FileService** - File operations (Add, Get, List, Pin)
18//! - **TensorService** - Zero-copy tensor operations (Get, Slice, Stream)
19//! - **Streaming RPCs** - Client, server, and bidirectional streaming
20//! - **Interceptors** - Authentication, logging, metrics, rate limiting
21//!
22//! ## WebSocket Support
23//! - **Real-time events** - Block additions, peer connections, DHT queries
24//! - **Pub/sub system** - Topic-based event subscriptions
25//! - **Connection management** - Automatic cleanup and heartbeat
26//!
27//! ## Advanced Features
28//! - **Zero-copy tensor API** - Efficient ML model distribution via Safetensors
29//! - **Streaming uploads/downloads** - Memory-efficient large file handling
30//! - **Batch operations** - Parallel processing with atomic transactions
31//! - **Flow control** - Adaptive window-based congestion control
32//! - **Resume/cancel** - Robust transfer management
33//!
34//! ## Security & Performance
35//! - **Authentication** - JWT tokens, API keys, and OAuth2 (Authorization Code, Client Credentials, PKCE)
36//! - **Rate limiting** - Token bucket algorithm with per-client limits
37//! - **CORS** - Configurable cross-origin resource sharing
38//! - **Compression** - Gzip, Brotli, and Deflate with tunable levels
39//! - **HTTP caching** - ETag and Cache-Control for CDN optimization
40//! - **TLS/HTTPS** - Production-ready SSL/TLS support
41//!
42//! # Quick Start
43//!
44//! ```rust,ignore
45//! // Example usage (see examples/server.rs for a complete working example)
46//! use ipfrs_interface::GatewayConfig;
47//! use ipfrs_storage::blockstore::BlockStoreConfig;
48//!
49//! #[tokio::main]
50//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
51//! // Create gateway with default configuration
52//! let storage_config = BlockStoreConfig::default();
53//! let mut gateway_config = GatewayConfig::default();
54//! gateway_config.storage_config = storage_config;
55//! gateway_config.listen_addr = "127.0.0.1:8080".to_string();
56//!
57//! // See the examples directory for complete implementation
58//! Ok(())
59//! }
60//! ```
61//!
62//! # Examples
63//!
64//! ## Upload a file (Kubo v0 API)
65//! ```bash
66//! curl -X POST -F "file=@myfile.txt" http://localhost:8080/api/v0/add
67//! ```
68//!
69//! ## Download via gateway
70//! ```bash
71//! curl http://localhost:8080/ipfs/<CID>
72//! ```
73//!
74//! ## Batch block operations (v1 API)
75//! ```bash
76//! curl -X POST -H "Content-Type: application/json" \
77//! -d '{"cids":["<CID1>","<CID2>"]}' \
78//! http://localhost:8080/v1/block/batch/get
79//! ```
80//!
81//! ## Get tensor slice (zero-copy)
82//! ```bash
83//! curl "http://localhost:8080/v1/tensor/<CID>?slice=0:10,5:15"
84//! ```
85//!
86//! # Architecture
87//!
88//! ```text
89//! ┌─────────────────┐
90//! │ HTTP Clients │ (curl, browsers, IPFS clients)
91//! └────────┬────────┘
92//! │
93//! ┌────────▼────────┐
94//! │ Axum Router │ (HTTP/HTTPS)
95//! ├─────────────────┤
96//! │ Middleware │ (CORS, Auth, Rate Limit, Compression)
97//! ├─────────────────┤
98//! │ API Handlers │ (v0, v1, Gateway, WebSocket)
99//! └────────┬────────┘
100//! │
101//! ┌────────▼────────┐
102//! │ gRPC Server │ (Tonic)
103//! ├─────────────────┤
104//! │ Interceptors │ (Auth, Logging, Metrics)
105//! ├─────────────────┤
106//! │ Services │ (Block, DAG, File, Tensor)
107//! └────────┬────────┘
108//! │
109//! ┌────────▼────────┐
110//! │ Core Layer │ (ipfrs-core, ipfrs-storage, etc.)
111//! └─────────────────┘
112//! ```
113//!
114//! # Performance
115//!
116//! Target performance characteristics:
117//! - Request latency: < 10ms (simple GET)
118//! - Throughput: > 1GB/s (range requests)
119//! - Concurrent connections: 10,000+
120//! - Memory per connection: < 100KB
121//!
122//! # See Also
123//!
124//! - [`Gateway`] - Main HTTP gateway implementation
125//! - [`middleware`] - CORS, rate limiting, and compression
126//! - [`streaming`] - Streaming operations and flow control
127//! - [`websocket`] - WebSocket real-time events
128
129pub mod arrow;
130pub mod auth;
131pub mod auth_handlers;
132pub mod backpressure;
133pub mod binary_protocol;
134pub mod ffi;
135pub mod gateway;
136pub mod gradient_sync;
137pub mod graphql;
138// The `grpc` module contains hand-written gRPC service implementations that
139// depend on tonic-generated protobuf code produced by `build.rs`. Because
140// tonic-build 0.14 service generation requires proto files to be present at
141// compile time and the feature-gated `grpc` build is still being stabilised,
142// the module is compiled only when the `grpc` Cargo feature is enabled.
143// See `crates/ipfrs-interface/build.rs` for the build configuration.
144#[cfg(feature = "grpc")]
145pub mod grpc;
146pub mod metrics;
147pub mod metrics_middleware;
148pub mod middleware;
149pub mod mmap;
150pub mod oauth2;
151pub mod python;
152pub mod safetensors;
153pub mod streaming;
154pub mod tensor;
155pub mod tls;
156pub mod websocket;
157pub mod zerocopy;
158
159pub use backpressure::{
160 BackpressureConfig, BackpressureController, BackpressureError, BackpressurePermit,
161 BackpressureStream,
162};
163pub use binary_protocol::{
164 BinaryMessage, ErrorResponse, GetBlockRequest, HasBlockRequest, MessageType, ProtocolError,
165 PutBlockRequest, SuccessResponse, PROTOCOL_VERSION,
166};
167pub use gateway::{Gateway, GatewayConfig};
168pub use gradient_sync::{GradientChunkResponse, GradientSyncRequest, GradientSyncService};
169pub use graphql::{create_schema, IpfrsSchema};
170pub use middleware::{
171 cors_middleware, rate_limit_middleware, CacheConfig, CompressionConfig, CompressionLevel,
172 CorsConfig, CorsState, RateLimitConfig, RateLimitState,
173};
174pub use mmap::{MmapCache, MmapConfig, MmapError, MmapFile};
175pub use oauth2::{
176 AccessToken, AuthorizationCode, CodeChallengeMethod, ErrorResponse as OAuth2ErrorResponse,
177 GrantType, OAuth2Client, OAuth2ProviderConfig, OAuth2Server, RefreshToken, ResponseType, Scope,
178 TokenResponse, TokenType,
179};
180pub use safetensors::{SafetensorsFile, SafetensorsHeader, TensorData, TensorInfo};
181pub use streaming::{
182 ConcurrencyConfig, FlowControlConfig, FlowController, OperationState, OperationStatus,
183 OperationType, ProgressEvent, ProgressTracker, ResumeToken,
184};
185pub use tensor::{TensorLayout, TensorMetadata, TensorSlice};
186pub use websocket::{ws_handler, RealtimeEvent, SubscriptionManager, WsMessage, WsState};
187pub use zerocopy::ZeroCopyBuffer;
188
189// gRPC exports — available only when the `grpc` feature is enabled.
190// The feature gate keeps the default build 100% pure-Rust without proto
191// compilation; enable `grpc` to get full tonic-backed service types.
192#[cfg(feature = "grpc")]
193pub use grpc::{
194 backpressure_support, AuthInterceptor, BlockServiceImpl, BlockServiceServer,
195 ChainedInterceptor, DagServiceImpl, DagServiceServer, FileServiceImpl, FileServiceServer,
196 GrpcServiceConfig, LoggingInterceptor, MetricsInterceptor, RateLimitInterceptor,
197 TensorServiceImpl, TensorServiceServer,
198};