use crate::{AnyaError, AnyaResult};
use ipfs_api_backend_hyper::{IpfsApi, IpfsClient, TryFromUri, response::AddResponse};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, VecDeque};
use std::io::Cursor;
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio::sync::{RwLock, Mutex};
use futures::{TryStreamExt, stream::StreamExt};
use sha2::{Sha256, Digest};
pub type ContentId = String;
#[derive(Debug, Clone)]
pub struct IPFSConfig {
pub endpoint: String,
pub backup_endpoints: Vec<String>,
pub enable_pinning: bool,
pub pinning_services: Vec<PinningServiceConfig>,
pub max_file_size: usize,
pub timeout: u64,
pub encrypt_sensitive: bool,
pub dht_enabled: bool,
pub cluster_enabled: bool,
pub cid_version: u8,
pub hash_algorithm: String,
pub pubsub_enabled: bool,
}
#[derive(Debug, Clone)]
pub struct PinningServiceConfig {
pub name: String,
pub endpoint: String,
pub api_key: Option<String>,
pub enabled: bool,
}
impl Default for IPFSConfig {
fn default() -> Self {
Self {
endpoint: "http://127.0.0.1:5001".to_string(),
backup_endpoints: vec![
"https://ipfs.io/api/v0".to_string(),
"https://gateway.pinata.cloud".to_string(),
],
enable_pinning: true,
pinning_services: vec![
PinningServiceConfig {
name: "local".to_string(),
endpoint: "http://127.0.0.1:5001".to_string(),
api_key: None,
enabled: true,
},
],
max_file_size: 100 * 1024 * 1024, timeout: 30,
encrypt_sensitive: true,
dht_enabled: true,
cluster_enabled: false,
cid_version: 1, hash_algorithm: "sha2-256".to_string(),
pubsub_enabled: true,
}
}
}
#[derive(Debug)]
pub struct IPFSStorage {
client: IpfsClient,
backup_clients: Vec<IpfsClient>,
config: IPFSConfig,
pin_cache: Arc<RwLock<HashMap<ContentId, PinStatus>>>,
content_router: Arc<Mutex<ContentRouter>>,
batch_queue: Arc<Mutex<VecDeque<BatchOperation>>>,
metrics: Arc<StorageMetrics>,
}
#[derive(Debug)]
pub struct ContentRouter {
provider_cache: HashMap<ContentId, Vec<PeerId>>,
dht_cache: HashMap<String, DHTResult>,
cache_ttl: Duration,
}
pub type PeerId = String;
#[derive(Debug, Clone)]
pub struct DHTResult {
pub providers: Vec<PeerId>,
pub timestamp: SystemTime,
}
#[derive(Debug, Clone)]
pub enum BatchOperation {
Store { data: Vec<u8>, filename: Option<String> },
Pin { content_id: ContentId },
Unpin { content_id: ContentId },
}
#[derive(Debug, Clone)]
pub struct BatchResult {
pub successful: Vec<ContentId>,
pub failed: Vec<(ContentId, String)>,
pub total_size: u64,
pub duration: Duration,
}
#[derive(Debug, Default)]
pub struct StorageMetrics {
pub operations_count: u64,
pub total_bytes_stored: u64,
pub total_bytes_retrieved: u64,
pub cache_hits: u64,
pub cache_misses: u64,
pub dht_queries: u64,
pub pin_operations: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PinStatus {
pub pinned: bool,
pub pin_type: PinType,
pub timestamp: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum PinType {
Direct,
Indirect,
Recursive,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IPFSFileMetadata {
pub content_id: ContentId,
pub size: u64,
pub mime_type: Option<String>,
pub uploaded_at: u64,
pub encryption_used: bool,
pub pin_status: Option<PinStatus>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EncryptedContent {
pub encrypted_data: Vec<u8>,
pub encryption_method: String,
pub key_hint: String, }
impl IPFSStorage {
pub async fn new(config: IPFSConfig) -> AnyaResult<Self> {
let client = IpfsClient::from_str(&config.endpoint)
.map_err(|e| AnyaError::Storage(format!("IPFS client error: {}", e)))?;
let mut backup_clients = Vec::new();
for endpoint in &config.backup_endpoints {
if let Ok(backup_client) = IpfsClient::from_str(endpoint) {
backup_clients.push(backup_client);
}
}
match client.version().await {
Ok(version) => {
log::info!("Connected to IPFS node version: {}", version.version);
log::info!("IPFS node ID: {}", version.commit.unwrap_or_default());
}
Err(e) => {
log::warn!("IPFS primary connection test failed: {}", e);
for (i, backup) in backup_clients.iter().enumerate() {
if let Ok(version) = backup.version().await {
log::info!("Connected to backup IPFS node {}: {}", i, version.version);
break;
}
}
}
}
let content_router = if config.dht_enabled {
Arc::new(Mutex::new(ContentRouter::new()))
} else {
Arc::new(Mutex::new(ContentRouter::disabled()))
};
Ok(Self {
client,
backup_clients,
config,
pin_cache: Arc::new(RwLock::new(HashMap::new())),
content_router,
batch_queue: Arc::new(Mutex::new(VecDeque::new())),
metrics: Arc::new(StorageMetrics::default()),
})
}
pub async fn store_content(&self, data: &[u8], filename: Option<&str>) -> AnyaResult<IPFSFileMetadata> {
if data.len() > self.config.max_file_size {
return Err(AnyaError::Storage(format!(
"File size {} exceeds maximum allowed size {}",
data.len(),
self.config.max_file_size
)));
}
let (final_data, encryption_used) = if self.config.encrypt_sensitive {
let encrypted = self.encrypt_content(data)?;
(serde_json::to_vec(&encrypted)?, true)
} else {
(data.to_vec(), false)
};
let cursor = Cursor::new(final_data.clone());
let add_result = match self.upload_with_retry(&self.client, cursor).await {
Ok(result) => result,
Err(e) => {
log::warn!("Primary IPFS upload failed: {}, trying backups", e);
self.upload_with_backups(&final_data).await?
}
};
let content_id = add_result.hash.clone();
let size = add_result.size;
let pin_status = if self.config.enable_pinning {
match self.pin_content(&content_id).await {
Ok(status) => Some(status),
Err(e) => {
log::warn!("Failed to pin content {}: {}", content_id, e);
None
}
}
} else {
None
};
if self.config.dht_enabled {
if let Err(e) = self.announce_to_dht(&content_id).await {
log::warn!("Failed to announce content to DHT: {}", e);
}
}
{
let mut metrics = Arc::get_mut(&self.metrics).unwrap();
metrics.operations_count += 1;
metrics.total_bytes_stored += data.len() as u64;
}
Ok(IPFSFileMetadata {
content_id,
size,
mime_type: self.detect_mime_type(data, filename),
uploaded_at: SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(),
encryption_used,
pin_status,
})
}
pub async fn get_content(&self, content_id: &ContentId) -> AnyaResult<Vec<u8>> {
match self.get_local_content(content_id).await {
Ok(data) => {
let mut metrics = Arc::get_mut(&self.metrics).unwrap();
metrics.cache_hits += 1;
return Ok(data);
}
Err(_) => {
let mut metrics = Arc::get_mut(&self.metrics).unwrap();
metrics.cache_misses += 1;
}
}
if self.config.dht_enabled {
if let Ok(providers) = self.find_content_providers(content_id).await {
for provider in providers {
if let Ok(data) = self.retrieve_from_provider(content_id, &provider).await {
return Ok(data);
}
}
}
}
self.retrieve_content_fallback(content_id).await
}
pub async fn pin_content(&self, content_id: &ContentId) -> AnyaResult<PinStatus> {
{
let cache = self.pin_cache.read().await;
if let Some(status) = cache.get(content_id) {
if status.pinned &&
SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() - status.timestamp < 3600 {
return Ok(status.clone());
}
}
}
match self.client.pin_add(content_id, Some(true)).await {
Ok(_) => {
let status = PinStatus {
pinned: true,
pin_type: PinType::Recursive,
timestamp: SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(),
};
{
let mut cache = self.pin_cache.write().await;
cache.insert(content_id.clone(), status.clone());
}
if !self.config.pinning_services.is_empty() {
tokio::spawn({
let content_id = content_id.clone();
let services = self.config.pinning_services.clone();
async move {
for service in services {
if service.enabled {
log::debug!("Pinning {} to service {}", content_id, service.name);
}
}
}
});
}
Ok(status)
}
Err(e) => {
Err(AnyaError::Storage(format!("Pin operation failed: {}", e)))
}
}
}
pub async fn process_batch(&self, operations: Vec<BatchOperation>) -> AnyaResult<BatchResult> {
let start_time = SystemTime::now();
let mut successful = Vec::new();
let mut failed = Vec::new();
let mut total_size = 0u64;
let mut stores = Vec::new();
let mut pins = Vec::new();
let mut unpins = Vec::new();
for op in operations {
match op {
BatchOperation::Store { data, filename } => stores.push((data, filename)),
BatchOperation::Pin { content_id } => pins.push(content_id),
BatchOperation::Unpin { content_id } => unpins.push(content_id),
}
}
for (data, filename) in stores {
match self.store_content(&data, filename.as_deref()).await {
Ok(metadata) => {
successful.push(metadata.content_id);
total_size += metadata.size;
}
Err(e) => {
failed.push(("unknown".to_string(), e.to_string()));
}
}
}
let pin_futures: Vec<_> = pins.into_iter()
.map(|cid| self.pin_content(&cid))
.collect();
let pin_results = futures::future::join_all(pin_futures).await;
for (i, result) in pin_results.into_iter().enumerate() {
match result {
Ok(_) => successful.push(format!("pin_{}", i)),
Err(e) => failed.push((format!("pin_{}", i), e.to_string())),
}
}
let duration = SystemTime::now().duration_since(start_time)?;
Ok(BatchResult {
successful,
failed,
total_size,
duration,
})
}
fn encrypt_content(&self, data: &[u8]) -> AnyaResult<EncryptedContent> {
let key = b"anya_core_encryption_key_placeholder";
let mut encrypted = Vec::with_capacity(data.len());
for (i, &byte) in data.iter().enumerate() {
encrypted.push(byte ^ key[i % key.len()]);
}
Ok(EncryptedContent {
encrypted_data: encrypted,
encryption_method: "XOR-Placeholder".to_string(),
key_hint: "derive_from_user_did".to_string(),
})
}
async fn upload_with_retry(&self, client: &IpfsClient, cursor: Cursor<Vec<u8>>) -> AnyaResult<AddResponse> {
let mut attempts = 0;
let max_attempts = 3;
let mut delay = Duration::from_millis(1000);
loop {
match client.add(cursor.clone()).await {
Ok(result) => return Ok(result),
Err(e) if attempts >= max_attempts => {
return Err(AnyaError::Storage(format!("Upload failed after {} attempts: {}", max_attempts, e)));
}
Err(_) => {
attempts += 1;
tokio::time::sleep(delay).await;
delay *= 2; }
}
}
}
async fn upload_with_backups(&self, data: &[u8]) -> AnyaResult<AddResponse> {
for (i, backup) in self.backup_clients.iter().enumerate() {
let cursor = Cursor::new(data.to_vec());
match self.upload_with_retry(backup, cursor).await {
Ok(result) => {
log::info!("Successfully uploaded to backup client {}", i);
return Ok(result);
}
Err(e) => {
log::warn!("Backup client {} failed: {}", i, e);
}
}
}
Err(AnyaError::Storage("All IPFS clients failed".to_string()))
}
async fn get_local_content(&self, content_id: &ContentId) -> AnyaResult<Vec<u8>> {
let stream = self.client.cat(content_id);
let data = stream
.try_collect::<Vec<_>>()
.await
.map_err(|e| AnyaError::Storage(format!("IPFS cat error: {}", e)))?
.concat();
Ok(data)
}
async fn find_content_providers(&self, content_id: &ContentId) -> AnyaResult<Vec<PeerId>> {
{
let router = self.content_router.lock().await;
if let Some(providers) = router.provider_cache.get(content_id) {
return Ok(providers.clone());
}
}
let providers = vec![
"12D3KooWExample1".to_string(),
"12D3KooWExample2".to_string(),
];
{
let mut router = self.content_router.lock().await;
router.provider_cache.insert(content_id.clone(), providers.clone());
}
Ok(providers)
}
async fn retrieve_from_provider(&self, content_id: &ContentId, provider: &PeerId) -> AnyaResult<Vec<u8>> {
log::debug!("Attempting to retrieve {} from provider {}", content_id, provider);
self.get_local_content(content_id).await
}
async fn retrieve_content_fallback(&self, content_id: &ContentId) -> AnyaResult<Vec<u8>> {
if let Ok(data) = self.get_local_content(content_id).await {
return Ok(data);
}
for backup in &self.backup_clients {
let stream = backup.cat(content_id);
if let Ok(chunks) = stream.try_collect::<Vec<_>>().await {
let data = chunks.concat();
if !data.is_empty() {
return Ok(data);
}
}
}
Err(AnyaError::Storage(format!("Content not found: {}", content_id)))
}
async fn announce_to_dht(&self, content_id: &ContentId) -> AnyaResult<()> {
log::debug!("Announcing content {} to DHT", content_id);
Ok(())
}
fn detect_mime_type(&self, data: &[u8], filename: Option<&str>) -> Option<String> {
if let Some(name) = filename {
if name.ends_with(".json") {
return Some("application/json".to_string());
} else if name.ends_with(".png") {
return Some("image/png".to_string());
} else if name.ends_with(".jpg") || name.ends_with(".jpeg") {
return Some("image/jpeg".to_string());
}
}
if data.starts_with(b"{") || data.starts_with(b"[") {
Some("application/json".to_string())
} else if data.starts_with(b"\x89PNG") {
Some("image/png".to_string())
} else if data.starts_with(&[0xFF, 0xD8, 0xFF]) {
Some("image/jpeg".to_string())
} else {
Some("application/octet-stream".to_string())
}
}
}
impl ContentRouter {
fn new() -> Self {
Self {
provider_cache: HashMap::new(),
dht_cache: HashMap::new(),
cache_ttl: Duration::from_secs(300), }
}
fn disabled() -> Self {
Self::new()
}
}
.await
.map_err(|e| AnyaError::Storage(format!("IPFS pin list error: {}", e)))?;
Ok(pins.keys.into_keys().collect())
}
pub async fn get_node_info(&self) -> AnyaResult<serde_json::Value> {
let id = self.client
.id(None)
.await
.map_err(|e| AnyaError::Storage(format!("IPFS id error: {}", e)))?;
let version = self.client
.version()
.await
.map_err(|e| AnyaError::Storage(format!("IPFS version error: {}", e)))?;
Ok(serde_json::json!({
"id": id.id,
"public_key": id.public_key,
"addresses": id.addresses,
"agent_version": id.agent_version,
"protocol_version": id.protocol_version,
"version": version.version,
"commit": version.commit,
"repo": version.repo,
"system": version.system,
"golang": version.golang
}))
}
pub async fn store_directory(&self, files: Vec<(String, Vec<u8>)>) -> AnyaResult<Vec<IPFSFileMetadata>> {
let mut metadata_list = Vec::new();
for (filename, data) in files {
let metadata = self.store_content(&data, Some(&filename)).await?;
metadata_list.push(metadata);
}
Ok(metadata_list)
}
pub async fn create_directory_structure(
&self,
structure: HashMap<String, Vec<u8>>,
) -> AnyaResult<ContentId> {
let mut add_requests = Vec::new();
for (path, content) in structure {
let cursor = Cursor::new(content);
add_requests.push((path, cursor));
}
let mut last_cid = String::new();
for (path, content) in add_requests {
let add_result = self.client
.add(content)
.await
.map_err(|e| AnyaError::Storage(format!("IPFS directory add error: {}", e)))?;
last_cid = add_result.hash;
log::debug!("Added file to directory: {} -> {}", path, last_cid);
}
Ok(last_cid)
}
pub async fn garbage_collect(&self) -> AnyaResult<serde_json::Value> {
let gc_result = self.client
.repo_gc()
.await
.map_err(|e| AnyaError::Storage(format!("IPFS garbage collection error: {}", e)))?;
Ok(serde_json::json!({
"collected_objects": gc_result.len(),
"collected_cids": gc_result
}))
}
pub async fn get_repo_stats(&self) -> AnyaResult<serde_json::Value> {
let stats = self.client
.repo_stat()
.await
.map_err(|e| AnyaError::Storage(format!("IPFS repo stat error: {}", e)))?;
Ok(serde_json::json!({
"num_objects": stats.num_objects,
"repo_size": stats.repo_size,
"repo_path": stats.repo_path,
"version": stats.version,
"storage_max": stats.storage_max
}))
}
fn encrypt_content(&self, data: &[u8]) -> AnyaResult<EncryptedContent> {
Ok(EncryptedContent {
encrypted_data: data.to_vec(), encryption_method: "chacha20poly1305".to_string(),
key_hint: "user-derived-key".to_string(),
})
}
fn decrypt_content(&self, encrypted: &EncryptedContent) -> AnyaResult<Vec<u8>> {
match encrypted.encryption_method.as_str() {
"chacha20poly1305" => Ok(encrypted.encrypted_data.clone()),
_ => Err(AnyaError::CryptographyError(
format!("Unsupported encryption method: {}", encrypted.encryption_method)
)),
}
}
fn mime_type_from_extension(&self, extension: &str) -> String {
match extension.to_lowercase().as_str() {
"json" => "application/json",
"txt" => "text/plain",
"html" => "text/html",
"css" => "text/css",
"js" => "application/javascript",
"png" => "image/png",
"jpg" | "jpeg" => "image/jpeg",
"gif" => "image/gif",
"pdf" => "application/pdf",
"zip" => "application/zip",
"mp4" => "video/mp4",
"mp3" => "audio/mpeg",
_ => "application/octet-stream",
}.to_string()
}
}
#[derive(Debug)]
pub struct IPFSBatch {
storage: IPFSStorage,
operations: Vec<BatchOperation>,
}
#[derive(Debug)]
enum BatchOperation {
Store { data: Vec<u8>, filename: Option<String> },
Pin { content_id: String },
Unpin { content_id: String },
}
impl IPFSBatch {
pub fn new(storage: IPFSStorage) -> Self {
Self {
storage,
operations: Vec::new(),
}
}
pub fn add_store(&mut self, data: Vec<u8>, filename: Option<String>) {
self.operations.push(BatchOperation::Store { data, filename });
}
pub fn add_pin(&mut self, content_id: String) {
self.operations.push(BatchOperation::Pin { content_id });
}
pub fn add_unpin(&mut self, content_id: String) {
self.operations.push(BatchOperation::Unpin { content_id });
}
pub async fn execute(self) -> AnyaResult<Vec<BatchResult>> {
let mut results = Vec::new();
for operation in self.operations {
let result = match operation {
BatchOperation::Store { data, filename } => {
match self.storage.store_content(&data, filename.as_deref()).await {
Ok(metadata) => BatchResult::StoreSuccess(metadata),
Err(e) => BatchResult::Error(e.to_string()),
}
}
BatchOperation::Pin { content_id } => {
match self.storage.pin_content(&content_id).await {
Ok(status) => BatchResult::PinSuccess(status),
Err(e) => BatchResult::Error(e.to_string()),
}
}
BatchOperation::Unpin { content_id } => {
match self.storage.unpin_content(&content_id).await {
Ok(()) => BatchResult::UnpinSuccess,
Err(e) => BatchResult::Error(e.to_string()),
}
}
};
results.push(result);
}
Ok(results)
}
}
#[derive(Debug)]
pub enum BatchResult {
StoreSuccess(IPFSFileMetadata),
PinSuccess(PinStatus),
UnpinSuccess,
Error(String),
}
fn current_timestamp() -> u64 {
use std::time::{SystemTime, UNIX_EPOCH};
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_ipfs_storage_creation() {
let config = IPFSConfig::default();
let storage = IPFSStorage::new(config).await;
assert!(storage.is_ok() || storage.is_err());
}
#[test]
fn test_mime_type_detection() {
let storage = IPFSStorage {
client: IpfsClient::from_str("http://127.0.0.1:5001").unwrap(),
config: IPFSConfig::default(),
pin_cache: Arc::new(RwLock::new(HashMap::new())),
};
assert_eq!(storage.mime_type_from_extension("json"), "application/json");
assert_eq!(storage.mime_type_from_extension("txt"), "text/plain");
assert_eq!(storage.mime_type_from_extension("unknown"), "application/octet-stream");
}
#[test]
fn test_batch_creation() {
let storage = IPFSStorage {
client: IpfsClient::from_str("http://127.0.0.1:5001").unwrap(),
config: IPFSConfig::default(),
pin_cache: Arc::new(RwLock::new(HashMap::new())),
};
let mut batch = IPFSBatch::new(storage);
batch.add_store(b"test data".to_vec(), Some("test.txt".to_string()));
batch.add_pin("QmTest123".to_string());
assert_eq!(batch.operations.len(), 2);
}
}