use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::RwLock;
use uuid::Uuid;
use zeroize::Zeroize;
use crate::error::{Result, VaultError};
pub type NodeId = String;
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct VectorClock {
pub timestamps: HashMap<NodeId, u64>,
}
impl VectorClock {
pub fn new() -> Self {
Self::default()
}
pub fn increment(&mut self, node_id: &str) {
*self.timestamps.entry(node_id.to_string()).or_insert(0) += 1;
}
pub fn merge(&mut self, other: &VectorClock) {
for (node, ts) in &other.timestamps {
let entry = self.timestamps.entry(node.clone()).or_insert(0);
*entry = (*entry).max(*ts);
}
}
pub fn compare(&self, other: &VectorClock) -> ClockComparison {
let mut dominated = true;
let mut dominates = true;
for (node, ts) in &self.timestamps {
let other_ts = other.timestamps.get(node).copied().unwrap_or(0);
if *ts > other_ts {
dominated = false;
}
if *ts < other_ts {
dominates = false;
}
}
for (node, ts) in &other.timestamps {
if !self.timestamps.contains_key(node) && *ts > 0 {
dominates = false;
}
}
match (dominates, dominated) {
(true, true) => ClockComparison::Equal,
(true, false) => ClockComparison::After,
(false, true) => ClockComparison::Before,
(false, false) => ClockComparison::Concurrent,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ClockComparison {
Equal,
Before,
After,
Concurrent,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FederationConfig {
pub node_id: NodeId,
pub node_name: String,
pub peers: Vec<PeerConfig>,
pub sync_interval_secs: u64,
pub auto_resolve_conflicts: bool,
pub max_concurrent_syncs: usize,
}
impl Default for FederationConfig {
fn default() -> Self {
Self {
node_id: Uuid::new_v4().to_string(),
node_name: hostname::get()
.map(|h| h.to_string_lossy().into_owned())
.unwrap_or_else(|_| "unknown".into()),
peers: Vec::new(),
sync_interval_secs: 300, auto_resolve_conflicts: true,
max_concurrent_syncs: 4,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PeerConfig {
pub node_id: NodeId,
pub name: String,
pub endpoint: String,
pub api_key: Option<String>,
pub enabled: bool,
}
impl Drop for PeerConfig {
fn drop(&mut self) {
if let Some(ref mut key) = self.api_key {
key.zeroize();
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelSyncState {
pub name: String,
pub clock: VectorClock,
pub last_sync: HashMap<NodeId, DateTime<Utc>>,
pub known_versions: HashSet<String>, pub pending_upload: HashSet<String>,
pub pending_download: HashSet<(NodeId, String)>, }
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SyncManifest {
pub source_node: NodeId,
pub timestamp: DateTime<Utc>,
pub models: Vec<ModelManifestEntry>,
pub clock: VectorClock,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelManifestEntry {
pub name: String,
pub versions: Vec<VersionManifestEntry>,
pub clock: VectorClock,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VersionManifestEntry {
pub version: u32,
pub checkpoint_id: String,
pub created_at: DateTime<Utc>,
pub checksum: String,
pub size_bytes: u64,
pub parent_id: Option<String>,
pub origin_node: NodeId,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SyncResult {
pub peer_id: NodeId,
pub timestamp: DateTime<Utc>,
pub duration_ms: u64,
pub models_synced: u32,
pub versions_uploaded: u32,
pub versions_downloaded: u32,
pub conflicts: Vec<SyncConflict>,
pub errors: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SyncConflict {
pub model: String,
pub local_version: String,
pub remote_version: String,
pub remote_node: NodeId,
pub resolution: Option<ConflictResolution>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ConflictResolution {
KeepLocal,
UseRemote,
Branch {
local_name: String,
remote_name: String,
},
Manual,
}
pub struct FederationManager {
config: FederationConfig,
state: Arc<RwLock<FederationState>>,
http_client: reqwest::Client,
}
struct FederationState {
models: HashMap<String, ModelSyncState>,
clock: VectorClock,
history: Vec<SyncResult>,
state_file: PathBuf,
}
impl FederationManager {
pub fn new(config: FederationConfig, state_dir: PathBuf) -> Result<Self> {
let state_file = state_dir.join("federation_state.json");
let federation_state = if state_file.exists() {
let contents = std::fs::read_to_string(&state_file)?;
let loaded: SavedFederationState = serde_json::from_str(&contents)?;
FederationState {
models: loaded.models,
clock: loaded.clock,
history: loaded.history,
state_file,
}
} else {
FederationState {
models: HashMap::new(),
clock: VectorClock::new(),
history: Vec::new(),
state_file,
}
};
let http_client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(300))
.build()
.map_err(|e| {
VaultError::IoError(std::io::Error::other(format!("HTTP client error: {e}")))
})?;
Ok(Self {
config,
state: Arc::new(RwLock::new(federation_state)),
http_client,
})
}
pub fn node_id(&self) -> &str {
&self.config.node_id
}
pub fn peers(&self) -> &[PeerConfig] {
&self.config.peers
}
pub fn add_peer(&mut self, peer: PeerConfig) {
self.config.peers.push(peer);
}
pub fn remove_peer(&mut self, node_id: &str) {
self.config.peers.retain(|p| p.node_id != node_id);
}
pub async fn generate_manifest(
&self,
vault_models: Vec<(String, Vec<crate::version::ModelVersion>)>,
) -> SyncManifest {
let state = self.state.read().await;
let models = vault_models
.into_iter()
.map(
|(name, versions): (String, Vec<crate::version::ModelVersion>)| {
let model_clock = state
.models
.get(&name)
.map(|s| s.clock.clone())
.unwrap_or_default();
let version_entries = versions
.into_iter()
.map(|v| VersionManifestEntry {
version: v.version,
checkpoint_id: crate::federation_transport::federation_checkpoint_id(
&v,
),
created_at: v.timestamp,
checksum: v.checksum_sha256,
size_bytes: v.size_bytes,
parent_id: v.parent_version.map(|p| p.to_string()),
origin_node: self.config.node_id.clone(),
})
.collect();
ModelManifestEntry {
name,
versions: version_entries,
clock: model_clock,
}
},
)
.collect();
SyncManifest {
source_node: self.config.node_id.clone(),
timestamp: Utc::now(),
models,
clock: state.clock.clone(),
}
}
pub fn compute_delta(&self, local: &SyncManifest, remote: &SyncManifest) -> SyncDelta {
let mut to_upload = Vec::new();
let mut to_download = Vec::new();
let mut conflicts = Vec::new();
let remote_models: HashMap<&str, &ModelManifestEntry> =
remote.models.iter().map(|m| (m.name.as_str(), m)).collect();
for local_model in &local.models {
if let Some(remote_model) = remote_models.get(local_model.name.as_str()) {
let local_checkpoints: HashSet<&str> = local_model
.versions
.iter()
.map(|v| v.checkpoint_id.as_str())
.collect();
let remote_checkpoints: HashSet<&str> = remote_model
.versions
.iter()
.map(|v| v.checkpoint_id.as_str())
.collect();
for version in &local_model.versions {
if !remote_checkpoints.contains(version.checkpoint_id.as_str()) {
to_upload.push(SyncItem {
model: local_model.name.clone(),
checkpoint_id: version.checkpoint_id.clone(),
size_bytes: version.size_bytes,
});
}
}
for version in &remote_model.versions {
if !local_checkpoints.contains(version.checkpoint_id.as_str()) {
to_download.push(SyncItem {
model: remote_model.name.clone(),
checkpoint_id: version.checkpoint_id.clone(),
size_bytes: version.size_bytes,
});
}
}
let local_by_version: HashMap<u32, &VersionManifestEntry> = local_model
.versions
.iter()
.map(|v| (v.version, v))
.collect();
let remote_by_version: HashMap<u32, &VersionManifestEntry> = remote_model
.versions
.iter()
.map(|v| (v.version, v))
.collect();
for (version, local_v) in &local_by_version {
if let Some(remote_v) = remote_by_version.get(version) {
if local_v.checkpoint_id != remote_v.checkpoint_id {
conflicts.push(SyncConflict {
model: local_model.name.clone(),
local_version: local_v.checkpoint_id.clone(),
remote_version: remote_v.checkpoint_id.clone(),
remote_node: remote.source_node.clone(),
resolution: None,
});
}
}
}
} else {
for version in &local_model.versions {
to_upload.push(SyncItem {
model: local_model.name.clone(),
checkpoint_id: version.checkpoint_id.clone(),
size_bytes: version.size_bytes,
});
}
}
}
let local_model_names: HashSet<&str> =
local.models.iter().map(|m| m.name.as_str()).collect();
for remote_model in &remote.models {
if !local_model_names.contains(remote_model.name.as_str()) {
for version in &remote_model.versions {
to_download.push(SyncItem {
model: remote_model.name.clone(),
checkpoint_id: version.checkpoint_id.clone(),
size_bytes: version.size_bytes,
});
}
}
}
SyncDelta {
to_upload,
to_download,
conflicts,
}
}
pub async fn sync_with_peer(
&self,
peer: &PeerConfig,
local_manifest: &SyncManifest,
download_fn: impl Fn(&str, &str) -> Result<Vec<u8>>,
upload_fn: impl Fn(&str, &str, &[u8]) -> Result<()>,
) -> Result<SyncResult> {
let start = std::time::Instant::now();
let mut errors = Vec::new();
let remote_manifest = self.fetch_manifest(peer).await?;
let delta = self.compute_delta(local_manifest, &remote_manifest);
let mut resolved_conflicts = Vec::new();
for conflict in delta.conflicts {
let resolution = if self.config.auto_resolve_conflicts {
Some(ConflictResolution::KeepLocal)
} else {
Some(ConflictResolution::Manual)
};
resolved_conflicts.push(SyncConflict {
resolution,
..conflict
});
}
let mut downloaded = 0;
for item in &delta.to_download {
match self
.download_version(peer, &item.model, &item.checkpoint_id)
.await
{
Ok(data) => {
if let Err(e) = upload_fn(&item.model, &item.checkpoint_id, &data) {
errors.push(format!(
"Failed to store {}/{}: {e}",
item.model, item.checkpoint_id
));
} else {
downloaded += 1;
}
}
Err(e) => {
errors.push(format!(
"Failed to download {}/{}: {e}",
item.model, item.checkpoint_id
));
}
}
}
let mut uploaded = 0;
for item in &delta.to_upload {
match download_fn(&item.model, &item.checkpoint_id) {
Ok(data) => {
if let Err(e) = self
.upload_version(peer, &item.model, &item.checkpoint_id, &data)
.await
{
errors.push(format!(
"Failed to upload {}/{}: {e}",
item.model, item.checkpoint_id
));
} else {
uploaded += 1;
}
}
Err(e) => {
errors.push(format!(
"Failed to read {}/{}: {e}",
item.model, item.checkpoint_id
));
}
}
}
{
let mut state = self.state.write().await;
state.clock.increment(&self.config.node_id);
state.clock.merge(&remote_manifest.clock);
}
let result = SyncResult {
peer_id: peer.node_id.clone(),
timestamp: Utc::now(),
duration_ms: start.elapsed().as_millis() as u64,
models_synced: (delta.to_upload.len() + delta.to_download.len()) as u32,
versions_uploaded: uploaded,
versions_downloaded: downloaded,
conflicts: resolved_conflicts,
errors,
};
{
let mut state = self.state.write().await;
state.history.push(result.clone());
if state.history.len() > 1000 {
state.history.remove(0);
}
let _ = self.save_state(&state);
}
Ok(result)
}
pub async fn fetch_peer_manifest(&self, peer: &PeerConfig) -> Result<SyncManifest> {
self.fetch_manifest(peer).await
}
async fn fetch_manifest(&self, peer: &PeerConfig) -> Result<SyncManifest> {
let url = format!("{}/api/v1/federation/manifest", peer.endpoint);
let mut req = self.http_client.get(&url);
if let Some(api_key) = &peer.api_key {
req = req.header("X-API-Key", api_key);
}
let response = req
.send()
.await
.map_err(|e| VaultError::IoError(std::io::Error::other(format!("HTTP error: {e}"))))?;
if !response.status().is_success() {
return Err(VaultError::IoError(std::io::Error::other(format!(
"Peer returned error: {}",
response.status()
))));
}
let manifest = response
.json::<SyncManifest>()
.await
.map_err(|e| VaultError::IoError(std::io::Error::other(format!("JSON error: {e}"))))?;
Ok(manifest)
}
async fn download_version(
&self,
peer: &PeerConfig,
model: &str,
checkpoint_id: &str,
) -> Result<Vec<u8>> {
let url = format!(
"{}/api/v1/federation/models/{}/versions/{}",
peer.endpoint, model, checkpoint_id
);
let mut req = self.http_client.get(&url);
if let Some(api_key) = &peer.api_key {
req = req.header("X-API-Key", api_key);
}
let response = req
.send()
.await
.map_err(|e| VaultError::IoError(std::io::Error::other(format!("HTTP error: {e}"))))?;
if !response.status().is_success() {
return Err(VaultError::IoError(std::io::Error::other(format!(
"Download failed: {}",
response.status()
))));
}
let data = response
.bytes()
.await
.map_err(|e| VaultError::IoError(std::io::Error::other(format!("Read error: {e}"))))?;
Ok(data.to_vec())
}
async fn upload_version(
&self,
peer: &PeerConfig,
model: &str,
checkpoint_id: &str,
data: &[u8],
) -> Result<()> {
let url = format!(
"{}/api/v1/federation/models/{}/versions/{}",
peer.endpoint, model, checkpoint_id
);
let mut req = self.http_client.put(&url).body(data.to_vec());
if let Some(api_key) = &peer.api_key {
req = req.header("X-API-Key", api_key);
}
let response = req
.send()
.await
.map_err(|e| VaultError::IoError(std::io::Error::other(format!("HTTP error: {e}"))))?;
if !response.status().is_success() {
return Err(VaultError::IoError(std::io::Error::other(format!(
"Upload failed: {}",
response.status()
))));
}
Ok(())
}
fn save_state(&self, state: &FederationState) -> Result<()> {
let saved = SavedFederationState {
models: state.models.clone(),
clock: state.clock.clone(),
history: state.history.clone(),
};
let json = serde_json::to_string_pretty(&saved)?;
{
use std::io::Write;
let mut opts = std::fs::OpenOptions::new();
opts.write(true).create(true).truncate(true);
crate::permissions::set_create_mode(&mut opts);
let mut f = opts.open(&state.state_file)?;
f.write_all(json.as_bytes())?;
}
crate::permissions::restrict_file(&state.state_file)?;
Ok(())
}
pub async fn get_history(&self, limit: Option<usize>) -> Vec<SyncResult> {
let state = self.state.read().await;
let history = &state.history;
if let Some(n) = limit {
history.iter().rev().take(n).cloned().collect()
} else {
history.clone()
}
}
pub async fn status(&self) -> FederationStatus {
let state = self.state.read().await;
FederationStatus {
node_id: self.config.node_id.clone(),
node_name: self.config.node_name.clone(),
peer_count: self.config.peers.len(),
model_count: state.models.len(),
last_sync: state.history.last().map(|r| r.timestamp),
clock: state.clock.clone(),
}
}
}
#[derive(Debug, Clone)]
pub struct SyncDelta {
pub to_upload: Vec<SyncItem>,
pub to_download: Vec<SyncItem>,
pub conflicts: Vec<SyncConflict>,
}
#[derive(Debug, Clone)]
pub struct SyncItem {
pub model: String,
pub checkpoint_id: String,
pub size_bytes: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SavedFederationState {
models: HashMap<String, ModelSyncState>,
clock: VectorClock,
history: Vec<SyncResult>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FederationStatus {
pub node_id: NodeId,
pub node_name: String,
pub peer_count: usize,
pub model_count: usize,
pub last_sync: Option<DateTime<Utc>>,
pub clock: VectorClock,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_vector_clock_basic() {
let mut clock1 = VectorClock::new();
let mut clock2 = VectorClock::new();
clock1.increment("node1");
assert_eq!(clock1.timestamps.get("node1"), Some(&1));
clock2.increment("node2");
clock1.merge(&clock2);
assert_eq!(clock1.timestamps.get("node1"), Some(&1));
assert_eq!(clock1.timestamps.get("node2"), Some(&1));
}
#[test]
fn test_vector_clock_comparison() {
let mut clock1 = VectorClock::new();
let mut clock2 = VectorClock::new();
assert_eq!(clock1.compare(&clock2), ClockComparison::Equal);
clock1.increment("node1");
assert_eq!(clock1.compare(&clock2), ClockComparison::After);
assert_eq!(clock2.compare(&clock1), ClockComparison::Before);
clock2.increment("node2");
assert_eq!(clock1.compare(&clock2), ClockComparison::Concurrent);
}
#[test]
fn test_vector_clock_multiple_increments() {
let mut clock = VectorClock::new();
clock.increment("a");
clock.increment("a");
clock.increment("a");
assert_eq!(clock.timestamps.get("a"), Some(&3));
}
#[test]
fn test_vector_clock_merge_takes_max() {
let mut c1 = VectorClock::new();
let mut c2 = VectorClock::new();
c1.increment("a");
c1.increment("a"); c2.increment("a"); c2.increment("b");
c1.merge(&c2);
assert_eq!(c1.timestamps.get("a"), Some(&2)); assert_eq!(c1.timestamps.get("b"), Some(&1)); }
#[test]
fn test_vector_clock_equal_after_merge() {
let mut c1 = VectorClock::new();
c1.increment("a");
let c2 = c1.clone();
assert_eq!(c1.compare(&c2), ClockComparison::Equal);
}
#[test]
fn test_vector_clock_serialization() {
let mut clock = VectorClock::new();
clock.increment("node1");
let json = serde_json::to_string(&clock).unwrap();
let deserialized: VectorClock = serde_json::from_str(&json).unwrap();
assert_eq!(clock, deserialized);
}
#[test]
fn test_clock_comparison_debug() {
assert_eq!(format!("{:?}", ClockComparison::Equal), "Equal");
assert_eq!(format!("{:?}", ClockComparison::Before), "Before");
assert_eq!(format!("{:?}", ClockComparison::After), "After");
assert_eq!(format!("{:?}", ClockComparison::Concurrent), "Concurrent");
}
#[test]
fn test_federation_config_default() {
let config = FederationConfig::default();
assert!(!config.node_id.is_empty());
assert_eq!(config.sync_interval_secs, 300);
assert!(config.auto_resolve_conflicts);
assert_eq!(config.max_concurrent_syncs, 4);
assert!(config.peers.is_empty());
assert!(!config.node_name.is_empty());
}
#[test]
fn test_federation_config_serialization() {
let config = FederationConfig::default();
let json = serde_json::to_string(&config).unwrap();
let deserialized: FederationConfig = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.sync_interval_secs, 300);
assert!(deserialized.auto_resolve_conflicts);
}
#[test]
fn test_peer_config_zeroize_on_drop() {
let peer = PeerConfig {
node_id: "peer1".to_string(),
name: "Test Peer".to_string(),
endpoint: "https://example.com".to_string(),
api_key: Some("secret-key-12345".to_string()),
enabled: true,
};
drop(peer);
}
#[test]
fn test_peer_config_without_api_key() {
let peer = PeerConfig {
node_id: "peer1".to_string(),
name: "Test Peer".to_string(),
endpoint: "https://example.com".to_string(),
api_key: None,
enabled: false,
};
assert!(!peer.enabled);
drop(peer);
}
#[test]
fn test_peer_config_serialization() {
let peer = PeerConfig {
node_id: "p1".to_string(),
name: "Peer 1".to_string(),
endpoint: "https://peer1.example.com".to_string(),
api_key: Some("key".to_string()),
enabled: true,
};
let json = serde_json::to_string(&peer).unwrap();
let deserialized: PeerConfig = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.node_id, "p1");
assert_eq!(deserialized.api_key.as_deref(), Some("key"));
}
#[test]
fn test_model_sync_state_default_fields() {
let state = ModelSyncState {
name: "test-model".to_string(),
clock: VectorClock::new(),
last_sync: HashMap::new(),
known_versions: HashSet::new(),
pending_upload: HashSet::new(),
pending_download: HashSet::new(),
};
assert_eq!(state.name, "test-model");
assert!(state.known_versions.is_empty());
}
#[test]
fn test_sync_manifest_serialization() {
let manifest = SyncManifest {
source_node: "node-a".to_string(),
timestamp: Utc::now(),
models: vec![],
clock: VectorClock::new(),
};
let json = serde_json::to_string(&manifest).unwrap();
let deserialized: SyncManifest = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.source_node, "node-a");
}
#[test]
fn test_sync_result_serialization() {
let result = SyncResult {
peer_id: "peer-1".to_string(),
timestamp: Utc::now(),
duration_ms: 1500,
models_synced: 3,
versions_uploaded: 2,
versions_downloaded: 1,
conflicts: vec![],
errors: vec![],
};
let json = serde_json::to_string(&result).unwrap();
assert!(json.contains("peer-1"));
}
#[test]
fn test_sync_conflict_serialization() {
let conflict = SyncConflict {
model: "llama".to_string(),
local_version: "abc".to_string(),
remote_version: "def".to_string(),
remote_node: "node-b".to_string(),
resolution: Some(ConflictResolution::KeepLocal),
};
let json = serde_json::to_string(&conflict).unwrap();
assert!(json.contains("KeepLocal"));
}
#[test]
fn test_conflict_resolution_variants() {
let keep = ConflictResolution::KeepLocal;
let remote = ConflictResolution::UseRemote;
let branch = ConflictResolution::Branch {
local_name: "local-v2".to_string(),
remote_name: "remote-v2".to_string(),
};
let manual = ConflictResolution::Manual;
for r in [keep, remote, manual] {
let json = serde_json::to_string(&r).unwrap();
let _: ConflictResolution = serde_json::from_str(&json).unwrap();
}
let json = serde_json::to_string(&branch).unwrap();
assert!(json.contains("local-v2"));
}
#[test]
fn test_version_manifest_entry() {
let entry = VersionManifestEntry {
version: 3,
checkpoint_id: "cp-xyz".to_string(),
created_at: Utc::now(),
checksum: "sha256-abc".to_string(),
size_bytes: 1024,
parent_id: Some("cp-prev".to_string()),
origin_node: "node-a".to_string(),
};
let json = serde_json::to_string(&entry).unwrap();
let deserialized: VersionManifestEntry = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.version, 3);
assert_eq!(deserialized.checkpoint_id, "cp-xyz");
}
#[test]
fn test_federation_status_serialization() {
let status = FederationStatus {
node_id: "n1".to_string(),
node_name: "Node One".to_string(),
peer_count: 2,
model_count: 5,
last_sync: Some(Utc::now()),
clock: VectorClock::new(),
};
let json = serde_json::to_string(&status).unwrap();
let d: FederationStatus = serde_json::from_str(&json).unwrap();
assert_eq!(d.peer_count, 2);
}
#[tokio::test]
async fn test_federation_manager_new() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
assert!(!mgr.node_id().is_empty());
assert!(mgr.peers().is_empty());
}
#[tokio::test]
async fn test_federation_manager_add_remove_peer() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let mut mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let peer = PeerConfig {
node_id: "peer-1".to_string(),
name: "Peer 1".to_string(),
endpoint: "https://peer1.example.com".to_string(),
api_key: None,
enabled: true,
};
mgr.add_peer(peer);
assert_eq!(mgr.peers().len(), 1);
mgr.remove_peer("peer-1");
assert!(mgr.peers().is_empty());
}
#[tokio::test]
async fn test_federation_manager_status() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig {
node_name: "test-node".to_string(),
..Default::default()
};
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let status = mgr.status().await;
assert_eq!(status.node_name, "test-node");
assert_eq!(status.peer_count, 0);
assert_eq!(status.model_count, 0);
assert!(status.last_sync.is_none());
}
#[tokio::test]
async fn test_federation_manager_get_history_empty() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let history = mgr.get_history(None).await;
assert!(history.is_empty());
let limited = mgr.get_history(Some(5)).await;
assert!(limited.is_empty());
}
#[tokio::test]
async fn test_federation_manager_generate_manifest_empty() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let manifest = mgr.generate_manifest(vec![]).await;
assert!(manifest.models.is_empty());
assert_eq!(manifest.source_node, mgr.node_id());
}
#[tokio::test]
async fn test_federation_manager_generate_manifest_with_models() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let versions = vec![crate::version::ModelVersion {
version: 1,
checkpoint_id: "cp-1".to_string(),
timestamp: Utc::now(),
checksum_sha256: "abc123".to_string(),
size_bytes: 1024,
compressed_size_bytes: 800,
parent_version: None,
format: "safetensors".to_string(),
metadata: HashMap::new(),
file_path: "models/cp-1.enc".to_string(),
}];
let manifest = mgr
.generate_manifest(vec![("my-model".to_string(), versions)])
.await;
assert_eq!(manifest.models.len(), 1);
assert_eq!(manifest.models[0].name, "my-model");
assert_eq!(manifest.models[0].versions.len(), 1);
}
#[test]
fn test_compute_delta_empty_manifests() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let local = SyncManifest {
source_node: "a".to_string(),
timestamp: Utc::now(),
models: vec![],
clock: VectorClock::new(),
};
let remote = SyncManifest {
source_node: "b".to_string(),
timestamp: Utc::now(),
models: vec![],
clock: VectorClock::new(),
};
let delta = mgr.compute_delta(&local, &remote);
assert!(delta.to_upload.is_empty());
assert!(delta.to_download.is_empty());
assert!(delta.conflicts.is_empty());
}
#[test]
fn test_compute_delta_local_only_model() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let local = SyncManifest {
source_node: "a".to_string(),
timestamp: Utc::now(),
models: vec![ModelManifestEntry {
name: "model-x".to_string(),
versions: vec![VersionManifestEntry {
version: 1,
checkpoint_id: "cp-1".to_string(),
created_at: Utc::now(),
checksum: "sha".to_string(),
size_bytes: 100,
parent_id: None,
origin_node: "a".to_string(),
}],
clock: VectorClock::new(),
}],
clock: VectorClock::new(),
};
let remote = SyncManifest {
source_node: "b".to_string(),
timestamp: Utc::now(),
models: vec![],
clock: VectorClock::new(),
};
let delta = mgr.compute_delta(&local, &remote);
assert_eq!(delta.to_upload.len(), 1);
assert_eq!(delta.to_upload[0].model, "model-x");
assert!(delta.to_download.is_empty());
}
#[test]
fn test_compute_delta_remote_only_model() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let local = SyncManifest {
source_node: "a".to_string(),
timestamp: Utc::now(),
models: vec![],
clock: VectorClock::new(),
};
let remote = SyncManifest {
source_node: "b".to_string(),
timestamp: Utc::now(),
models: vec![ModelManifestEntry {
name: "model-y".to_string(),
versions: vec![VersionManifestEntry {
version: 1,
checkpoint_id: "cp-2".to_string(),
created_at: Utc::now(),
checksum: "sha2".to_string(),
size_bytes: 200,
parent_id: None,
origin_node: "b".to_string(),
}],
clock: VectorClock::new(),
}],
clock: VectorClock::new(),
};
let delta = mgr.compute_delta(&local, &remote);
assert!(delta.to_upload.is_empty());
assert_eq!(delta.to_download.len(), 1);
assert_eq!(delta.to_download[0].model, "model-y");
}
#[test]
fn test_compute_delta_conflict() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let local = SyncManifest {
source_node: "a".to_string(),
timestamp: Utc::now(),
models: vec![ModelManifestEntry {
name: "model-z".to_string(),
versions: vec![VersionManifestEntry {
version: 1,
checkpoint_id: "cp-local".to_string(),
created_at: Utc::now(),
checksum: "sha-l".to_string(),
size_bytes: 100,
parent_id: None,
origin_node: "a".to_string(),
}],
clock: VectorClock::new(),
}],
clock: VectorClock::new(),
};
let remote = SyncManifest {
source_node: "b".to_string(),
timestamp: Utc::now(),
models: vec![ModelManifestEntry {
name: "model-z".to_string(),
versions: vec![VersionManifestEntry {
version: 1,
checkpoint_id: "cp-remote".to_string(),
created_at: Utc::now(),
checksum: "sha-r".to_string(),
size_bytes: 100,
parent_id: None,
origin_node: "b".to_string(),
}],
clock: VectorClock::new(),
}],
clock: VectorClock::new(),
};
let delta = mgr.compute_delta(&local, &remote);
assert_eq!(delta.conflicts.len(), 1);
assert_eq!(delta.conflicts[0].model, "model-z");
}
#[test]
fn test_compute_delta_shared_versions() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let entry = VersionManifestEntry {
version: 1,
checkpoint_id: "cp-same".to_string(),
created_at: Utc::now(),
checksum: "sha".to_string(),
size_bytes: 100,
parent_id: None,
origin_node: "a".to_string(),
};
let local = SyncManifest {
source_node: "a".to_string(),
timestamp: Utc::now(),
models: vec![ModelManifestEntry {
name: "m".to_string(),
versions: vec![entry.clone()],
clock: VectorClock::new(),
}],
clock: VectorClock::new(),
};
let remote = SyncManifest {
source_node: "b".to_string(),
timestamp: Utc::now(),
models: vec![ModelManifestEntry {
name: "m".to_string(),
versions: vec![entry],
clock: VectorClock::new(),
}],
clock: VectorClock::new(),
};
let delta = mgr.compute_delta(&local, &remote);
assert!(delta.to_upload.is_empty());
assert!(delta.to_download.is_empty());
assert!(delta.conflicts.is_empty());
}
#[tokio::test]
async fn test_federation_manager_load_existing_state() {
let dir = tempfile::tempdir().unwrap();
let state_file = dir.path().join("federation_state.json");
let saved = SavedFederationState {
models: HashMap::new(),
clock: {
let mut c = VectorClock::new();
c.increment("existing");
c
},
history: vec![],
};
std::fs::write(&state_file, serde_json::to_string(&saved).unwrap()).unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let status = mgr.status().await;
assert_eq!(status.clock.timestamps.get("existing"), Some(&1));
}
#[test]
fn test_sync_item_debug() {
let item = SyncItem {
model: "m".to_string(),
checkpoint_id: "cp".to_string(),
size_bytes: 42,
};
let dbg = format!("{:?}", item);
assert!(dbg.contains("42"));
}
#[test]
fn test_model_manifest_entry_serialization() {
let entry = ModelManifestEntry {
name: "llm".to_string(),
versions: vec![],
clock: VectorClock::new(),
};
let json = serde_json::to_string(&entry).unwrap();
let d: ModelManifestEntry = serde_json::from_str(&json).unwrap();
assert_eq!(d.name, "llm");
}
#[tokio::test]
async fn test_federation_manager_save_and_reload_state() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let node_id = config.node_id.clone();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
{
let mut state = mgr.state.write().await;
state.clock.increment(&node_id);
state.models.insert(
"saved-model".to_string(),
ModelSyncState {
name: "saved-model".to_string(),
clock: VectorClock::new(),
last_sync: HashMap::new(),
known_versions: HashSet::new(),
pending_upload: HashSet::new(),
pending_download: HashSet::new(),
},
);
mgr.save_state(&state).unwrap();
}
let config2 = FederationConfig::default();
let mgr2 = FederationManager::new(config2, dir.path().to_path_buf()).unwrap();
let status2 = mgr2.status().await;
assert_eq!(status2.model_count, 1);
assert_eq!(status2.clock.timestamps.get(&node_id), Some(&1));
}
#[test]
fn test_compute_delta_mixed_versions() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let shared = VersionManifestEntry {
version: 1,
checkpoint_id: "cp-shared".to_string(),
created_at: Utc::now(),
checksum: "sha".to_string(),
size_bytes: 100,
parent_id: None,
origin_node: "a".to_string(),
};
let local_only = VersionManifestEntry {
version: 2,
checkpoint_id: "cp-local-v2".to_string(),
created_at: Utc::now(),
checksum: "sha2".to_string(),
size_bytes: 200,
parent_id: Some("cp-shared".to_string()),
origin_node: "a".to_string(),
};
let remote_only = VersionManifestEntry {
version: 3,
checkpoint_id: "cp-remote-v3".to_string(),
created_at: Utc::now(),
checksum: "sha3".to_string(),
size_bytes: 300,
parent_id: Some("cp-shared".to_string()),
origin_node: "b".to_string(),
};
let local = SyncManifest {
source_node: "a".to_string(),
timestamp: Utc::now(),
models: vec![ModelManifestEntry {
name: "m".to_string(),
versions: vec![shared.clone(), local_only],
clock: VectorClock::new(),
}],
clock: VectorClock::new(),
};
let remote = SyncManifest {
source_node: "b".to_string(),
timestamp: Utc::now(),
models: vec![ModelManifestEntry {
name: "m".to_string(),
versions: vec![shared, remote_only],
clock: VectorClock::new(),
}],
clock: VectorClock::new(),
};
let delta = mgr.compute_delta(&local, &remote);
assert_eq!(delta.to_upload.len(), 1);
assert_eq!(delta.to_upload[0].checkpoint_id, "cp-local-v2");
assert_eq!(delta.to_download.len(), 1);
assert_eq!(delta.to_download[0].checkpoint_id, "cp-remote-v3");
assert!(delta.conflicts.is_empty()); }
#[tokio::test]
async fn test_federation_manager_generate_manifest_multi_version() {
let dir = tempfile::tempdir().unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let versions = vec![
crate::version::ModelVersion {
version: 1,
checkpoint_id: "cp-1".to_string(),
timestamp: Utc::now(),
checksum_sha256: "abc".to_string(),
size_bytes: 512,
compressed_size_bytes: 400,
parent_version: None,
format: "safetensors".to_string(),
metadata: HashMap::new(),
file_path: "models/cp-1.enc".to_string(),
},
crate::version::ModelVersion {
version: 2,
checkpoint_id: "cp-2".to_string(),
timestamp: Utc::now(),
checksum_sha256: "def".to_string(),
size_bytes: 1024,
compressed_size_bytes: 900,
parent_version: Some(1),
format: "gguf".to_string(),
metadata: HashMap::new(),
file_path: "models/cp-2.enc".to_string(),
},
];
let manifest = mgr
.generate_manifest(vec![("multi-v".to_string(), versions)])
.await;
assert_eq!(manifest.models.len(), 1);
assert_eq!(manifest.models[0].versions.len(), 2);
assert_eq!(manifest.models[0].versions[0].version, 1);
assert_eq!(manifest.models[0].versions[1].version, 2);
assert_eq!(
manifest.models[0].versions[1].parent_id.as_deref(),
Some("1")
);
}
#[test]
fn test_saved_federation_state_roundtrip() {
let mut clock = VectorClock::new();
clock.increment("n1");
clock.increment("n2");
let saved = SavedFederationState {
models: {
let mut m = HashMap::new();
m.insert(
"model-a".to_string(),
ModelSyncState {
name: "model-a".to_string(),
clock: clock.clone(),
last_sync: HashMap::new(),
known_versions: {
let mut s = HashSet::new();
s.insert("cp-1".to_string());
s
},
pending_upload: HashSet::new(),
pending_download: HashSet::new(),
},
);
m
},
clock,
history: vec![SyncResult {
peer_id: "p1".to_string(),
timestamp: Utc::now(),
duration_ms: 500,
models_synced: 1,
versions_uploaded: 0,
versions_downloaded: 1,
conflicts: vec![],
errors: vec![],
}],
};
let json = serde_json::to_string(&saved).unwrap();
let loaded: SavedFederationState = serde_json::from_str(&json).unwrap();
assert_eq!(loaded.models.len(), 1);
assert!(loaded.models.contains_key("model-a"));
assert_eq!(loaded.history.len(), 1);
assert_eq!(loaded.clock.timestamps.get("n1"), Some(&1));
}
#[test]
fn test_sync_delta_debug() {
let delta = SyncDelta {
to_upload: vec![],
to_download: vec![],
conflicts: vec![],
};
let dbg = format!("{:?}", delta);
assert!(dbg.contains("SyncDelta"));
}
#[test]
fn test_model_sync_state_with_pending() {
let mut state = ModelSyncState {
name: "pending-model".to_string(),
clock: VectorClock::new(),
last_sync: HashMap::new(),
known_versions: HashSet::new(),
pending_upload: HashSet::new(),
pending_download: HashSet::new(),
};
state.pending_upload.insert("cp-up-1".to_string());
state
.pending_download
.insert(("node-b".to_string(), "cp-down-1".to_string()));
state.known_versions.insert("cp-1".to_string());
state.last_sync.insert("node-b".to_string(), Utc::now());
let json = serde_json::to_string(&state).unwrap();
let d: ModelSyncState = serde_json::from_str(&json).unwrap();
assert_eq!(d.pending_upload.len(), 1);
assert_eq!(d.pending_download.len(), 1);
assert_eq!(d.known_versions.len(), 1);
assert_eq!(d.last_sync.len(), 1);
}
#[tokio::test]
async fn test_federation_manager_history_with_limit() {
let dir = tempfile::tempdir().unwrap();
let state_file = dir.path().join("federation_state.json");
let history: Vec<SyncResult> = (0..5)
.map(|i| SyncResult {
peer_id: format!("peer-{}", i),
timestamp: Utc::now(),
duration_ms: 100 * (i as u64 + 1),
models_synced: 1,
versions_uploaded: 0,
versions_downloaded: 1,
conflicts: vec![],
errors: vec![],
})
.collect();
let saved = SavedFederationState {
models: HashMap::new(),
clock: VectorClock::new(),
history,
};
std::fs::write(&state_file, serde_json::to_string(&saved).unwrap()).unwrap();
let config = FederationConfig::default();
let mgr = FederationManager::new(config, dir.path().to_path_buf()).unwrap();
let all = mgr.get_history(None).await;
assert_eq!(all.len(), 5);
let limited = mgr.get_history(Some(3)).await;
assert_eq!(limited.len(), 3);
assert_eq!(limited[0].peer_id, "peer-4");
}
#[test]
fn test_conflict_resolution_branch_serialization() {
let res = ConflictResolution::Branch {
local_name: "main-v2".to_string(),
remote_name: "fork-v2".to_string(),
};
let json = serde_json::to_string(&res).unwrap();
let d: ConflictResolution = serde_json::from_str(&json).unwrap();
match d {
ConflictResolution::Branch {
local_name,
remote_name,
} => {
assert_eq!(local_name, "main-v2");
assert_eq!(remote_name, "fork-v2");
}
_ => panic!("Expected Branch variant"),
}
}
#[test]
fn test_federation_status_without_last_sync() {
let status = FederationStatus {
node_id: "n1".to_string(),
node_name: "node1".to_string(),
peer_count: 0,
model_count: 0,
last_sync: None,
clock: VectorClock::new(),
};
let json = serde_json::to_string(&status).unwrap();
let d: FederationStatus = serde_json::from_str(&json).unwrap();
assert!(d.last_sync.is_none());
}
}