mod common;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::{Duration, Instant};
use common::{NODE_A, NODE_B};
use sapphire_backend::protocol as proto;
use sapphire_bridge_api::EmbedModelInfo;
use sapphire_framework_bridge::{EmbedProvider, LoopbackNetwork};
struct CountingProvider(AtomicUsize);
#[async_trait::async_trait]
impl EmbedProvider for CountingProvider {
fn info(&self) -> Option<EmbedModelInfo> {
Some(EmbedModelInfo {
model: "fake-3d".into(),
dimension: 3,
template_version: 1,
revision: None,
max_tokens: None,
images: false,
})
}
fn loaded(&self) -> bool {
true
}
async fn embed(&self, texts: Vec<String>) -> Result<Vec<Vec<f32>>, String> {
self.0.fetch_add(texts.len(), Ordering::SeqCst);
Ok(texts
.iter()
.map(|t| vec![t.len() as f32, 1.0, 0.5])
.collect())
}
}
fn counting() -> Arc<CountingProvider> {
Arc::new(CountingProvider(AtomicUsize::new(0)))
}
async fn write(host: &common::Host, rel: &str, content: &str) {
let _: proto::Ack = host
.client
.call(
proto::WRITE_FILE,
proto::ContentParams {
path: PathBuf::from(rel),
content: content.into(),
},
)
.await
.unwrap();
}
fn vector_files(host: &common::Host) -> Vec<String> {
let embedded = host
.ws
.join(format!(".{}", common::ctx().app_name))
.join("embedded");
let mut out = Vec::new();
for profile in std::fs::read_dir(&embedded).into_iter().flatten().flatten() {
for file in std::fs::read_dir(profile.path())
.into_iter()
.flatten()
.flatten()
{
let name = file.file_name().to_string_lossy().into_owned();
if name.ends_with(".vec") {
out.push(format!("{}/{name}", profile.file_name().to_string_lossy()));
}
}
}
out.sort();
out
}
fn hash_of(content: &str) -> String {
sapphire_workspace::vectors::content_hash(content.as_bytes())
}
async fn await_condition(what: &str, mut ok: impl FnMut() -> bool) {
let deadline = Instant::now() + Duration::from_secs(30);
while !ok() {
assert!(Instant::now() < deadline, "{what} never happened");
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
fn has_vector(host: &common::Host, content: &str) -> bool {
let want = format!("{}.vec", hash_of(content));
vector_files(host).iter().any(|f| f.ends_with(&want))
}
fn file_at(root: &Path, rel: &str) -> Option<String> {
std::fs::read_to_string(root.join(rel)).ok()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn the_author_embeds_and_the_other_device_reads_the_vector() {
let net = LoopbackNetwork::new();
let (pa, pb) = (counting(), counting());
let a = common::start_host_embedding(&net, NODE_A, "a", 0, pa.clone()).await;
let b = common::start_host_embedding(&net, NODE_B, "b", 0, pb.clone()).await;
common::introduce_all(&[&a, &b]);
common::enable_sync(&a).await;
common::enable_sync(&b).await;
let content = "a note written on A";
write(&a, "note.md", content).await;
await_condition("A's vector reaching B", || {
file_at(&b.ws, "note.md").is_some() && has_vector(&b, content)
})
.await;
assert!(has_vector(&a, content), "A wrote the vector it computed");
tokio::time::sleep(Duration::from_secs(1)).await;
assert_eq!(pa.0.load(Ordering::SeqCst), 1, "the author embedded once");
assert_eq!(
pb.0.load(Ordering::SeqCst),
0,
"the other device read the vector"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn the_primary_embeds_what_a_device_without_embedding_wrote() {
let net = LoopbackNetwork::new();
let pb = counting();
let a = common::start_host_with_priority(&net, NODE_A, "a", 0).await;
let b = common::start_host_embedding(&net, NODE_B, "b", 1, pb.clone()).await;
common::introduce_all(&[&a, &b]);
common::enable_sync(&a).await;
common::enable_sync(&b).await;
common::await_primary(&[&a, &b], b.device_id().await).await;
let content = "written where nothing embeds";
write(&a, "plain.md", content).await;
await_condition("the primary's vector reaching A", || {
has_vector(&a, content)
})
.await;
assert_eq!(pb.0.load(Ordering::SeqCst), 1);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn the_primary_waits_out_the_grace_period_then_backfills_on_its_own() {
let net = LoopbackNetwork::new();
let pb = counting();
let a = common::start_host_with_priority(&net, NODE_A, "a", 0).await;
let b = common::start_host_embedding(&net, NODE_B, "b", 1, pb.clone()).await;
let runtime_b = b.runtime().unwrap();
runtime_b.set_backfill_timing(sapphire_framework_server::BackfillTiming {
grace: Duration::from_secs(3600),
interval: Duration::from_millis(300),
});
common::introduce_all(&[&a, &b]);
common::enable_sync(&a).await;
common::enable_sync(&b).await;
common::await_primary(&[&a, &b], b.device_id().await).await;
let content = "fresh from a device that does not embed";
write(&a, "fresh.md", content).await;
await_condition("the file reaching B", || {
file_at(&b.ws, "fresh.md").is_some()
})
.await;
tokio::time::sleep(Duration::from_secs(2)).await;
assert_eq!(
pb.0.load(Ordering::SeqCst),
0,
"the author's version is too fresh"
);
let status: proto::SyncStatusResult = b
.client
.call(proto::SYNC_STATUS, serde_json::json!({}))
.await
.unwrap();
let progress = status.embedding.expect("B embeds");
assert!(progress.pending >= 1, "{progress:?}");
runtime_b.set_backfill_timing(sapphire_framework_server::BackfillTiming {
grace: Duration::ZERO,
interval: Duration::from_millis(300),
});
await_condition("the backfilled vector reaching A", || {
has_vector(&a, content)
})
.await;
assert_eq!(pb.0.load(Ordering::SeqCst), 1);
}