#![allow(dead_code)]
use std::collections::HashSet;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
use grain_id::GrainId;
use sapphire_backend::protocol as proto;
use sapphire_bridge_client::BridgeClient;
use sapphire_framework_bridge::{
Bridge, BridgeDir, HelloTiming, LoopbackNetwork, NetConfig, Workgroup, serve_restarting,
};
use sapphire_framework_server::{AppServer, BridgeConnector, Settings, SyncRuntime};
use sapphire_ipc::ManagedBy;
use sapphire_ipc::{Client, ClientInfo, Endpoint, connect_or_absent};
use sapphire_workspace::AppContext;
pub const NODE_A: &str = "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90";
pub const NODE_B: &str = "b1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90";
pub const NODE_S: &str = "c1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90";
pub const NODE_C: &str = "d1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90";
pub const VERSION: &str = "0.0.0";
pub static CTX: AppContext = AppContext::new("sapphire-converge");
pub fn ctx() -> &'static AppContext {
static DIR: std::sync::OnceLock<tempfile::TempDir> = std::sync::OnceLock::new();
let dir = DIR.get_or_init(|| tempfile::tempdir().expect("a scratch tree"));
CTX.set_cache_dir(dir.path().join("cache"));
CTX.set_data_dir(dir.path().join("data"));
CTX.set_config_dir(dir.path().join("config"));
&CTX
}
pub struct Host {
tmp: Option<tempfile::TempDir>,
node_id: String,
device_name: String,
pub ws: PathBuf,
pub bridge_dir: BridgeDir,
pub workgroup_id: GrainId,
runtime_dir: PathBuf,
pub client: Client,
net: LoopbackNetwork,
bridge_client: Arc<BridgeClient>,
runtime: Option<Arc<SyncRuntime>>,
server: Option<tokio::task::JoinHandle<sapphire_framework_server::Result<()>>>,
bridge: Option<tokio::runtime::Runtime>,
embed: Option<Arc<dyn sapphire_framework_bridge::EmbedProvider>>,
restarting: bool,
}
pub async fn start_host(net: &LoopbackNetwork, node_id: &str, device_name: &str) -> Host {
start_host_with_priority(net, node_id, device_name, 0).await
}
pub async fn start_host_with_priority(
net: &LoopbackNetwork,
node_id: &str,
device_name: &str,
priority: u8,
) -> Host {
build(
net,
node_id,
device_name,
priority,
None,
tempfile::tempdir().expect("a host tree"),
false,
false,
)
.await
}
pub async fn start_host_embedding(
net: &LoopbackNetwork,
node_id: &str,
device_name: &str,
priority: u8,
provider: Arc<dyn sapphire_framework_bridge::EmbedProvider>,
) -> Host {
build(
net,
node_id,
device_name,
priority,
Some(provider),
tempfile::tempdir().expect("a host tree"),
false,
false,
)
.await
}
pub async fn start_host_restarting(
net: &LoopbackNetwork,
node_id: &str,
device_name: &str,
) -> Host {
build(
net,
node_id,
device_name,
0,
None,
tempfile::tempdir().expect("a host tree"),
false,
true,
)
.await
}
pub async fn star_hosts(net: &LoopbackNetwork, spec: &[(&str, &str, u8)]) -> Vec<Host> {
let mut hosts = Vec::new();
for (node, name, priority) in spec {
hosts.push(start_host_with_priority(net, node, name, *priority).await);
}
let refs: Vec<&Host> = hosts.iter().collect();
introduce_all(&refs);
for host in &hosts {
enable_sync(host).await;
}
hosts
}
#[allow(clippy::too_many_arguments)] async fn build(
net: &LoopbackNetwork,
node_id: &str,
device_name: &str,
priority: u8,
embed: Option<Arc<dyn sapphire_framework_bridge::EmbedProvider>>,
tmp: tempfile::TempDir,
server_first: bool,
restarting: bool,
) -> Host {
let ctx = ctx();
let runtime_dir = tmp.path().join("run");
std::fs::create_dir_all(&runtime_dir).unwrap();
let bridge_dir = BridgeDir::at(tmp.path().join("bridge")).unwrap();
let workgroup = match Workgroup::open(&bridge_dir).unwrap() {
Some(existing) => existing,
None => Workgroup::create(&bridge_dir, "converge", device_name, node_id).unwrap(),
};
workgroup.set_priority(device_name, priority).unwrap();
let root = tmp.path().join("ws");
std::fs::create_dir_all(root.join(format!(".{}", ctx.app_name))).unwrap();
let root = root.canonicalize().unwrap();
let control = Endpoint::in_dir("bridge", runtime_dir.clone());
let mut bridge_runtime = None;
if !server_first {
bridge_runtime =
Some(start_bridge(net, node_id, &bridge_dir, &runtime_dir, embed.clone()).await);
}
let endpoint = Endpoint::in_dir(ctx.app_name, runtime_dir.clone());
let selection = tmp.path().join("workspace.toml");
let (runtime, mut server_task) = if restarting {
let (endpoint, control) = (endpoint.clone(), control.clone());
let build = move |_: &Settings| {
AppServer::new(ctx, VERSION)
.endpoint(endpoint.clone())
.selection_file(selection.clone())
.sync(sync_runtime(ctx, &control))
};
(
None,
tokio::spawn(sapphire_framework_server::serve_restarting(ctx, (), build)),
)
} else {
let runtime = sync_runtime(ctx, &control);
let server = AppServer::new(ctx, VERSION)
.endpoint(endpoint.clone())
.selection_file(selection)
.sync(Arc::clone(&runtime));
(
Some(runtime),
tokio::spawn(async move { server.run().await }),
)
};
wait_until_listening(&endpoint, "the app server", &mut server_task).await;
if server_first {
bridge_runtime =
Some(start_bridge(net, node_id, &bridge_dir, &runtime_dir, embed.clone()).await);
}
let (control_client, _) = connect_or_absent(
&control,
"bridge",
ClientInfo {
api: sapphire_bridge_api::API_VERSION,
..client_info("test")
},
)
.await
.unwrap()
.expect("the bridge is listening");
let bridge_client = Arc::new(BridgeClient::from_client(
Arc::new(control_client),
runtime_dir.clone(),
));
let (client, _) = connect_or_absent(&endpoint, ctx.app_name, client_info("cli"))
.await
.unwrap()
.expect("the app server is listening");
let _: proto::CurrentWorkspace = client
.call(
proto::WORKSPACE_SELECT,
proto::WorkspaceSelectParams { dir: root.clone() },
)
.await
.expect("selecting the host's workspace");
Host {
tmp: Some(tmp),
node_id: node_id.to_owned(),
device_name: device_name.to_owned(),
ws: root,
bridge_dir,
workgroup_id: workgroup.id,
runtime_dir,
client,
net: net.clone(),
bridge_client,
runtime,
server: Some(server_task),
bridge: bridge_runtime,
embed,
restarting,
}
}
fn sync_runtime(ctx: &'static AppContext, control: &Endpoint) -> Arc<SyncRuntime> {
let runtime = Arc::new(
SyncRuntime::connecting(
ctx,
BridgeConnector::at(control.clone(), "test", VERSION),
std::env::current_exe().unwrap_or_else(|_| PathBuf::from("sapphire")),
ManagedBy::Service,
)
.with_bridge_endpoint(control.clone()),
);
runtime.set_backfill_timing(sapphire_framework_server::BackfillTiming {
grace: Duration::ZERO,
interval: Duration::from_secs(1),
});
runtime
}
impl Host {
pub fn node_id(&self) -> &str {
&self.node_id
}
pub fn runtime(&self) -> Option<Arc<SyncRuntime>> {
self.runtime.clone()
}
pub fn bridge(&self) -> &BridgeClient {
&self.bridge_client
}
pub async fn device_id(&self) -> GrainId {
Workgroup::open(&self.bridge_dir)
.unwrap()
.unwrap()
.this_device(&self.node_id)
.unwrap()
.id
}
pub async fn workspace_id(&self) -> GrainId {
std::fs::read_to_string(sync_id_path(self))
.expect("the workspace has a sync-id")
.trim()
.parse()
.expect("the sync-id is a grain-id")
}
pub async fn drop_connections(&self) {
if let Some(runtime) = self.runtime() {
runtime.drop_connections().await;
}
}
pub async fn stop(&mut self) {
let _: std::result::Result<proto::Ack, _> = self
.client
.call(sapphire_ipc::SHUTDOWN_METHOD, serde_json::json!({}))
.await;
if let Some(server) = self.server.take() {
let _ = server.await;
}
self.runtime = None;
self.stop_bridge().await;
}
pub async fn restart(self, net: &LoopbackNetwork) -> Host {
self.rebuild(net, false).await
}
pub async fn restart_server_first(self, net: &LoopbackNetwork) -> Host {
self.rebuild(net, true).await
}
async fn rebuild(mut self, net: &LoopbackNetwork, server_first: bool) -> Host {
self.stop_bridge().await;
let tmp = self
.tmp
.take()
.expect("a host keeps its directories until it restarts");
let priority = Workgroup::open(&self.bridge_dir)
.unwrap()
.expect("a restarting host has a workgroup")
.this_device(&self.node_id)
.unwrap()
.priority;
let embed = self.embed.take();
build(
net,
&self.node_id,
&self.device_name,
priority,
embed,
tmp,
server_first,
self.restarting,
)
.await
}
pub async fn restart_in_place(&mut self) {
assert!(self.restarting, "only a restarting host restarts in place");
let _: proto::Ack = self
.client
.call(proto::SERVER_RESTART, serde_json::json!({}))
.await
.expect("server.restart answers");
let deadline = Instant::now() + Duration::from_secs(10);
loop {
let answer: std::result::Result<serde_json::Value, _> = self
.client
.call(proto::SERVER_INFO, serde_json::json!({}))
.await;
if answer.is_err() {
break;
}
assert!(Instant::now() < deadline, "the old app server kept serving");
tokio::time::sleep(Duration::from_millis(20)).await;
}
let endpoint = Endpoint::in_dir(ctx().app_name, self.runtime_dir.clone());
let (client, _) = connect_or_absent(&endpoint, ctx().app_name, client_info("cli"))
.await
.unwrap()
.expect("the app server is listening again");
self.client = client;
}
pub async fn restart_bridge(&mut self) {
assert!(self.bridge.is_none(), "the bridge is still running");
let runtime = start_bridge(
&self.net,
&self.node_id,
&self.bridge_dir,
&self.runtime_dir,
self.embed.clone(),
)
.await;
self.bridge = Some(runtime);
}
pub async fn stop_bridge(&mut self) {
if let Some(bridge) = self.bridge.take() {
tear_down(bridge);
}
wait_until_not_listening(&Endpoint::in_dir("bridge", self.runtime_dir.clone())).await;
}
}
impl Drop for Host {
fn drop(&mut self) {
if let Some(bridge) = self.bridge.take() {
std::thread::spawn(move || drop(bridge));
}
}
}
pub fn introduce(a: &Host, b: &Host) {
let shared = GrainId::random();
for host in [a, b] {
std::fs::write(sync_id_path(host), format!("{shared}\n")).unwrap();
}
copy_device(a, b);
copy_device(b, a);
}
pub fn introduce_all(hosts: &[&Host]) {
let shared = GrainId::random();
for host in hosts {
std::fs::write(sync_id_path(host), format!("{shared}\n")).unwrap();
}
for from in hosts {
for to in hosts {
if !std::ptr::eq(*from, *to) {
copy_device(from, to);
}
}
}
}
fn sync_id_path(host: &Host) -> PathBuf {
host.ws.join(format!(".{}", ctx().app_name)).join("sync-id")
}
fn copy_device(from: &Host, to: &Host) {
let workgroup = Workgroup::open(&from.bridge_dir).unwrap().unwrap();
let own = workgroup.this_device(&from.node_id).unwrap();
let source = from
.bridge_dir
.devices_dir(from.workgroup_id)
.join(own.file_name());
let text = std::fs::read_to_string(&source).unwrap();
let mut record: toml::Table = toml::from_str(&text).unwrap();
let name = record
.get("name")
.and_then(|v| v.as_str())
.unwrap_or("peer")
.to_owned();
record.insert("name".into(), toml::Value::String(format!("~{name}")));
let destination = to
.bridge_dir
.devices_dir(to.workgroup_id)
.join(own.file_name());
std::fs::create_dir_all(destination.parent().unwrap()).unwrap();
std::fs::write(&destination, toml::to_string_pretty(&record).unwrap()).unwrap();
}
fn client_info(kind: &str) -> ClientInfo {
ClientInfo {
kind: kind.to_owned(),
version: VERSION.to_owned(),
api: sapphire_backend::protocol::API_VERSION,
app_api: None,
pid: std::process::id(),
}
}
async fn start_bridge(
net: &LoopbackNetwork,
node_id: &str,
bridge_dir: &BridgeDir,
runtime_dir: &std::path::Path,
embed: Option<Arc<dyn sapphire_framework_bridge::EmbedProvider>>,
) -> tokio::runtime::Runtime {
let control = Endpoint::in_dir("bridge", runtime_dir.to_path_buf());
let data = Endpoint::in_dir("bridge-data", runtime_dir.to_path_buf());
let build = {
let (net, node_id, bridge_dir) = (net.clone(), node_id.to_owned(), bridge_dir.clone());
let control = control.clone();
move || {
let bridge = Bridge::new(
bridge_dir.clone(),
Arc::new(net.transport(&node_id)),
VERSION,
)
.map(|bridge| {
let bridge = bridge
.net(NetConfig {
wake_on_sync: false,
discovery: false,
relays: Vec::new(),
use_default_relays: false,
..NetConfig::default()
})
.control_endpoint(control.clone())
.data_endpoint(data.clone())
.hello_timing(HelloTiming {
interval: Duration::from_millis(100),
dead: Duration::from_millis(600),
});
match &embed {
Some(provider) => bridge.embed_provider(Arc::clone(provider)),
None => bridge,
}
});
std::future::ready(bridge)
}
};
let bridge_runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.expect("a runtime for the bridge");
let mut bridge_task = bridge_runtime.spawn(serve_restarting(build));
wait_until_listening(&control, "the bridge", &mut bridge_task).await;
bridge_runtime
}
fn tear_down(runtime: tokio::runtime::Runtime) {
std::thread::spawn(move || drop(runtime))
.join()
.expect("the bridge's runtime is torn down");
}
async fn wait_until_not_listening(endpoint: &Endpoint) {
let deadline = Instant::now() + Duration::from_secs(10);
while sapphire_ipc::probe(endpoint).await.unwrap_or(false) {
assert!(
Instant::now() < deadline,
"the bridge kept listening after it was stopped"
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
async fn wait_until_listening<T: std::fmt::Debug>(
endpoint: &Endpoint,
what: &str,
task: &mut tokio::task::JoinHandle<T>,
) {
let deadline = Instant::now() + Duration::from_secs(10);
while !sapphire_ipc::probe(endpoint).await.unwrap_or(false) {
if task.is_finished() {
panic!("{what} stopped before it listened: {:?}", task.await);
}
assert!(Instant::now() < deadline, "{what} never started listening");
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
pub async fn enable_sync(host: &Host) {
let _: proto::SyncEnableResult = host
.client
.call(proto::SYNC_ENABLE, serde_json::json!({}))
.await
.unwrap();
}
pub async fn enable_sync_after_restart(host: &Host) {
let deadline = Instant::now() + Duration::from_secs(10);
loop {
let answer: std::result::Result<proto::SyncEnableResult, _> = host
.client
.call(proto::SYNC_ENABLE, serde_json::json!({}))
.await;
match answer {
Ok(_) => return,
Err(err)
if format!("{err:?}").contains("Database already open")
&& Instant::now() < deadline =>
{
tokio::time::sleep(Duration::from_millis(50)).await;
}
Err(err) => panic!("sync.enable failed: {err:?}"),
}
}
}
pub async fn synced_pair(net: &LoopbackNetwork) -> (Host, Host) {
let a = start_host(net, NODE_A, "host-a").await;
let b = start_host(net, NODE_B, "host-b").await;
introduce(&a, &b);
enable_sync(&a).await;
enable_sync(&b).await;
(a, b)
}
pub async fn synced_triple(net: &LoopbackNetwork) -> (Host, Host, Host) {
let a = start_host(net, NODE_A, "host-a").await;
let s = start_host(net, NODE_S, "host-s").await;
let b = start_host(net, NODE_B, "host-b").await;
introduce_all(&[&a, &s, &b]);
enable_sync(&a).await;
enable_sync(&s).await;
enable_sync(&b).await;
(a, s, b)
}
pub async fn settle(hosts: &[&Host]) {
let deadline = Instant::now() + Duration::from_secs(30);
loop {
let mut settled = true;
for host in hosts {
let Some(runtime) = host.runtime() else {
settled = false;
break;
};
let peers = host
.bridge_client
.peers()
.await
.expect("the bridge answers peers");
let connected: HashSet<GrainId> = peers
.peers
.iter()
.filter(|p| p.connected && p.node_id != host.node_id())
.map(|p| p.device_id)
.collect();
let open: HashSet<GrainId> = runtime
.live_session_devices(&host.ws)
.await
.into_iter()
.collect();
if !connected.is_subset(&open) {
eprintln!(
"NOT SETTLED on {}: connected={:?} open={:?}",
host.node_id(),
connected,
open
);
settled = false;
break;
}
}
if settled {
return;
}
assert!(
Instant::now() < deadline,
"the hosts never settled into live sessions"
);
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
pub async fn await_primary(hosts: &[&Host], want: GrainId) {
let deadline = Instant::now() + Duration::from_secs(30);
loop {
let mut all = true;
for host in hosts {
let peers = host.bridge().peers().await.expect("peers");
let ws = host.workspace_id().await;
if peers.roles_for(ws).and_then(|r| r.primary) != Some(want) {
all = false;
}
}
if all {
return;
}
if Instant::now() >= deadline {
for host in hosts {
eprintln!(
"peers of {}: {:?}",
host.node_id(),
host.bridge().peers().await
);
}
panic!("the hosts never agreed on {want}");
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
pub fn frames_sent(host: &Host) -> u64 {
host.net.frames_sent(host.node_id())
}
pub async fn quiet(hosts: &[&Host], quiet_for: Duration, deadline: Duration) -> Vec<u64> {
let deadline = Instant::now() + deadline;
let mut last: Vec<u64> = hosts.iter().map(|host| frames_sent(host)).collect();
let mut unchanged_since = Instant::now();
loop {
assert!(
Instant::now() < deadline,
"the hosts never went quiet: their counters sat at {last:?} while writes kept arriving"
);
tokio::time::sleep(Duration::from_millis(50)).await;
let current: Vec<u64> = hosts.iter().map(|host| frames_sent(host)).collect();
if current != last {
last = current;
unchanged_since = Instant::now();
continue;
}
if unchanged_since.elapsed() >= quiet_for {
return last;
}
}
}