use crate::command::harness::config::{BlueprintSpec, HarnessConfig};
use crate::command::service::build_request_params;
use crate::command::tangle::DevnetStack;
use alloy_primitives::{Address, Bytes, U256};
use blueprint_client_tangle::{TangleClient, TangleClientConfig, TangleSettings};
use color_eyre::eyre::{Result, eyre};
use std::io::Write as _;
use std::net::TcpListener;
use std::time::Duration;
use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::process::{Child, Command};
use tokio::task::JoinHandle;
struct SpawnedBlueprint {
name: String,
#[allow(dead_code)]
service_id: u64,
port: u16,
child: Child,
_log_task: JoinHandle<()>,
_settings_dir: tempfile::TempDir,
}
pub struct Orchestrator {
stack: DevnetStack,
client: TangleClient,
blueprints: Vec<SpawnedBlueprint>,
}
impl Orchestrator {
pub async fn bootstrap(config: &HarnessConfig) -> Result<Self> {
if !config.chain.anvil {
return Err(eyre!(
"remote RPC mode not yet supported — set chain.anvil = true"
));
}
println!("Starting local Tangle devnet (anvil + contracts)...");
let stack = DevnetStack::spawn(config.chain.include_anvil_logs).await?;
println!(" HTTP RPC: {}", stack.http_rpc_url());
println!(" WS RPC: {}", stack.ws_rpc_url());
println!(" Tangle: {:?}", stack.tangle_contract());
let settings = TangleSettings {
blueprint_id: 0,
service_id: None,
tangle_contract: stack.tangle_contract(),
staking_contract: stack.staking_contract(),
status_registry_contract: stack.status_registry_contract(),
};
let client_config = TangleClientConfig::new(
stack.http_rpc_url(),
stack.ws_rpc_url(),
stack.keystore_path(),
settings,
);
let client = TangleClient::new(client_config)
.await
.map_err(|e| eyre!("failed to build TangleClient: {e}"))?;
println!(" Operator: {:?}", client.account());
println!();
Ok(Self {
stack,
client,
blueprints: Vec::new(),
})
}
pub async fn spawn_blueprints(&mut self, config: &HarnessConfig) -> Result<()> {
let self_exe = std::env::current_exe()
.map_err(|e| eyre!("failed to find cargo-tangle binary: {e}"))?;
let operator_address = self.client.account();
for (idx, bp) in config.blueprints.iter().enumerate() {
let port = bp
.port
.unwrap_or_else(|| allocate_free_port().unwrap_or(9000 + idx as u16));
let blueprint_id = 0u64; let params = build_request_params(
blueprint_id,
vec![operator_address], None, vec![], 7200, Address::ZERO, U256::ZERO, Bytes::new(), );
let (_tx, service_id) = self
.client
.request_service(params)
.await
.map_err(|e| eyre!("[{}] failed to create on-chain service: {e}", bp.name))?;
println!(
"[{}] Service created on-chain: service_id={}, port={}, path={}",
bp.name,
service_id,
port,
bp.path.display()
);
let settings_dir = tempfile::TempDir::new()
.map_err(|e| eyre!("failed to create temp settings dir: {e}"))?;
let settings_path = settings_dir.path().join("settings.env");
write_settings_env(&settings_path, blueprint_id, service_id, &self.stack)?;
let binary = bp
.binary
.as_ref()
.map(|p| p.to_path_buf())
.unwrap_or_else(|| self_exe.clone());
let mut cmd = Command::new(&binary);
if bp.binary.is_some() {
cmd.arg("run");
}
if bp.binary.is_none() {
cmd.args(["tangle", "blueprint", "run", "--no-vm", "--settings-file"]);
cmd.arg(&settings_path);
cmd.args(["--http-rpc-url"]);
cmd.arg(self.stack.http_rpc_url().as_str());
cmd.args(["--ws-rpc-url"]);
cmd.arg(self.stack.ws_rpc_url().as_str());
cmd.args(["--keystore-path"]);
cmd.arg(&self.stack.keystore_path());
}
cmd.env_clear();
for key in &[
"PATH",
"HOME",
"USER",
"TMPDIR",
"RUST_LOG",
"RUST_BACKTRACE",
] {
if let Ok(val) = std::env::var(key) {
cmd.env(key, &val);
}
}
cmd.env("HTTP_RPC_URL", self.stack.http_rpc_url().as_str());
cmd.env("WS_RPC_URL", self.stack.ws_rpc_url().as_str());
cmd.env("KEYSTORE_URI", &self.stack.keystore_path());
cmd.env("DATA_DIR", self.stack.data_dir().display().to_string());
cmd.env("BLUEPRINT_ID", blueprint_id.to_string());
cmd.env("SERVICE_ID", service_id.to_string());
cmd.env(
"TANGLE_CONTRACT",
format!("{:?}", self.stack.tangle_contract()),
);
cmd.env(
"STAKING_CONTRACT",
format!("{:?}", self.stack.staking_contract()),
);
cmd.env(
"STATUS_REGISTRY_CONTRACT",
format!("{:?}", self.stack.status_registry_contract()),
);
cmd.env("PROTOCOL", "tangle");
cmd.env("TEST_MODE", "true");
const PROTOCOL_VARS: &[&str] = &[
"HTTP_RPC_URL",
"WS_RPC_URL",
"KEYSTORE_URI",
"DATA_DIR",
"BLUEPRINT_ID",
"SERVICE_ID",
"TANGLE_CONTRACT",
"STAKING_CONTRACT",
"STATUS_REGISTRY_CONTRACT",
"PROTOCOL",
"TEST_MODE",
];
for (k, v) in &bp.env {
if PROTOCOL_VARS.contains(&k.as_str()) {
eprintln!(
"[{}] WARNING: ignoring bp.env override of protocol var {k}",
bp.name
);
continue;
}
cmd.env(k, v);
}
cmd.env("PORT", port.to_string());
cmd.env("MODAL_OP_SERVER__PORT", port.to_string());
cmd.env("VLLM_OP_SERVER__PORT", port.to_string());
cmd.current_dir(&bp.path);
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
let mut child = cmd.spawn().map_err(|e| {
eyre!(
"failed to spawn blueprint-manager for '{}': {e}\n binary: {}\n cwd: {}",
bp.name,
binary.display(),
bp.path.display()
)
})?;
let log_task =
spawn_log_forwarder(bp.name.clone(), child.stdout.take(), child.stderr.take());
self.blueprints.push(SpawnedBlueprint {
name: bp.name.clone(),
service_id,
port,
child,
_log_task: log_task,
_settings_dir: settings_dir,
});
}
tokio::time::sleep(Duration::from_secs(3)).await;
for bp in &self.blueprints {
let spec = config
.blueprints
.iter()
.find(|s| s.name == bp.name)
.unwrap();
let timeout = Duration::from_secs(spec.startup_timeout_secs);
let health_url = format!("http://127.0.0.1:{}{}", bp.port, spec.health_path);
println!("[{}] Waiting for health at {} ...", bp.name, health_url);
match wait_for_health(&health_url, timeout).await {
Ok(()) => println!("[{}] Healthy", bp.name),
Err(e) => {
eprintln!("[{}] Health check failed: {e}", bp.name);
}
}
}
if let Some(router_url) = &config.router.url {
println!();
println!("Registering operators with router at {router_url} ...");
for bp in &self.blueprints {
let spec = config
.blueprints
.iter()
.find(|s| s.name == bp.name)
.unwrap();
let endpoint_url = spec
.public_url
.clone()
.unwrap_or_else(|| format!("http://127.0.0.1:{}", bp.port));
match register_with_router(router_url, &bp.name, &endpoint_url, spec).await {
Ok(()) => println!("[{}] Registered with router", bp.name),
Err(e) => eprintln!("[{}] Router registration failed: {e}", bp.name),
}
}
}
Ok(())
}
pub async fn run_until_shutdown(mut self) -> Result<()> {
println!();
println!(
"Harness up. {} blueprint(s) running.",
self.blueprints.len()
);
println!("Press Ctrl+C to stop.");
println!();
tokio::select! {
_ = tokio::signal::ctrl_c() => {
println!();
println!("Shutdown signal received, stopping blueprints...");
}
result = wait_for_any_exit(&mut self.blueprints) => {
match result {
Some((name, code)) => {
eprintln!();
eprintln!("[{name}] exited unexpectedly with code {code:?}, shutting down...");
}
None => {
eprintln!("All blueprints exited.");
}
}
}
}
for bp in &mut self.blueprints {
let _ = bp.child.start_kill();
}
for bp in &mut self.blueprints {
let _ = tokio::time::timeout(Duration::from_secs(5), bp.child.wait()).await;
}
self.stack.shutdown().await;
println!("Harness stopped.");
Ok(())
}
}
fn write_settings_env(
path: &std::path::Path,
blueprint_id: u64,
service_id: u64,
stack: &DevnetStack,
) -> Result<()> {
let mut f = std::fs::File::create(path)
.map_err(|e| eyre!("failed to create settings.env at {}: {e}", path.display()))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600));
}
writeln!(f, "BLUEPRINT_ID={blueprint_id}")?;
writeln!(f, "SERVICE_ID={service_id}")?;
writeln!(f, "TANGLE_CONTRACT={:?}", stack.tangle_contract())?;
writeln!(f, "STAKING_CONTRACT={:?}", stack.staking_contract())?;
writeln!(
f,
"STATUS_REGISTRY_CONTRACT={:?}",
stack.status_registry_contract()
)?;
Ok(())
}
fn allocate_free_port() -> Result<u16> {
let listener =
TcpListener::bind("127.0.0.1:0").map_err(|e| eyre!("failed to allocate free port: {e}"))?;
let port = listener.local_addr()?.port();
drop(listener);
Ok(port)
}
async fn wait_for_health(url: &str, timeout: Duration) -> Result<()> {
let start = tokio::time::Instant::now();
let mut last_error = String::new();
let url = url.trim_start_matches("http://");
let (addr, path) = url.split_once('/').unwrap_or((url, ""));
let path = format!("/{}", path.replace(['\r', '\n'], ""));
loop {
if start.elapsed() > timeout {
return Err(eyre!(
"health check timed out after {}s — last error: {last_error}",
timeout.as_secs()
));
}
match tokio::net::TcpStream::connect(addr).await {
Ok(mut stream) => {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let req =
format!("GET {path} HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\n\r\n");
if stream.write_all(req.as_bytes()).await.is_ok() {
let mut buf = vec![0u8; 256];
if let Ok(n) = stream.read(&mut buf).await {
let response = String::from_utf8_lossy(&buf[..n]);
if response.contains("200") {
return Ok(());
}
last_error = response.lines().next().unwrap_or("unknown").to_string();
}
}
}
Err(e) => {
last_error = e.to_string();
}
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
}
fn spawn_log_forwarder(
name: String,
stdout: Option<tokio::process::ChildStdout>,
stderr: Option<tokio::process::ChildStderr>,
) -> JoinHandle<()> {
tokio::spawn(async move {
let name2 = name.clone();
let stdout_task = tokio::spawn(async move {
if let Some(out) = stdout {
let reader = BufReader::new(out);
let mut lines = reader.lines();
while let Ok(Some(line)) = lines.next_line().await {
println!("[{name}] {line}");
}
}
});
let stderr_task = tokio::spawn(async move {
if let Some(err) = stderr {
let reader = BufReader::new(err);
let mut lines = reader.lines();
while let Ok(Some(line)) = lines.next_line().await {
eprintln!("[{name2}] {line}");
}
}
});
let _ = tokio::join!(stdout_task, stderr_task);
})
}
async fn wait_for_any_exit(blueprints: &mut [SpawnedBlueprint]) -> Option<(String, Option<i32>)> {
loop {
for bp in blueprints.iter_mut() {
match bp.child.try_wait() {
Ok(Some(status)) => {
return Some((bp.name.clone(), status.code()));
}
Ok(None) => {} Err(_) => {
return Some((bp.name.clone(), None));
}
}
}
tokio::time::sleep(Duration::from_millis(250)).await;
}
}
async fn register_with_router(
router_url: &str,
name: &str,
endpoint_url: &str,
spec: &BlueprintSpec,
) -> Result<()> {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let blueprint_type = spec.blueprint_type.as_deref().unwrap_or("inference");
let models_value: Vec<serde_json::Value> = spec
.models
.iter()
.map(|m| {
serde_json::json!({
"modelId": m.id,
"inputPrice": m.input_price,
"outputPrice": m.output_price,
})
})
.collect();
let body = serde_json::to_string(&serde_json::json!({
"name": name,
"endpointUrl": endpoint_url,
"blueprintType": blueprint_type,
"models": models_value,
}))
.unwrap_or_default();
let url = router_url
.trim_start_matches("https://")
.trim_start_matches("http://");
let is_https = router_url.starts_with("https://");
if is_https {
let output = tokio::process::Command::new("curl")
.args([
"-s",
"-X",
"POST",
&format!("{router_url}/api/operators"),
"-H",
"Content-Type: application/json",
"-d",
&body,
])
.output()
.await
.map_err(|e| eyre!("curl failed: {e}"))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(eyre!("router registration failed: {stderr}"));
}
let stdout = String::from_utf8_lossy(&output.stdout);
if stdout.contains("error") {
return Err(eyre!("router returned error: {stdout}"));
}
} else {
let (addr, _) = url.split_once('/').unwrap_or((url, ""));
let mut stream = tokio::net::TcpStream::connect(addr)
.await
.map_err(|e| eyre!("failed to connect to router at {addr}: {e}"))?;
let req = format!(
"POST /api/operators HTTP/1.1\r\nHost: {addr}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
stream.write_all(req.as_bytes()).await?;
let mut buf = vec![0u8; 4096];
let n = stream.read(&mut buf).await?;
let response = String::from_utf8_lossy(&buf[..n]);
if !response.contains("200") && !response.contains("201") {
return Err(eyre!(
"router registration failed: {}",
response.lines().next().unwrap_or("unknown")
));
}
}
Ok(())
}