pub mod inventory;
pub mod remote;
pub mod serve;
use std::path::PathBuf;
use std::sync::Arc;
use car_fleet::{FleetComposite, InstanceInventory, InstanceRef, InventoryProvider, WorkerProfile};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::handler::JsonRpcMessage;
use crate::session::{ClientSession, ServerState};
pub use remote::RemoteWorktreeAgent;
const DEFAULT_MAX_PARALLEL: u32 = 2;
const DEFAULT_LOCAL_PARALLEL: u32 = 2;
pub(super) const DEFAULT_MAX_SUBTASK_SECS: u64 = 1800;
const INVENTORY_TIMEOUT: std::time::Duration = car_fleet::DEFAULT_INVENTORY_TIMEOUT;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FleetWorkerConfig {
#[serde(default)]
pub accepts_work: bool,
#[serde(default)]
pub repos: Vec<PathBuf>,
#[serde(default = "default_max_parallel")]
pub max_parallel: u32,
#[serde(default = "default_local_parallel")]
pub local_parallel: u32,
#[serde(default = "default_dispatches_per_hour")]
pub dispatches_per_hour: u32,
#[serde(default = "default_max_subtask_secs")]
pub max_subtask_secs: u64,
#[serde(default)]
pub fetch_missing_base: bool,
#[serde(default = "default_fetch_remote")]
pub fetch_remote: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub allowed_tools: Option<Vec<String>>,
}
fn default_max_parallel() -> u32 {
DEFAULT_MAX_PARALLEL
}
fn default_local_parallel() -> u32 {
DEFAULT_LOCAL_PARALLEL
}
fn default_dispatches_per_hour() -> u32 {
car_fleet::DEFAULT_DISPATCHES_PER_WINDOW
}
fn default_max_subtask_secs() -> u64 {
DEFAULT_MAX_SUBTASK_SECS
}
fn default_fetch_remote() -> String {
"origin".to_string()
}
impl Default for FleetWorkerConfig {
fn default() -> Self {
Self {
accepts_work: false,
repos: Vec::new(),
max_parallel: DEFAULT_MAX_PARALLEL,
local_parallel: DEFAULT_LOCAL_PARALLEL,
dispatches_per_hour: car_fleet::DEFAULT_DISPATCHES_PER_WINDOW,
max_subtask_secs: DEFAULT_MAX_SUBTASK_SECS,
fetch_missing_base: false,
fetch_remote: default_fetch_remote(),
allowed_tools: None,
}
}
}
impl FleetWorkerConfig {
fn path() -> Option<PathBuf> {
car_home::root().map(|r| r.join("fleet-worker.json"))
}
pub fn load() -> Self {
let Some(path) = Self::path() else {
return Self::default();
};
match std::fs::read_to_string(&path) {
Ok(text) => serde_json::from_str(&text).unwrap_or_else(|e| {
tracing::warn!(path = %path.display(), error = %e, "unreadable fleet worker config; declining work");
Self::default()
}),
Err(_) => Self::default(),
}
}
pub fn save(&self) -> Result<(), String> {
let path = Self::path().ok_or("cannot resolve the CAR state root")?;
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| format!("create {}: {e}", parent.display()))?;
}
let json = serde_json::to_string_pretty(self).map_err(|e| e.to_string())?;
let tmp = path.with_extension("json.tmp");
std::fs::write(&tmp, json).map_err(|e| format!("write {}: {e}", tmp.display()))?;
std::fs::rename(&tmp, &path).map_err(|e| format!("rename into {}: {e}", path.display()))
}
}
fn worktree_base() -> PathBuf {
car_home::root_or_relative()
.join("fleet-worker")
.join("worktrees")
}
async fn detected_adapters() -> Vec<car_external_agents::ExternalAgentSpec> {
use tokio::sync::Mutex;
static CACHE: std::sync::OnceLock<
Mutex<
Option<(
std::time::Instant,
Vec<car_external_agents::ExternalAgentSpec>,
)>,
>,
> = std::sync::OnceLock::new();
const TTL: std::time::Duration = std::time::Duration::from_secs(60);
let cache = CACHE.get_or_init(|| Mutex::new(None));
let mut guard = cache.lock().await;
if let Some((at, specs)) = guard.as_ref() {
if at.elapsed() < TTL {
return specs.clone();
}
}
let specs = car_external_agents::detect_runnable().await;
*guard = Some((std::time::Instant::now(), specs.clone()));
specs
}
pub async fn worker_profile() -> WorkerProfile {
let config = FleetWorkerConfig::load();
let adapters: Vec<String> = detected_adapters()
.await
.into_iter()
.map(|s| s.id)
.collect();
let repo_root_commits = config
.repos
.iter()
.filter_map(|p| car_fleet::root_commit(p).ok())
.collect();
WorkerProfile {
accepts_work: config.accepts_work,
adapters,
max_parallel: config.max_parallel,
repo_root_commits,
}
}
async fn remote_providers(
state: &ServerState,
) -> (Vec<Arc<dyn InventoryProvider>>, Vec<InstanceInventory>) {
let identity = {
state
.peer_identity
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone()
};
let mut providers: Vec<Arc<dyn InventoryProvider>> = Vec::new();
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
for peer in crate::peers::snapshot_parslee(state).await {
if let car_peers::PeerAddress::A2a { base_url } = &peer.address {
if seen.insert(base_url.clone()) {
providers.push(Arc::new(inventory::PeerInventoryProvider::new(
InstanceRef::remote(peer.name.clone(), "parslee", base_url.clone()),
identity.clone(),
)));
}
}
}
if let Ok(registry) = car_a2a::peers::PeerRegistry::user_default() {
for entry in registry.list() {
if seen.insert(entry.url.clone()) {
let name = entry.label.clone().unwrap_or_else(|| entry.slug.clone());
providers.push(Arc::new(inventory::PeerInventoryProvider::new(
InstanceRef::remote(name, "registry", entry.url.clone()),
identity.clone(),
)));
}
}
}
let visible_only = crate::peers::snapshot_lan(state)
.into_iter()
.filter_map(|peer| {
let car_peers::PeerAddress::A2a { base_url } = &peer.address else {
return None;
};
if seen.contains(base_url) {
return None;
}
Some(InstanceInventory::unreachable(
InstanceRef::remote(peer.name.clone(), "lan", base_url.clone()),
"discovered on the local network but not a trusted peer — anyone can advertise \
any name, so promote it with `a2a.peers.add` before CAR will contact it",
car_fleet::now_ms(),
))
})
.collect();
(providers, visible_only)
}
pub async fn composite(
state: &ServerState,
session: Option<&ClientSession>,
include_remote: bool,
timeout: std::time::Duration,
) -> FleetComposite {
let local = match session {
Some(s) => inventory::local_inventory(state, Some(&s.runtime), Some(&s.memgine)).await,
None => inventory::local_inventory(state, None, None).await,
};
let self_name = local.instance.name.clone();
let mut all = vec![local];
if include_remote {
let (providers, visible_only) = remote_providers(state).await;
all.extend(car_fleet::gather(&providers, timeout).await);
all.extend(visible_only);
}
car_fleet::compose(self_name, all)
}
pub async fn handle_fleet_inventory(
state: &ServerState,
session: &ClientSession,
) -> Result<Value, String> {
let inv =
inventory::local_inventory(state, Some(&session.runtime), Some(&session.memgine)).await;
serde_json::to_value(inv).map_err(|e| e.to_string())
}
pub async fn handle_fleet_composite(
msg: &JsonRpcMessage,
state: &ServerState,
session: &ClientSession,
) -> Result<Value, String> {
let include_remote = msg
.params
.get("include_remote")
.and_then(|v| v.as_bool())
.unwrap_or(true);
let timeout = msg
.params
.get("timeout_ms")
.and_then(|v| v.as_u64())
.map(std::time::Duration::from_millis)
.unwrap_or(INVENTORY_TIMEOUT);
let composite = composite(state, Some(session), include_remote, timeout).await;
serde_json::to_value(composite).map_err(|e| e.to_string())
}
pub async fn handle_fleet_worker_get() -> Result<Value, String> {
let config = FleetWorkerConfig::load();
let profile = worker_profile().await;
Ok(serde_json::json!({
"config": config,
"profile": profile,
}))
}
pub async fn handle_fleet_worker_set(
msg: &JsonRpcMessage,
session: &ClientSession,
) -> Result<Value, String> {
let bound_agent = session.agent_id.lock().await.clone();
if let Some(agent) = bound_agent {
if !session.is_host.load(std::sync::atomic::Ordering::Acquire) {
return Err(format!(
"`fleet.worker.set` is operator-only: `{agent}` cannot enroll this machine to \
run peers' coding subtasks against local checkouts"
));
}
}
let mut config = FleetWorkerConfig::load();
if let Some(v) = msg.params.get("accepts_work").and_then(|v| v.as_bool()) {
config.accepts_work = v;
}
if let Some(list) = msg.params.get("repos").and_then(|v| v.as_array()) {
let mut repos = Vec::new();
for entry in list {
let path = PathBuf::from(entry.as_str().ok_or("`repos` entries must be strings")?);
car_fleet::root_commit(&path)
.map_err(|e| format!("`{}` is not a git repository: {e}", path.display()))?;
repos.push(path);
}
config.repos = repos;
}
if let Some(n) = msg.params.get("max_parallel").and_then(|v| v.as_u64()) {
config.max_parallel = n.min(64) as u32;
}
if let Some(n) = msg.params.get("local_parallel").and_then(|v| v.as_u64()) {
config.local_parallel = n.min(64) as u32;
}
if let Some(n) = msg
.params
.get("dispatches_per_hour")
.and_then(|v| v.as_u64())
{
config.dispatches_per_hour = n.min(10_000) as u32;
}
if let Some(n) = msg.params.get("max_subtask_secs").and_then(|v| v.as_u64()) {
config.max_subtask_secs = n.clamp(60, 24 * 3600);
}
if let Some(v) = msg
.params
.get("fetch_missing_base")
.and_then(|v| v.as_bool())
{
config.fetch_missing_base = v;
}
if let Some(remote) = msg.params.get("fetch_remote").and_then(|v| v.as_str()) {
if remote.is_empty()
|| remote.starts_with('-')
|| !remote
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/'))
{
return Err(format!("`{remote}` is not a usable git remote name"));
}
config.fetch_remote = remote.to_string();
}
if let Some(list) = msg.params.get("allowed_tools").and_then(|v| v.as_array()) {
config.allowed_tools = Some(
list.iter()
.filter_map(|v| v.as_str().map(String::from))
.collect(),
);
}
config.save()?;
Ok(serde_json::json!({
"config": config,
"profile": worker_profile().await,
}))
}
pub struct DaemonFleetResponder {
state: std::sync::Weak<ServerState>,
runtime: Arc<car_engine::Runtime>,
}
impl DaemonFleetResponder {
pub fn new(state: std::sync::Weak<ServerState>, runtime: Arc<car_engine::Runtime>) -> Self {
Self { state, runtime }
}
}
#[async_trait::async_trait]
impl car_a2a::FleetResponder for DaemonFleetResponder {
async fn inventory(&self) -> Result<Value, String> {
let state = self
.state
.upgrade()
.ok_or_else(|| "daemon is shutting down".to_string())?;
let inv = inventory::local_inventory(&state, Some(&self.runtime), None).await;
serde_json::to_value(inv).map_err(|e| e.to_string())
}
async fn run_subtask(&self, dispatch: Value, caller: Option<&str>) -> Result<Value, String> {
let Some(caller) = caller else {
return Err(
"refusing an unattributed fleet dispatch: this surface accepts work only from a CAR peer whose signature identifies it"
.to_string(),
);
};
let dispatch: car_fleet::SubtaskDispatch =
serde_json::from_value(dispatch).map_err(|e| format!("invalid dispatch: {e}"))?;
let outcome = serve::run_dispatch(dispatch, caller).await?;
serde_json::to_value(outcome).map_err(|e| e.to_string())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum PoolExclusion {
NotEnrolled,
RepositoryNotServed,
NotRequested,
Unreachable,
NoAddress,
}
impl PoolExclusion {
pub fn as_str(self) -> &'static str {
match self {
PoolExclusion::NotEnrolled => "not_enrolled",
PoolExclusion::RepositoryNotServed => "repository_not_served",
PoolExclusion::NotRequested => "not_requested",
PoolExclusion::Unreachable => "unreachable",
PoolExclusion::NoAddress => "no_address",
}
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct PoolPlan {
pub remote_workers: Vec<String>,
pub excluded: Vec<(String, String)>,
}
impl PoolPlan {
pub fn local_only(&self) -> bool {
self.remote_workers.is_empty()
}
pub fn degraded_reason(&self) -> Option<String> {
if !self.local_only() {
return None;
}
if self.excluded.is_empty() {
return Some(
"no other CAR instance is reachable, so this ran on one machine".to_string(),
);
}
let mut counts: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new();
for (_, reason) in &self.excluded {
*counts.entry(reason.as_str()).or_default() += 1;
}
let detail = counts
.into_iter()
.map(|(reason, n)| format!("{n} {reason}"))
.collect::<Vec<_>>()
.join(", ");
Some(format!(
"no peer could take a subtask, so this ran on one machine ({detail})"
))
}
}
pub async fn build_pool(
state: &ServerState,
repo_root: &std::path::Path,
run_id: &str,
adapter: &str,
only: Option<&[String]>,
) -> Result<(car_multi::FleetPool, PoolPlan), String> {
let fingerprint = car_fleet::read_fingerprint(repo_root).map_err(|e| {
format!(
"cannot identify the repository at {}: {e}",
repo_root.display()
)
})?;
let config = FleetWorkerConfig::load();
let local: Arc<dyn car_multi::WorktreeAgent> = Arc::new(
car_external_agents::ForemanExternalAgent::new(adapter.to_string()),
);
let mut workers = vec![car_multi::FleetWorker::local(
car_a2a::lan::host_label(),
local,
config.local_parallel.max(1) as usize,
)];
let composite = composite(state, None, true, INVENTORY_TIMEOUT).await;
let identity = {
state
.peer_identity
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone()
};
let mut plan = PoolPlan::default();
let eligible: std::collections::HashSet<&str> = composite
.workers_for(&fingerprint.root_commit)
.into_iter()
.map(|c| c.instance.as_str())
.collect();
for inv in &composite.instances {
if inv.instance.kind == car_fleet::InstanceKind::Local {
continue;
}
let name = inv.instance.name.clone();
let reason = if !inv.reachable() {
PoolExclusion::Unreachable
} else if !eligible.contains(name.as_str()) {
match &inv.worker {
Some(w) if w.accepts_work => PoolExclusion::RepositoryNotServed,
_ => PoolExclusion::NotEnrolled,
}
} else if only.is_some_and(|only| !only.iter().any(|n| n == &name)) {
PoolExclusion::NotRequested
} else if inv.instance.base_url.is_none() {
PoolExclusion::NoAddress
} else {
continue;
};
plan.excluded.push((name, reason.as_str().to_string()));
}
for candidate in composite.workers_for(&fingerprint.root_commit) {
if candidate.kind == car_fleet::InstanceKind::Local {
continue;
}
if let Some(only) = only {
if !only.iter().any(|n| n == &candidate.instance) {
continue;
}
}
let Some(base_url) = composite
.instances
.iter()
.find(|i| i.instance.name == candidate.instance)
.and_then(|i| i.instance.base_url.clone())
else {
continue;
};
let wanted = candidate
.adapters
.iter()
.any(|a| a == adapter)
.then(|| adapter.to_string());
let agent = RemoteWorktreeAgent::new(
candidate.instance.clone(),
base_url,
identity.clone(),
fingerprint.clone(),
run_id,
)
.with_adapter(wanted);
plan.remote_workers.push(candidate.instance.clone());
workers.push(car_multi::FleetWorker::remote(
candidate.instance.clone(),
Arc::new(agent),
candidate.max_parallel.max(1) as usize,
));
}
if let Some(reason) = plan.degraded_reason() {
tracing::warn!(%reason, "distributed foreman run has no peer workers");
}
Ok((car_multi::FleetPool::new(workers), plan))
}
pub fn placements_value(placements: &[car_multi::Placement]) -> Value {
serde_json::to_value(placements).expect("placement ledger is plain data")
}
pub fn placements_json(pool: &car_multi::FleetPool) -> Value {
placements_value(&pool.placements())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_distributed_run_with_no_peers_says_why_rather_than_going_quiet() {
let plan = PoolPlan {
remote_workers: Vec::new(),
excluded: vec![
("studio".into(), "not_enrolled".into()),
("laptop".into(), "not_enrolled".into()),
("ci-box".into(), "repository_not_served".into()),
],
};
assert!(plan.local_only());
let reason = plan.degraded_reason().expect("degraded");
assert!(reason.contains("2 not_enrolled"), "{reason}");
assert!(reason.contains("1 repository_not_served"), "{reason}");
}
#[test]
fn a_pool_with_peers_reports_no_degradation() {
let plan = PoolPlan {
remote_workers: vec!["studio".into()],
excluded: vec![("laptop".into(), "not_enrolled".into())],
};
assert!(!plan.local_only());
assert!(plan.degraded_reason().is_none());
}
#[test]
fn no_reachable_peers_at_all_is_its_own_message() {
let plan = PoolPlan::default();
let reason = plan.degraded_reason().expect("degraded");
assert!(
reason.contains("no other CAR instance is reachable"),
"{reason}"
);
}
#[test]
fn the_shipped_default_declines_work() {
let c = FleetWorkerConfig::default();
assert!(!c.accepts_work);
assert!(c.repos.is_empty());
}
#[test]
fn a_laptop_does_not_fetch_unless_told_to() {
let c = FleetWorkerConfig::default();
assert!(!c.fetch_missing_base);
assert_eq!(c.fetch_remote, "origin");
}
#[test]
fn a_config_written_before_runner_mode_existed_still_declines() {
let parsed: FleetWorkerConfig =
serde_json::from_str("{\"accepts_work\": true}").expect("parses");
assert!(
!parsed.fetch_missing_base,
"absent must not read as enabled"
);
assert_eq!(parsed.fetch_remote, "origin");
}
#[test]
fn the_sender_does_not_choose_this_machines_limits() {
let parsed: FleetWorkerConfig =
serde_json::from_str("{\"accepts_work\": true, \"repos\": []}").expect("parses");
assert_eq!(
parsed.dispatches_per_hour,
car_fleet::DEFAULT_DISPATCHES_PER_WINDOW
);
assert_eq!(parsed.max_subtask_secs, DEFAULT_MAX_SUBTASK_SECS);
assert!(parsed.allowed_tools.is_none());
}
#[test]
fn an_unparseable_config_declines_rather_than_half_accepting() {
let parsed: FleetWorkerConfig =
serde_json::from_str("{\"accepts_work\": true}").expect("partial config parses");
assert!(parsed.accepts_work);
assert_eq!(parsed.max_parallel, DEFAULT_MAX_PARALLEL, "limits default");
assert!(
parsed.repos.is_empty(),
"and with no repos it can still serve nothing"
);
}
#[test]
fn worker_worktrees_live_outside_every_served_repository() {
let base = worktree_base();
assert!(
base.ends_with("fleet-worker/worktrees") || base.ends_with("fleet-worker\\worktrees")
);
}
}