Skip to main content

car_server_core/fleet/
mod.rs

1//! `fleet.*` — the composite view of everything CAR can reach, and the
2//! placement layer that lets one Foreman run span several instances.
3//!
4//! ## Why this exists
5//!
6//! `agents.peers` answers *who* is out there. `discovery.resolve` ranks
7//! services against a need. Neither answers the question a caller with work has:
8//! **what can the machines I can reach actually do, and which of them should
9//! run this?** Before this, a Foreman run's answer to "where does this subtask
10//! go" was structurally "the machine you are standing on", because nothing else
11//! was ever enumerated.
12//!
13//! `fleet.composite` enumerates it: every agent, capability, and model across
14//! this daemon and every reachable CAR instance, folded so one row names every
15//! instance that offers it. `foreman.run { distributed: true }` then spreads the
16//! subtasks over the instances that can serve the repository.
17//!
18//! ## The trust story is unchanged
19//!
20//! - Remote reads and dispatches ride the **existing** peer-authenticated A2A
21//!   listener (`build_router_with_peer_auth`). No new listener, no new door.
22//! - Discovery still is not trust: a LAN advertisement is listed as a visible
23//!   candidate and is never contacted until an operator promotes it, exactly as
24//!   `agents.message` requires.
25//! - Accepting farmed-out work is off until an operator turns it on, is limited
26//!   to named repositories, and is audited per dispatch (see [`serve`]).
27//! - The merge-verify gate does not move. A peer produces a patch; this host
28//!   gates it. See [`remote`].
29
30pub mod inventory;
31pub mod remote;
32pub mod serve;
33
34use std::path::PathBuf;
35use std::sync::Arc;
36
37use car_fleet::{FleetComposite, InstanceInventory, InstanceRef, InventoryProvider, WorkerProfile};
38use serde::{Deserialize, Serialize};
39use serde_json::Value;
40
41use crate::handler::JsonRpcMessage;
42use crate::session::{ClientSession, ServerState};
43
44pub use remote::RemoteWorktreeAgent;
45
46/// Subtasks this host runs for peers at once, when enrolled and unset.
47const DEFAULT_MAX_PARALLEL: u32 = 2;
48
49/// Subtasks this host runs *for itself* at once during a distributed run.
50const DEFAULT_LOCAL_PARALLEL: u32 = 2;
51
52/// Longest a peer's subtask may occupy this machine, when the operator has not
53/// said otherwise. Thirty minutes: past that a coding CLI is wedged, not slow.
54///
55/// The ORCHESTRATOR needs this too — its HTTP deadline has to cover the work the
56/// far side is allowed to spend, and the dispatch carries no negotiated budget
57/// (`SubtaskDispatch::timeout_secs` is `None` unless a caller sets one). One
58/// constant so the two sides cannot drift apart; see `remote::dispatch_client`.
59pub(super) const DEFAULT_MAX_SUBTASK_SECS: u64 = 1800;
60
61/// How long a peer gets to answer `car/fleetInventory` before it is reported
62/// unreachable. See `car_fleet::provider`.
63const INVENTORY_TIMEOUT: std::time::Duration = car_fleet::DEFAULT_INVENTORY_TIMEOUT;
64
65/// Whether this instance takes farmed-out coding work, and under what limits.
66///
67/// Persisted at `~/.car/fleet-worker.json` (under `CAR_HOME` when set) rather
68/// than read from the environment, because it is an operator decision that must
69/// survive a restart and be inspectable — an env var would make "does this
70/// machine run my peers' prompts?" depend on how the daemon happened to be
71/// launched.
72#[derive(Debug, Clone, Serialize, Deserialize)]
73pub struct FleetWorkerConfig {
74    /// Off by default. Turning it on lets a trusted peer run a coding CLI
75    /// against the checkouts named in `repos`.
76    #[serde(default)]
77    pub accepts_work: bool,
78    /// Repository checkouts this host will serve. A dispatch is matched to one
79    /// of these by root commit; anything else is declined.
80    #[serde(default)]
81    pub repos: Vec<PathBuf>,
82    /// Peers' subtasks run here at once.
83    #[serde(default = "default_max_parallel")]
84    pub max_parallel: u32,
85    /// This host's own share when it orchestrates a distributed run. Separate
86    /// from `max_parallel`, which bounds what peers may spend here.
87    #[serde(default = "default_local_parallel")]
88    pub local_parallel: u32,
89    /// Dispatches ONE peer may start here per hour.
90    ///
91    /// `max_parallel` bounds concurrency, which is not a bound on spend: a peer
92    /// that dispatches one subtask, waits, and dispatches the next never
93    /// exceeds it and can still drain this machine's coding-CLI quota. This is
94    /// the budget. `0` accepts nothing while keeping the rest of the config.
95    #[serde(default = "default_dispatches_per_hour")]
96    pub dispatches_per_hour: u32,
97    /// Hard ceiling on a peer-supplied `timeout_secs`.
98    ///
99    /// The dispatch carries a timeout the *sender* chose. A machine agreeing to
100    /// take work is not agreeing to be occupied for as long as the caller
101    /// likes, so the sender's value is clamped to this and an omitted one
102    /// defaults to it.
103    #[serde(default = "default_max_subtask_secs")]
104    pub max_subtask_secs: u64,
105    /// Fetch a missing base commit from this checkout's own remote instead of
106    /// declining — the posture that makes a machine a **runner** rather than a
107    /// person's laptop.
108    ///
109    /// Off by default, because on a laptop it means a peer's dispatch can cause
110    /// a network fetch in a repository the operator is working in. On a runner
111    /// it is the whole point: a machine that continuously tracks `origin`
112    /// always has the base commit, which is what turns an idle pool into a
113    /// working one. Adds no trust — the fetch targets the remote this checkout
114    /// is already configured with, never anything the dispatch supplied.
115    #[serde(default)]
116    pub fetch_missing_base: bool,
117    /// Remote a runner fetches from. `origin` unless an operator says otherwise.
118    #[serde(default = "default_fetch_remote")]
119    pub fetch_remote: String,
120    /// Tools a peer's coding CLI may use here, intersected with whatever the
121    /// dispatch asks for.
122    ///
123    /// `None` (the default) adds no restriction beyond the CLI's own, which is
124    /// the honest default: a coding subtask needs to read, edit, and usually
125    /// build, so a narrow allowlist mostly turns into every dispatch failing
126    /// for reasons the sender cannot see. Set it when this machine should be
127    /// stricter than the work requires.
128    #[serde(default, skip_serializing_if = "Option::is_none")]
129    pub allowed_tools: Option<Vec<String>>,
130}
131
132fn default_max_parallel() -> u32 {
133    DEFAULT_MAX_PARALLEL
134}
135
136fn default_local_parallel() -> u32 {
137    DEFAULT_LOCAL_PARALLEL
138}
139
140fn default_dispatches_per_hour() -> u32 {
141    car_fleet::DEFAULT_DISPATCHES_PER_WINDOW
142}
143
144fn default_max_subtask_secs() -> u64 {
145    DEFAULT_MAX_SUBTASK_SECS
146}
147
148fn default_fetch_remote() -> String {
149    "origin".to_string()
150}
151
152impl Default for FleetWorkerConfig {
153    fn default() -> Self {
154        Self {
155            accepts_work: false,
156            repos: Vec::new(),
157            max_parallel: DEFAULT_MAX_PARALLEL,
158            local_parallel: DEFAULT_LOCAL_PARALLEL,
159            dispatches_per_hour: car_fleet::DEFAULT_DISPATCHES_PER_WINDOW,
160            max_subtask_secs: DEFAULT_MAX_SUBTASK_SECS,
161            fetch_missing_base: false,
162            fetch_remote: default_fetch_remote(),
163            allowed_tools: None,
164        }
165    }
166}
167
168impl FleetWorkerConfig {
169    fn path() -> Option<PathBuf> {
170        car_home::root().map(|r| r.join("fleet-worker.json"))
171    }
172
173    /// Read the config, falling back to the declining default.
174    ///
175    /// A malformed file reads as the default rather than as an error: the
176    /// failure mode of "unparseable config" must be *declining work*, never
177    /// accepting it under a half-read set of limits.
178    pub fn load() -> Self {
179        let Some(path) = Self::path() else {
180            return Self::default();
181        };
182        match std::fs::read_to_string(&path) {
183            Ok(text) => serde_json::from_str(&text).unwrap_or_else(|e| {
184                tracing::warn!(path = %path.display(), error = %e, "unreadable fleet worker config; declining work");
185                Self::default()
186            }),
187            Err(_) => Self::default(),
188        }
189    }
190
191    pub fn save(&self) -> Result<(), String> {
192        let path = Self::path().ok_or("cannot resolve the CAR state root")?;
193        if let Some(parent) = path.parent() {
194            std::fs::create_dir_all(parent)
195                .map_err(|e| format!("create {}: {e}", parent.display()))?;
196        }
197        let json = serde_json::to_string_pretty(self).map_err(|e| e.to_string())?;
198        let tmp = path.with_extension("json.tmp");
199        std::fs::write(&tmp, json).map_err(|e| format!("write {}: {e}", tmp.display()))?;
200        std::fs::rename(&tmp, &path).map_err(|e| format!("rename into {}: {e}", path.display()))
201    }
202}
203
204/// Where worker worktrees are provisioned — under the state root, never inside
205/// the served repository, so a crashed dispatch cannot leave untracked files in
206/// an operator's checkout.
207fn worktree_base() -> PathBuf {
208    car_home::root_or_relative()
209        .join("fleet-worker")
210        .join("worktrees")
211}
212
213/// Installed coding CLIs, cached.
214///
215/// Detection spawns a `--version` subprocess per adapter, which is far too
216/// costly to repeat on every inventory read (and an inventory is read on every
217/// composite, by every peer). Installs change rarely; a minute is ample — the
218/// same reasoning and TTL `discovery.resolve` uses.
219async fn detected_adapters() -> Vec<car_external_agents::ExternalAgentSpec> {
220    use tokio::sync::Mutex;
221    static CACHE: std::sync::OnceLock<
222        Mutex<
223            Option<(
224                std::time::Instant,
225                Vec<car_external_agents::ExternalAgentSpec>,
226            )>,
227        >,
228    > = std::sync::OnceLock::new();
229    const TTL: std::time::Duration = std::time::Duration::from_secs(60);
230
231    let cache = CACHE.get_or_init(|| Mutex::new(None));
232    let mut guard = cache.lock().await;
233    if let Some((at, specs)) = guard.as_ref() {
234        if at.elapsed() < TTL {
235            return specs.clone();
236        }
237    }
238    let specs = car_external_agents::detect_runnable().await;
239    *guard = Some((std::time::Instant::now(), specs.clone()));
240    specs
241}
242
243/// This instance's worker posture, as peers see it.
244pub async fn worker_profile() -> WorkerProfile {
245    let config = FleetWorkerConfig::load();
246    let adapters: Vec<String> = detected_adapters()
247        .await
248        .into_iter()
249        .map(|s| s.id)
250        .collect();
251    // Root commits only. A peer needs to know *whether* this host can reproduce
252    // a base, not where the checkout lives — a path would leak the layout of a
253    // machine the peer has no business mapping.
254    let repo_root_commits = config
255        .repos
256        .iter()
257        .filter_map(|p| car_fleet::root_commit(p).ok())
258        .collect();
259    WorkerProfile {
260        accepts_work: config.accepts_work,
261        adapters,
262        max_parallel: config.max_parallel,
263        repo_root_commits,
264    }
265}
266
267/// Trusted peers, as inventory providers, plus rows for instances that are
268/// visible but deliberately not contacted.
269///
270/// The second half matters as much as the first. A LAN advertisement is an
271/// unauthenticated claim, so CAR never dials it until an operator promotes it —
272/// but leaving those hosts out of the composite entirely would leave an operator
273/// wondering why the machine they can see is not in the fleet. They appear as
274/// unreachable rows whose reason names the fix.
275async fn remote_providers(
276    state: &ServerState,
277) -> (Vec<Arc<dyn InventoryProvider>>, Vec<InstanceInventory>) {
278    let identity = {
279        state
280            .peer_identity
281            .lock()
282            .unwrap_or_else(|e| e.into_inner())
283            .clone()
284    };
285
286    let mut providers: Vec<Arc<dyn InventoryProvider>> = Vec::new();
287    let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
288
289    // The user's own devices, from the synced oplog. Authenticated by
290    // construction — the roster is readable only with their credentials.
291    for peer in crate::peers::snapshot_parslee(state).await {
292        if let car_peers::PeerAddress::A2a { base_url } = &peer.address {
293            if seen.insert(base_url.clone()) {
294                providers.push(Arc::new(inventory::PeerInventoryProvider::new(
295                    InstanceRef::remote(peer.name.clone(), "parslee", base_url.clone()),
296                    identity.clone(),
297                )));
298            }
299        }
300    }
301
302    // Peers an operator promoted through `a2a.peers.add`.
303    if let Ok(registry) = car_a2a::peers::PeerRegistry::user_default() {
304        for entry in registry.list() {
305            if seen.insert(entry.url.clone()) {
306                let name = entry.label.clone().unwrap_or_else(|| entry.slug.clone());
307                providers.push(Arc::new(inventory::PeerInventoryProvider::new(
308                    InstanceRef::remote(name, "registry", entry.url.clone()),
309                    identity.clone(),
310                )));
311            }
312        }
313    }
314
315    let visible_only = crate::peers::snapshot_lan(state)
316        .into_iter()
317        .filter_map(|peer| {
318            let car_peers::PeerAddress::A2a { base_url } = &peer.address else {
319                return None;
320            };
321            if seen.contains(base_url) {
322                return None;
323            }
324            Some(InstanceInventory::unreachable(
325                InstanceRef::remote(peer.name.clone(), "lan", base_url.clone()),
326                "discovered on the local network but not a trusted peer — anyone can advertise \
327                 any name, so promote it with `a2a.peers.add` before CAR will contact it",
328                car_fleet::now_ms(),
329            ))
330        })
331        .collect();
332
333    (providers, visible_only)
334}
335
336/// Assemble the composite: this instance plus every reachable peer.
337pub async fn composite(
338    state: &ServerState,
339    session: Option<&ClientSession>,
340    include_remote: bool,
341    timeout: std::time::Duration,
342) -> FleetComposite {
343    let local = match session {
344        Some(s) => inventory::local_inventory(state, Some(&s.runtime), Some(&s.memgine)).await,
345        None => inventory::local_inventory(state, None, None).await,
346    };
347    let self_name = local.instance.name.clone();
348
349    let mut all = vec![local];
350    if include_remote {
351        let (providers, visible_only) = remote_providers(state).await;
352        all.extend(car_fleet::gather(&providers, timeout).await);
353        all.extend(visible_only);
354    }
355    car_fleet::compose(self_name, all)
356}
357
358/// `fleet.inventory` — this instance only. The same report peers receive,
359/// plus the calling session's own tools and skills.
360pub async fn handle_fleet_inventory(
361    state: &ServerState,
362    session: &ClientSession,
363) -> Result<Value, String> {
364    let inv =
365        inventory::local_inventory(state, Some(&session.runtime), Some(&session.memgine)).await;
366    serde_json::to_value(inv).map_err(|e| e.to_string())
367}
368
369/// `fleet.composite` — every agent, capability, and model this daemon can reach.
370///
371/// Params: `{ include_remote?: bool = true, timeout_ms?: u64 }`. Remote reads
372/// are concurrent and individually bounded, so one sleeping machine costs the
373/// timeout, not the call.
374pub async fn handle_fleet_composite(
375    msg: &JsonRpcMessage,
376    state: &ServerState,
377    session: &ClientSession,
378) -> Result<Value, String> {
379    let include_remote = msg
380        .params
381        .get("include_remote")
382        .and_then(|v| v.as_bool())
383        .unwrap_or(true);
384    let timeout = msg
385        .params
386        .get("timeout_ms")
387        .and_then(|v| v.as_u64())
388        .map(std::time::Duration::from_millis)
389        .unwrap_or(INVENTORY_TIMEOUT);
390    let composite = composite(state, Some(session), include_remote, timeout).await;
391    serde_json::to_value(composite).map_err(|e| e.to_string())
392}
393
394/// `fleet.worker.get` — whether this instance takes farmed-out work.
395pub async fn handle_fleet_worker_get() -> Result<Value, String> {
396    let config = FleetWorkerConfig::load();
397    let profile = worker_profile().await;
398    Ok(serde_json::json!({
399        "config": config,
400        "profile": profile,
401    }))
402}
403
404/// `fleet.worker.set` — enroll (or withdraw) this instance as a fleet worker.
405///
406/// **Operator-only**: refused for any session bound to an agent id. Enrolling
407/// means a peer's prompt runs a coding CLI against a local checkout, which is a
408/// decision for the person at the machine — and an agent must not be able to
409/// grant it, least of all the agent that would benefit.
410///
411/// Deliberately *not* `is_host`, which the peer-approval surfaces use. Those
412/// gate an operator's arbitration of messages an agent sent; this gates a
413/// configuration change, and the operator's own CLI (which holds the daemon's
414/// auth token, and can already start a coder session on this machine) is as much
415/// the operator as the host app is. Host-only here would put the feature behind
416/// a surface the CLI cannot reach without buying any safety the token boundary
417/// does not already provide.
418pub async fn handle_fleet_worker_set(
419    msg: &JsonRpcMessage,
420    session: &ClientSession,
421) -> Result<Value, String> {
422    let bound_agent = session.agent_id.lock().await.clone();
423    if let Some(agent) = bound_agent {
424        if !session.is_host.load(std::sync::atomic::Ordering::Acquire) {
425            return Err(format!(
426                "`fleet.worker.set` is operator-only: `{agent}` cannot enroll this machine to \
427                 run peers' coding subtasks against local checkouts"
428            ));
429        }
430    }
431
432    let mut config = FleetWorkerConfig::load();
433    if let Some(v) = msg.params.get("accepts_work").and_then(|v| v.as_bool()) {
434        config.accepts_work = v;
435    }
436    if let Some(list) = msg.params.get("repos").and_then(|v| v.as_array()) {
437        let mut repos = Vec::new();
438        for entry in list {
439            let path = PathBuf::from(entry.as_str().ok_or("`repos` entries must be strings")?);
440            // Validate here rather than at dispatch time: an operator naming a
441            // directory that is not a checkout should find out now, not when a
442            // peer's subtask is declined for reasons that look like the peer's
443            // fault.
444            car_fleet::root_commit(&path)
445                .map_err(|e| format!("`{}` is not a git repository: {e}", path.display()))?;
446            repos.push(path);
447        }
448        config.repos = repos;
449    }
450    if let Some(n) = msg.params.get("max_parallel").and_then(|v| v.as_u64()) {
451        config.max_parallel = n.min(64) as u32;
452    }
453    if let Some(n) = msg.params.get("local_parallel").and_then(|v| v.as_u64()) {
454        config.local_parallel = n.min(64) as u32;
455    }
456    if let Some(n) = msg
457        .params
458        .get("dispatches_per_hour")
459        .and_then(|v| v.as_u64())
460    {
461        config.dispatches_per_hour = n.min(10_000) as u32;
462    }
463    if let Some(n) = msg.params.get("max_subtask_secs").and_then(|v| v.as_u64()) {
464        // A zero would mean "no time at all", which reads as a mistake rather
465        // than a posture; the floor keeps a typo from silently declining
466        // everything with a timeout error.
467        config.max_subtask_secs = n.clamp(60, 24 * 3600);
468    }
469    if let Some(v) = msg
470        .params
471        .get("fetch_missing_base")
472        .and_then(|v| v.as_bool())
473    {
474        config.fetch_missing_base = v;
475    }
476    if let Some(remote) = msg.params.get("fetch_remote").and_then(|v| v.as_str()) {
477        // A remote name reaches a `git fetch` argv. Keep it to the shape git
478        // itself accepts so nothing that looks like a flag or a URL can be
479        // smuggled in through a config write.
480        if remote.is_empty()
481            || remote.starts_with('-')
482            || !remote
483                .chars()
484                .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/'))
485        {
486            return Err(format!("`{remote}` is not a usable git remote name"));
487        }
488        config.fetch_remote = remote.to_string();
489    }
490    if let Some(list) = msg.params.get("allowed_tools").and_then(|v| v.as_array()) {
491        config.allowed_tools = Some(
492            list.iter()
493                .filter_map(|v| v.as_str().map(String::from))
494                .collect(),
495        );
496    }
497    config.save()?;
498    Ok(serde_json::json!({
499        "config": config,
500        "profile": worker_profile().await,
501    }))
502}
503
504/// Answers the CAR fleet extension methods on the peer-facing A2A listener.
505///
506/// Holds a weak reference to the daemon state so a stopped listener cannot keep
507/// it alive, mirroring `WsChatResponder`.
508pub struct DaemonFleetResponder {
509    state: std::sync::Weak<ServerState>,
510    /// The A2A listener's runtime — the tools a peer could actually reach here,
511    /// which is the honest answer to "what capabilities does this host offer
512    /// *you*".
513    runtime: Arc<car_engine::Runtime>,
514}
515
516impl DaemonFleetResponder {
517    pub fn new(state: std::sync::Weak<ServerState>, runtime: Arc<car_engine::Runtime>) -> Self {
518        Self { state, runtime }
519    }
520}
521
522#[async_trait::async_trait]
523impl car_a2a::FleetResponder for DaemonFleetResponder {
524    async fn inventory(&self) -> Result<Value, String> {
525        let state = self
526            .state
527            .upgrade()
528            .ok_or_else(|| "daemon is shutting down".to_string())?;
529        // No session: a peer sees this host's agents, models, and the tools its
530        // own A2A surface exposes — never another session's private skill graph.
531        let inv = inventory::local_inventory(&state, Some(&self.runtime), None).await;
532        serde_json::to_value(inv).map_err(|e| e.to_string())
533    }
534
535    async fn run_subtask(&self, dispatch: Value, caller: Option<&str>) -> Result<Value, String> {
536        // No verified caller, no work. The transport that reaches this method
537        // authenticates every request by peer signature, so `None` means
538        // something changed underneath it — and an unattributable dispatch is
539        // exactly the one that cannot be rate-limited or audited.
540        let Some(caller) = caller else {
541            return Err(
542                "refusing an unattributed fleet dispatch: this surface accepts work only from a CAR peer whose signature identifies it"
543                    .to_string(),
544            );
545        };
546        let dispatch: car_fleet::SubtaskDispatch =
547            serde_json::from_value(dispatch).map_err(|e| format!("invalid dispatch: {e}"))?;
548        let outcome = serve::run_dispatch(dispatch, caller).await?;
549        serde_json::to_value(outcome).map_err(|e| e.to_string())
550    }
551}
552
553/// Why a reachable instance is not in the pool.
554///
555/// Exhaustive and reported, because the alternative is the failure mode that
556/// kills a distributed run quietly: every peer turns out to be ineligible, the
557/// pool collapses to this host, the run completes correctly but at local speed,
558/// and nothing ever says so. The caller asked for distribution and got a slower
559/// local run that looks identical to a fast one that happened to be slow.
560#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
561#[serde(rename_all = "snake_case")]
562pub enum PoolExclusion {
563    /// Reachable, but its operator has not enrolled it as a fleet worker.
564    NotEnrolled,
565    /// Enrolled, but has no checkout of this repository.
566    RepositoryNotServed,
567    /// Eligible, but the caller's `workers` list did not name it.
568    NotRequested,
569    /// Could not be read at all — see the instance's inventory row.
570    Unreachable,
571    /// Answered, but published no address to dispatch to.
572    NoAddress,
573}
574
575impl PoolExclusion {
576    pub fn as_str(self) -> &'static str {
577        match self {
578            PoolExclusion::NotEnrolled => "not_enrolled",
579            PoolExclusion::RepositoryNotServed => "repository_not_served",
580            PoolExclusion::NotRequested => "not_requested",
581            PoolExclusion::Unreachable => "unreachable",
582            PoolExclusion::NoAddress => "no_address",
583        }
584    }
585}
586
587/// What the pool ended up containing, and what it left out.
588#[derive(Debug, Clone, Default, Serialize, Deserialize)]
589pub struct PoolPlan {
590    /// Peer instances actually in the pool. Empty means the run is distributed
591    /// in name only.
592    pub remote_workers: Vec<String>,
593    /// Every other reachable instance, with the reason it is not a worker.
594    pub excluded: Vec<(String, String)>,
595}
596
597impl PoolPlan {
598    /// Whether a run asked to be distributed will in fact run only here.
599    pub fn local_only(&self) -> bool {
600        self.remote_workers.is_empty()
601    }
602
603    /// One line a person can act on, or `None` when peers are in the pool.
604    pub fn degraded_reason(&self) -> Option<String> {
605        if !self.local_only() {
606            return None;
607        }
608        if self.excluded.is_empty() {
609            return Some(
610                "no other CAR instance is reachable, so this ran on one machine".to_string(),
611            );
612        }
613        let mut counts: std::collections::BTreeMap<&str, usize> = std::collections::BTreeMap::new();
614        for (_, reason) in &self.excluded {
615            *counts.entry(reason.as_str()).or_default() += 1;
616        }
617        let detail = counts
618            .into_iter()
619            .map(|(reason, n)| format!("{n} {reason}"))
620            .collect::<Vec<_>>()
621            .join(", ");
622        Some(format!(
623            "no peer could take a subtask, so this ran on one machine ({detail})"
624        ))
625    }
626}
627
628/// Build the worker pool for a distributed Foreman run.
629///
630/// Always includes this host, so a run whose peers all decline still completes
631/// locally rather than failing — distribution is an optimization, and an
632/// optimization that can fail the job is a liability. Returns the pool and the
633/// worker ids in it, for the run report.
634///
635/// `only` restricts placement to the named instances (the caller's
636/// `workers: [...]` parameter); `None` uses every instance that reports it can
637/// serve this repository.
638pub async fn build_pool(
639    state: &ServerState,
640    repo_root: &std::path::Path,
641    run_id: &str,
642    adapter: &str,
643    only: Option<&[String]>,
644) -> Result<(car_multi::FleetPool, PoolPlan), String> {
645    let fingerprint = car_fleet::read_fingerprint(repo_root).map_err(|e| {
646        format!(
647            "cannot identify the repository at {}: {e}",
648            repo_root.display()
649        )
650    })?;
651    let config = FleetWorkerConfig::load();
652
653    let local: Arc<dyn car_multi::WorktreeAgent> = Arc::new(
654        car_external_agents::ForemanExternalAgent::new(adapter.to_string()),
655    );
656    let mut workers = vec![car_multi::FleetWorker::local(
657        car_a2a::lan::host_label(),
658        local,
659        config.local_parallel.max(1) as usize,
660    )];
661
662    let composite = composite(state, None, true, INVENTORY_TIMEOUT).await;
663    let identity = {
664        state
665            .peer_identity
666            .lock()
667            .unwrap_or_else(|e| e.into_inner())
668            .clone()
669    };
670
671    // Classify every reachable peer, in or out, so the caller is told when a
672    // "distributed" run has quietly become a local one.
673    let mut plan = PoolPlan::default();
674    let eligible: std::collections::HashSet<&str> = composite
675        .workers_for(&fingerprint.root_commit)
676        .into_iter()
677        .map(|c| c.instance.as_str())
678        .collect();
679    for inv in &composite.instances {
680        if inv.instance.kind == car_fleet::InstanceKind::Local {
681            continue;
682        }
683        let name = inv.instance.name.clone();
684        let reason = if !inv.reachable() {
685            PoolExclusion::Unreachable
686        } else if !eligible.contains(name.as_str()) {
687            // Enrolled-but-wrong-repo and not-enrolled-at-all are different
688            // problems with different fixes, so they are not collapsed.
689            match &inv.worker {
690                Some(w) if w.accepts_work => PoolExclusion::RepositoryNotServed,
691                _ => PoolExclusion::NotEnrolled,
692            }
693        } else if only.is_some_and(|only| !only.iter().any(|n| n == &name)) {
694            PoolExclusion::NotRequested
695        } else if inv.instance.base_url.is_none() {
696            PoolExclusion::NoAddress
697        } else {
698            continue;
699        };
700        plan.excluded.push((name, reason.as_str().to_string()));
701    }
702
703    for candidate in composite.workers_for(&fingerprint.root_commit) {
704        if candidate.kind == car_fleet::InstanceKind::Local {
705            continue;
706        }
707        if let Some(only) = only {
708            if !only.iter().any(|n| n == &candidate.instance) {
709                continue;
710            }
711        }
712        // The peer's own inventory row carries the URL it answered on.
713        let Some(base_url) = composite
714            .instances
715            .iter()
716            .find(|i| i.instance.name == candidate.instance)
717            .and_then(|i| i.instance.base_url.clone())
718        else {
719            continue;
720        };
721        // Ask for the same adapter the local run uses when the peer has it;
722        // otherwise let the peer choose, rather than declining a machine that
723        // could still help.
724        let wanted = candidate
725            .adapters
726            .iter()
727            .any(|a| a == adapter)
728            .then(|| adapter.to_string());
729        let agent = RemoteWorktreeAgent::new(
730            candidate.instance.clone(),
731            base_url,
732            identity.clone(),
733            fingerprint.clone(),
734            run_id,
735        )
736        .with_adapter(wanted);
737        plan.remote_workers.push(candidate.instance.clone());
738        workers.push(car_multi::FleetWorker::remote(
739            candidate.instance.clone(),
740            Arc::new(agent),
741            candidate.max_parallel.max(1) as usize,
742        ));
743    }
744
745    if let Some(reason) = plan.degraded_reason() {
746        tracing::warn!(%reason, "distributed foreman run has no peer workers");
747    }
748    Ok((car_multi::FleetPool::new(workers), plan))
749}
750
751/// Render a placement ledger for a run report.
752///
753/// Takes the ledger rather than the pool: `placements()` clones the whole vec
754/// under a mutex, so a caller that already holds one would otherwise pay for a
755/// second, non-atomic read of the same thing.
756///
757/// The field names live on `Placement`/`FailedAttempt` as serde renames, which
758/// this crate does not own — see their doc comments and the round-trip test that
759/// pins them (car#1322).
760pub fn placements_value(placements: &[car_multi::Placement]) -> Value {
761    // Infallible: plain structs of `String`/`bool`/`Vec`, no map with non-string
762    // keys and no custom `Serialize`. A silent `null` would be the wrong answer
763    // for a receipt anyway.
764    serde_json::to_value(placements).expect("placement ledger is plain data")
765}
766
767/// [`placements_value`] for a caller holding the pool.
768pub fn placements_json(pool: &car_multi::FleetPool) -> Value {
769    placements_value(&pool.placements())
770}
771
772#[cfg(test)]
773mod tests {
774    use super::*;
775
776    #[test]
777    fn a_distributed_run_with_no_peers_says_why_rather_than_going_quiet() {
778        let plan = PoolPlan {
779            remote_workers: Vec::new(),
780            excluded: vec![
781                ("studio".into(), "not_enrolled".into()),
782                ("laptop".into(), "not_enrolled".into()),
783                ("ci-box".into(), "repository_not_served".into()),
784            ],
785        };
786        assert!(plan.local_only());
787        let reason = plan.degraded_reason().expect("degraded");
788        assert!(reason.contains("2 not_enrolled"), "{reason}");
789        assert!(reason.contains("1 repository_not_served"), "{reason}");
790    }
791
792    #[test]
793    fn a_pool_with_peers_reports_no_degradation() {
794        let plan = PoolPlan {
795            remote_workers: vec!["studio".into()],
796            excluded: vec![("laptop".into(), "not_enrolled".into())],
797        };
798        assert!(!plan.local_only());
799        assert!(plan.degraded_reason().is_none());
800    }
801
802    #[test]
803    fn no_reachable_peers_at_all_is_its_own_message() {
804        let plan = PoolPlan::default();
805        let reason = plan.degraded_reason().expect("degraded");
806        assert!(
807            reason.contains("no other CAR instance is reachable"),
808            "{reason}"
809        );
810    }
811
812    #[test]
813    fn the_shipped_default_declines_work() {
814        // The one default that must never drift: an instance nobody configured
815        // does not run other machines' prompts.
816        let c = FleetWorkerConfig::default();
817        assert!(!c.accepts_work);
818        assert!(c.repos.is_empty());
819    }
820
821    #[test]
822    fn a_laptop_does_not_fetch_unless_told_to() {
823        // Fetching is the runner posture. On a machine someone is working at,
824        // a peer's dispatch should not reach into a repository and pull.
825        let c = FleetWorkerConfig::default();
826        assert!(!c.fetch_missing_base);
827        assert_eq!(c.fetch_remote, "origin");
828    }
829
830    #[test]
831    fn a_config_written_before_runner_mode_existed_still_declines() {
832        let parsed: FleetWorkerConfig =
833            serde_json::from_str("{\"accepts_work\": true}").expect("parses");
834        assert!(
835            !parsed.fetch_missing_base,
836            "absent must not read as enabled"
837        );
838        assert_eq!(parsed.fetch_remote, "origin");
839    }
840
841    #[test]
842    fn the_sender_does_not_choose_this_machines_limits() {
843        // Every field here bounds what a peer may spend, so each one must have
844        // a default — a config written before they existed must not read as
845        // "unlimited".
846        let parsed: FleetWorkerConfig =
847            serde_json::from_str("{\"accepts_work\": true, \"repos\": []}").expect("parses");
848        assert_eq!(
849            parsed.dispatches_per_hour,
850            car_fleet::DEFAULT_DISPATCHES_PER_WINDOW
851        );
852        assert_eq!(parsed.max_subtask_secs, DEFAULT_MAX_SUBTASK_SECS);
853        assert!(parsed.allowed_tools.is_none());
854    }
855
856    #[test]
857    fn an_unparseable_config_declines_rather_than_half_accepting() {
858        let parsed: FleetWorkerConfig =
859            serde_json::from_str("{\"accepts_work\": true}").expect("partial config parses");
860        assert!(parsed.accepts_work);
861        assert_eq!(parsed.max_parallel, DEFAULT_MAX_PARALLEL, "limits default");
862        assert!(
863            parsed.repos.is_empty(),
864            "and with no repos it can still serve nothing"
865        );
866    }
867
868    #[test]
869    fn worker_worktrees_live_outside_every_served_repository() {
870        let base = worktree_base();
871        assert!(
872            base.ends_with("fleet-worker/worktrees") || base.ends_with("fleet-worker\\worktrees")
873        );
874    }
875}