Skip to main content

glass_dev/
agents.rs

1//! Glass-owned scheduling for independent resident Pi agent sessions.
2
3use crate::development::{DevelopmentError, DevelopmentResult};
4use crate::pi_runtime::{GlassPiRuntime, PiRuntimeOptions, PiSessionRequest};
5use serde::{Deserialize, Serialize};
6use serde_json::Value;
7use std::collections::{BTreeMap, BTreeSet, VecDeque};
8use std::path::{Path, PathBuf};
9use std::sync::Arc;
10use std::sync::atomic::{AtomicU64, Ordering};
11use std::sync::mpsc::{self, Receiver, SyncSender, TryRecvError, TrySendError};
12use std::thread;
13use std::time::{Duration, SystemTime, UNIX_EPOCH};
14
15const COMMAND_CAPACITY: usize = 32;
16const EVENT_CAPACITY: usize = 256;
17const HISTORY_CAPACITY: usize = 512;
18const AGENT_EVIDENCE_CAPACITY: usize = 64;
19const MAX_AGENTS: usize = 32;
20const MAX_PROMPT_BYTES: usize = 64 * 1024;
21
22#[derive(Debug, Clone)]
23pub struct ResidentAgentBroker {
24    pub socket: PathBuf,
25    pub token: String,
26    pub workspace_id: String,
27}
28
29#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
30pub struct AgentId(String);
31
32impl AgentId {
33    pub fn as_str(&self) -> &str {
34        &self.0
35    }
36
37    pub fn parse(value: impl Into<String>) -> DevelopmentResult<Self> {
38        let value = value.into();
39        if !value.starts_with("agent-")
40            || value.len() > 64
41            || value.chars().any(|character| character.is_control())
42        {
43            return Err(DevelopmentError::InvalidInput("invalid agent id".into()));
44        }
45        Ok(Self(value))
46    }
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
50#[serde(rename_all = "camelCase")]
51pub enum AgentStatus {
52    Queued,
53    Starting,
54    Idle,
55    Working,
56    Waiting,
57    Completed,
58    Failed,
59    Cancelled,
60}
61
62impl AgentStatus {
63    fn terminal(self) -> bool {
64        matches!(self, Self::Completed | Self::Failed | Self::Cancelled)
65    }
66}
67
68#[derive(Debug, Clone, Serialize, Deserialize)]
69#[serde(rename_all = "camelCase")]
70pub struct AgentSpec {
71    pub role: String,
72    pub task: String,
73    #[serde(default)]
74    pub dependencies: Vec<AgentId>,
75    pub model: Option<String>,
76    pub thinking: Option<String>,
77    pub worktree: Option<PathBuf>,
78    #[serde(default)]
79    pub unrestricted: bool,
80    pub max_runtime_seconds: Option<u64>,
81    pub max_events: Option<u64>,
82}
83
84impl AgentSpec {
85    pub fn new(role: impl Into<String>, task: impl Into<String>) -> Self {
86        Self {
87            role: role.into(),
88            task: task.into(),
89            dependencies: Vec::new(),
90            model: None,
91            thinking: None,
92            worktree: None,
93            unrestricted: false,
94            max_runtime_seconds: Some(3_600),
95            max_events: Some(10_000),
96        }
97    }
98}
99
100#[derive(Debug, Clone, Serialize, Deserialize)]
101#[serde(rename_all = "camelCase")]
102pub struct AgentSnapshot {
103    pub id: AgentId,
104    pub role: String,
105    pub task: String,
106    pub status: AgentStatus,
107    pub dependencies: Vec<AgentId>,
108    pub model: Option<String>,
109    pub thinking: Option<String>,
110    pub worktree: PathBuf,
111    pub unrestricted: bool,
112    pub created_at_ms: u128,
113    pub started_at_ms: Option<u128>,
114    pub updated_at_ms: u128,
115    pub event_count: u64,
116    pub dropped_event_count: u64,
117    pub last_error: Option<String>,
118    pub last_response_id: Option<String>,
119    pub evidence: Vec<Value>,
120}
121
122#[derive(Debug, Clone, Serialize, Deserialize)]
123#[serde(rename_all = "camelCase")]
124pub struct AgentEvent {
125    pub sequence: u64,
126    pub agent_id: AgentId,
127    pub timestamp_ms: u128,
128    pub kind: String,
129    pub payload: Value,
130}
131
132#[derive(Debug)]
133enum WorkerCommand {
134    Request(PiSessionRequest),
135    Shutdown,
136}
137
138#[derive(Debug)]
139enum WorkerEvent {
140    Ready(AgentId),
141    RequestStarted(AgentId, String, bool),
142    Pi(AgentId, Value),
143    Failed(AgentId, String),
144    Stopped(AgentId),
145}
146
147struct AgentRecord {
148    snapshot: AgentSnapshot,
149    spec: AgentSpec,
150    command: Option<SyncSender<WorkerCommand>>,
151    worker: Option<thread::JoinHandle<()>>,
152    awaiting_agent_settle: bool,
153    dropped_events: Arc<AtomicU64>,
154}
155
156struct WorkerRuntime {
157    worktree: PathBuf,
158    sessions_dir: PathBuf,
159    broker: Option<ResidentAgentBroker>,
160    additional_system_prompt: Option<String>,
161}
162
163/// Registry and scheduler for independent Pi subprocesses.
164///
165/// Each running agent owns a distinct Pi process and persistent session. The
166/// registry only starts dependency-ready work, bounds all queues and retained
167/// evidence, and never silently promotes a failed dependency to success.
168pub struct AgentRegistry {
169    root: PathBuf,
170    sessions_dir: PathBuf,
171    records: BTreeMap<AgentId, AgentRecord>,
172    events_tx: SyncSender<WorkerEvent>,
173    events_rx: Receiver<WorkerEvent>,
174    history: VecDeque<AgentEvent>,
175    next_agent: u64,
176    next_event: u64,
177    broker: Option<ResidentAgentBroker>,
178    additional_system_prompt: Option<String>,
179    default_model: Option<String>,
180    default_thinking: Option<String>,
181}
182
183impl AgentRegistry {
184    pub fn new(root: impl AsRef<Path>) -> DevelopmentResult<Self> {
185        let root = std::fs::canonicalize(root)?;
186        let sessions_dir = root.join(".glass").join("pi-sessions");
187        let (events_tx, events_rx) = mpsc::sync_channel(EVENT_CAPACITY);
188        Ok(Self {
189            root,
190            sessions_dir,
191            records: BTreeMap::new(),
192            events_tx,
193            events_rx,
194            history: VecDeque::new(),
195            next_agent: 1,
196            next_event: 1,
197            broker: None,
198            additional_system_prompt: None,
199            default_model: None,
200            default_thinking: None,
201        })
202    }
203
204    pub fn set_resident_broker(&mut self, broker: ResidentAgentBroker) -> DevelopmentResult<()> {
205        if !broker.socket.is_absolute()
206            || broker.socket == Path::new("/")
207            || broker.token.is_empty()
208            || broker.token.len() > 256
209            || broker.workspace_id.is_empty()
210            || broker.workspace_id.len() > 128
211            || broker
212                .workspace_id
213                .chars()
214                .any(|character| character.is_control())
215        {
216            return Err(DevelopmentError::InvalidInput(
217                "invalid resident agent broker context".into(),
218            ));
219        }
220        self.broker = Some(broker);
221        Ok(())
222    }
223
224    pub fn set_additional_system_prompt(
225        &mut self,
226        prompt: Option<String>,
227    ) -> DevelopmentResult<()> {
228        if prompt
229            .as_ref()
230            .is_some_and(|prompt| prompt.len() > 128 * 1024 || prompt.contains('\0'))
231        {
232            return Err(DevelopmentError::InvalidInput(
233                "agent project instructions exceed 128 KiB or contain NUL".into(),
234            ));
235        }
236        self.additional_system_prompt = prompt;
237        Ok(())
238    }
239
240    /// Active user-global and trusted-project instructions supplied to Pi.
241    pub fn additional_system_prompt(&self) -> Option<&str> {
242        self.additional_system_prompt.as_deref()
243    }
244
245    pub fn set_defaults(
246        &mut self,
247        model: Option<String>,
248        thinking: Option<String>,
249    ) -> DevelopmentResult<()> {
250        for (label, value) in [("model", &model), ("thinking", &thinking)] {
251            if value
252                .as_ref()
253                .is_some_and(|value| value.is_empty() || value.len() > 256 || value.contains('\0'))
254            {
255                return Err(DevelopmentError::InvalidInput(format!(
256                    "default agent {label} must contain 1..=256 bytes without NUL"
257                )));
258            }
259        }
260        self.default_model = model;
261        self.default_thinking = thinking;
262        Ok(())
263    }
264
265    pub fn create(&mut self, mut spec: AgentSpec) -> DevelopmentResult<AgentId> {
266        self.refresh()?;
267        if spec.model.is_none() {
268            spec.model.clone_from(&self.default_model);
269        }
270        if spec.thinking.is_none() {
271            spec.thinking.clone_from(&self.default_thinking);
272        }
273        validate_spec(&self.root, &self.records, &spec)?;
274        if self.records.len() >= MAX_AGENTS {
275            return Err(DevelopmentError::Conflict(format!(
276                "agent registry reached its {MAX_AGENTS} session limit"
277            )));
278        }
279        let id = AgentId(format!("agent-{:04}", self.next_agent));
280        self.next_agent = self
281            .next_agent
282            .checked_add(1)
283            .ok_or_else(|| DevelopmentError::Conflict("agent identifier space exhausted".into()))?;
284        let now = now_ms();
285        let worktree = spec.worktree.clone().unwrap_or_else(|| self.root.clone());
286        let status = if spec.dependencies.is_empty() {
287            AgentStatus::Starting
288        } else {
289            AgentStatus::Queued
290        };
291        self.records.insert(
292            id.clone(),
293            AgentRecord {
294                snapshot: AgentSnapshot {
295                    id: id.clone(),
296                    role: spec.role.clone(),
297                    task: spec.task.clone(),
298                    status,
299                    dependencies: spec.dependencies.clone(),
300                    model: spec.model.clone(),
301                    thinking: spec.thinking.clone(),
302                    worktree,
303                    unrestricted: spec.unrestricted,
304                    created_at_ms: now,
305                    started_at_ms: None,
306                    updated_at_ms: now,
307                    event_count: 0,
308                    dropped_event_count: 0,
309                    last_error: None,
310                    last_response_id: None,
311                    evidence: Vec::new(),
312                },
313                spec,
314                command: None,
315                worker: None,
316                awaiting_agent_settle: false,
317                dropped_events: Arc::new(AtomicU64::new(0)),
318            },
319        );
320        if status == AgentStatus::Starting {
321            self.spawn(&id)?;
322        }
323        Ok(id)
324    }
325
326    pub fn refresh(&mut self) -> DevelopmentResult<()> {
327        loop {
328            match self.events_rx.try_recv() {
329                Ok(event) => self.apply_worker_event(event),
330                Err(TryRecvError::Empty) => break,
331                Err(TryRecvError::Disconnected) => {
332                    return Err(DevelopmentError::Process(
333                        "agent scheduler event channel closed".into(),
334                    ));
335                }
336            }
337        }
338        for record in self.records.values_mut() {
339            record.snapshot.dropped_event_count = record.dropped_events.load(Ordering::Relaxed);
340        }
341        self.reap_finished_workers();
342        self.enforce_budgets();
343        self.fail_blocked_dependents();
344        self.start_ready_agents()?;
345        Ok(())
346    }
347
348    pub fn snapshot(&mut self, id: &AgentId) -> DevelopmentResult<AgentSnapshot> {
349        self.refresh()?;
350        self.records
351            .get(id)
352            .map(|record| record.snapshot.clone())
353            .ok_or_else(|| DevelopmentError::NotFound(format!("agent {}", id.as_str())))
354    }
355
356    pub fn list(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>> {
357        self.refresh()?;
358        Ok(self
359            .records
360            .values()
361            .map(|record| record.snapshot.clone())
362            .collect())
363    }
364
365    pub fn history(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>> {
366        self.refresh()?;
367        Ok(self
368            .history
369            .iter()
370            .filter(|event| event.sequence > since)
371            .cloned()
372            .collect())
373    }
374
375    pub fn prompt(&mut self, id: &AgentId, text: impl Into<String>) -> DevelopmentResult<()> {
376        let text = text.into();
377        validate_text("agent prompt", &text, MAX_PROMPT_BYTES)?;
378        self.send(id, PiSessionRequest::Prompt { text })
379    }
380
381    pub fn steer(&mut self, id: &AgentId, text: impl Into<String>) -> DevelopmentResult<()> {
382        let text = text.into();
383        validate_text("agent steering", &text, MAX_PROMPT_BYTES)?;
384        self.send(id, PiSessionRequest::Steer { text })
385    }
386
387    pub fn follow_up(&mut self, id: &AgentId, text: impl Into<String>) -> DevelopmentResult<()> {
388        let text = text.into();
389        validate_text("agent follow-up", &text, MAX_PROMPT_BYTES)?;
390        self.send(id, PiSessionRequest::FollowUp { text })
391    }
392
393    pub fn request(&mut self, id: &AgentId, request: PiSessionRequest) -> DevelopmentResult<()> {
394        self.send(id, request)
395    }
396
397    pub fn complete(&mut self, id: &AgentId) -> DevelopmentResult<()> {
398        self.refresh()?;
399        if self.record_mut(id)?.snapshot.status.terminal() {
400            return Err(DevelopmentError::Conflict(format!(
401                "agent {} is already terminal",
402                id.as_str()
403            )));
404        }
405        if let Some(sender) = self.record_mut(id)?.command.take() {
406            let _ = sender.try_send(WorkerCommand::Shutdown);
407        }
408        self.join_worker(id, AgentStatus::Completed)?;
409        self.record_event(id, "completed", Value::Null);
410        self.start_ready_agents()?;
411        Ok(())
412    }
413
414    pub fn cancel(&mut self, id: &AgentId) -> DevelopmentResult<()> {
415        self.refresh()?;
416        if self.record_mut(id)?.snapshot.status.terminal() {
417            return Ok(());
418        }
419        if let Some(sender) = &self.record_mut(id)?.command {
420            let _ = sender.try_send(WorkerCommand::Request(PiSessionRequest::Abort));
421            let _ = sender.try_send(WorkerCommand::Shutdown);
422        }
423        self.record_mut(id)?.command = None;
424        self.join_worker(id, AgentStatus::Cancelled)?;
425        self.record_event(id, "cancelled", Value::Null);
426        self.fail_blocked_dependents();
427        Ok(())
428    }
429
430    fn send(&mut self, id: &AgentId, request: PiSessionRequest) -> DevelopmentResult<()> {
431        self.refresh()?;
432        let record = self.record_mut(id)?;
433        if record.snapshot.status.terminal() || record.snapshot.status == AgentStatus::Queued {
434            return Err(DevelopmentError::Conflict(format!(
435                "agent {} cannot accept requests while {:?}",
436                id.as_str(),
437                record.snapshot.status
438            )));
439        }
440        let sender = record.command.as_ref().ok_or_else(|| {
441            DevelopmentError::Conflict(format!("agent {} is still starting", id.as_str()))
442        })?;
443        sender
444            .try_send(WorkerCommand::Request(request))
445            .map_err(|error| match error {
446                TrySendError::Full(_) => DevelopmentError::Conflict(format!(
447                    "agent {} command queue is full",
448                    id.as_str()
449                )),
450                TrySendError::Disconnected(_) => DevelopmentError::Process(format!(
451                    "agent {} command channel closed",
452                    id.as_str()
453                )),
454            })?;
455        Ok(())
456    }
457
458    fn spawn(&mut self, id: &AgentId) -> DevelopmentResult<()> {
459        let sessions_dir = self.sessions_dir.clone();
460        let events = self.events_tx.clone();
461        let broker = self.broker.clone();
462        let additional_system_prompt = self.additional_system_prompt.clone();
463        let record = self.record_mut(id)?;
464        let dropped_events = Arc::clone(&record.dropped_events);
465        let spec = record.spec.clone();
466        let worktree = record.snapshot.worktree.clone();
467        let worker_id = id.clone();
468        let (commands_tx, commands_rx) = mpsc::sync_channel(COMMAND_CAPACITY);
469        record.command = Some(commands_tx);
470        record.snapshot.status = AgentStatus::Starting;
471        record.snapshot.started_at_ms = Some(now_ms());
472        record.snapshot.updated_at_ms = now_ms();
473        let worker = thread::Builder::new()
474            .name(format!("glass-{}", id.as_str()))
475            .spawn(move || {
476                run_worker(
477                    worker_id,
478                    spec,
479                    WorkerRuntime {
480                        worktree,
481                        sessions_dir,
482                        broker,
483                        additional_system_prompt,
484                    },
485                    commands_rx,
486                    events,
487                    dropped_events,
488                )
489            })
490            .map_err(DevelopmentError::Io)?;
491        record.worker = Some(worker);
492        self.record_event(id, "starting", Value::Null);
493        Ok(())
494    }
495
496    fn start_ready_agents(&mut self) -> DevelopmentResult<()> {
497        let ready: Vec<_> = self
498            .records
499            .iter()
500            .filter(|(_, record)| record.snapshot.status == AgentStatus::Queued)
501            .filter(|(_, record)| {
502                record.spec.dependencies.iter().all(|dependency| {
503                    self.records.get(dependency).is_some_and(|dependency| {
504                        dependency.snapshot.status == AgentStatus::Completed
505                    })
506                })
507            })
508            .map(|(id, _)| id.clone())
509            .collect();
510        for id in ready {
511            self.spawn(&id)?;
512        }
513        Ok(())
514    }
515
516    fn fail_blocked_dependents(&mut self) {
517        let terminal_failures: BTreeSet<_> = self
518            .records
519            .iter()
520            .filter(|(_, record)| {
521                matches!(
522                    record.snapshot.status,
523                    AgentStatus::Failed | AgentStatus::Cancelled
524                )
525            })
526            .map(|(id, _)| id.clone())
527            .collect();
528        let blocked: Vec<_> = self
529            .records
530            .iter()
531            .filter(|(_, record)| record.snapshot.status == AgentStatus::Queued)
532            .filter(|(_, record)| {
533                record
534                    .spec
535                    .dependencies
536                    .iter()
537                    .any(|dependency| terminal_failures.contains(dependency))
538            })
539            .map(|(id, _)| id.clone())
540            .collect();
541        for id in blocked {
542            if let Some(record) = self.records.get_mut(&id) {
543                record.snapshot.status = AgentStatus::Failed;
544                record.snapshot.last_error = Some("a required agent did not complete".into());
545                record.snapshot.updated_at_ms = now_ms();
546            }
547            self.record_event(
548                &id,
549                "dependencyFailed",
550                Value::String("a required agent did not complete".into()),
551            );
552        }
553    }
554
555    fn enforce_budgets(&mut self) {
556        let now = now_ms();
557        let exceeded: Vec<_> = self
558            .records
559            .iter()
560            .filter(|(_, record)| !record.snapshot.status.terminal())
561            .filter(|(_, record)| {
562                record
563                    .spec
564                    .max_runtime_seconds
565                    .zip(record.snapshot.started_at_ms)
566                    .is_some_and(|(seconds, started)| {
567                        now.saturating_sub(started) > u128::from(seconds) * 1_000
568                    })
569                    || record
570                        .spec
571                        .max_events
572                        .is_some_and(|limit| record.snapshot.event_count >= limit)
573            })
574            .map(|(id, _)| id.clone())
575            .collect();
576        for id in exceeded {
577            if let Some(record) = self.records.get_mut(&id) {
578                if let Some(sender) = record.command.take() {
579                    let _ = sender.try_send(WorkerCommand::Request(PiSessionRequest::Abort));
580                    let _ = sender.try_send(WorkerCommand::Shutdown);
581                }
582                record.snapshot.status = AgentStatus::Failed;
583                record.snapshot.last_error = Some("agent budget exceeded".into());
584                record.snapshot.updated_at_ms = now;
585            }
586            self.record_event(&id, "budgetExceeded", Value::Null);
587        }
588    }
589
590    fn apply_worker_event(&mut self, event: WorkerEvent) {
591        match event {
592            WorkerEvent::Ready(id) => {
593                if let Some(record) = self.records.get_mut(&id)
594                    && !record.snapshot.status.terminal()
595                {
596                    record.snapshot.status = AgentStatus::Idle;
597                    record.snapshot.updated_at_ms = now_ms();
598                }
599                self.record_event(&id, "ready", Value::Null);
600            }
601            WorkerEvent::RequestStarted(id, request_id, waits_for_agent) => {
602                if let Some(record) = self.records.get_mut(&id)
603                    && !record.snapshot.status.terminal()
604                {
605                    record.snapshot.last_response_id = Some(request_id.clone());
606                    record.snapshot.status = AgentStatus::Working;
607                    record.awaiting_agent_settle |= waits_for_agent;
608                    record.snapshot.updated_at_ms = now_ms();
609                }
610                self.record_event(&id, "requestStarted", Value::String(request_id));
611            }
612            WorkerEvent::Pi(id, value) => {
613                if let Some(record) = self.records.get_mut(&id) {
614                    let event_type = value.get("type").and_then(Value::as_str);
615                    if event_type == Some("agent_settled") && !record.snapshot.status.terminal() {
616                        record.awaiting_agent_settle = false;
617                        record.snapshot.status = AgentStatus::Idle;
618                    }
619                    if event_type == Some("response")
620                        && !record.awaiting_agent_settle
621                        && !record.snapshot.status.terminal()
622                    {
623                        record.snapshot.status = AgentStatus::Idle;
624                    }
625                    record.snapshot.event_count = record.snapshot.event_count.saturating_add(1);
626                    record.snapshot.updated_at_ms = now_ms();
627                    if record.snapshot.evidence.len() == AGENT_EVIDENCE_CAPACITY {
628                        record.snapshot.evidence.remove(0);
629                    }
630                    record.snapshot.evidence.push(value.clone());
631                }
632                let kind = value
633                    .get("type")
634                    .and_then(Value::as_str)
635                    .unwrap_or("piEvent")
636                    .to_string();
637                self.record_event(&id, &kind, value);
638            }
639            WorkerEvent::Failed(id, error) => {
640                if let Some(record) = self.records.get_mut(&id)
641                    && !record.snapshot.status.terminal()
642                {
643                    record.snapshot.status = AgentStatus::Failed;
644                    record.snapshot.last_error = Some(error.clone());
645                    record.snapshot.updated_at_ms = now_ms();
646                    record.command = None;
647                }
648                self.record_event(&id, "failed", Value::String(error));
649            }
650            WorkerEvent::Stopped(id) => {
651                if let Some(record) = self.records.get_mut(&id) {
652                    record.command = None;
653                }
654                self.record_event(&id, "stopped", Value::Null);
655            }
656        }
657    }
658
659    fn join_worker(&mut self, id: &AgentId, terminal: AgentStatus) -> DevelopmentResult<()> {
660        let worker = self.record_mut(id)?.worker.take();
661        if let Some(worker) = worker
662            && worker.join().is_err()
663        {
664            let error = format!("agent {} worker panicked", id.as_str());
665            let record = self.record_mut(id)?;
666            record.command = None;
667            record.snapshot.status = AgentStatus::Failed;
668            record.snapshot.last_error = Some(error.clone());
669            record.snapshot.updated_at_ms = now_ms();
670            self.record_event(
671                id,
672                "workerPanicked",
673                serde_json::json!({"agentId":id.as_str(),"error":error}),
674            );
675            return Err(DevelopmentError::Process(error));
676        }
677        let record = self.record_mut(id)?;
678        record.command = None;
679        record.snapshot.status = terminal;
680        record.snapshot.updated_at_ms = now_ms();
681        Ok(())
682    }
683
684    fn reap_finished_workers(&mut self) {
685        let finished = self
686            .records
687            .iter()
688            .filter(|(_, record)| {
689                record
690                    .worker
691                    .as_ref()
692                    .is_some_and(|worker| worker.is_finished())
693            })
694            .map(|(id, _)| id.clone())
695            .collect::<Vec<_>>();
696        for id in finished {
697            let worker = self
698                .records
699                .get_mut(&id)
700                .and_then(|record| record.worker.take());
701            let panicked = worker.is_some_and(|worker| worker.join().is_err());
702            let mut event = None;
703            if let Some(record) = self.records.get_mut(&id) {
704                record.command = None;
705                if panicked {
706                    let error = format!("agent {} worker panicked", id.as_str());
707                    record.snapshot.status = AgentStatus::Failed;
708                    record.snapshot.last_error = Some(error.clone());
709                    record.snapshot.updated_at_ms = now_ms();
710                    event = Some((
711                        "workerPanicked",
712                        serde_json::json!({
713                            "agentId":id.as_str(),
714                            "role":record.snapshot.role.clone(),
715                            "task":record.snapshot.task.clone(),
716                            "error":error,
717                        }),
718                    ));
719                } else if !record.snapshot.status.terminal() {
720                    let error = format!(
721                        "agent {} worker exited before a terminal outcome",
722                        id.as_str()
723                    );
724                    record.snapshot.status = AgentStatus::Failed;
725                    record.snapshot.last_error = Some(error.clone());
726                    record.snapshot.updated_at_ms = now_ms();
727                    event = Some(("workerExited", Value::String(error)));
728                }
729            }
730            if let Some((kind, payload)) = event {
731                self.record_event(&id, kind, payload);
732            }
733        }
734    }
735
736    fn record_event(&mut self, id: &AgentId, kind: &str, payload: Value) {
737        if self.history.len() == HISTORY_CAPACITY {
738            self.history.pop_front();
739        }
740        self.history.push_back(AgentEvent {
741            sequence: self.next_event,
742            agent_id: id.clone(),
743            timestamp_ms: now_ms(),
744            kind: kind.into(),
745            payload,
746        });
747        self.next_event = self.next_event.saturating_add(1);
748    }
749
750    fn record_mut(&mut self, id: &AgentId) -> DevelopmentResult<&mut AgentRecord> {
751        self.records
752            .get_mut(id)
753            .ok_or_else(|| DevelopmentError::NotFound(format!("agent {}", id.as_str())))
754    }
755}
756
757impl Drop for AgentRegistry {
758    fn drop(&mut self) {
759        let mut workers = Vec::new();
760        for (id, record) in &mut self.records {
761            if let Some(sender) = record.command.take() {
762                let _ = sender.try_send(WorkerCommand::Shutdown);
763            }
764            if let Some(worker) = record.worker.take() {
765                workers.push((id.clone(), worker));
766            }
767        }
768        for (id, worker) in workers {
769            if worker.join().is_err() {
770                tracing::error!(agent_id = %id.as_str(), "agent worker panicked during registry drop");
771            }
772        }
773    }
774}
775
776fn run_worker(
777    id: AgentId,
778    spec: AgentSpec,
779    runtime: WorkerRuntime,
780    commands: Receiver<WorkerCommand>,
781    events: SyncSender<WorkerEvent>,
782    dropped_events: Arc<AtomicU64>,
783) {
784    let options = PiRuntimeOptions {
785        unrestricted: spec.unrestricted,
786        session_dir: runtime.sessions_dir,
787        name: Some(format!("{}: {}", spec.role, id.as_str())),
788        model: spec.model,
789        thinking: spec.thinking,
790        broker: runtime.broker,
791        additional_system_prompt: runtime.additional_system_prompt,
792        resume: false,
793    };
794    let mut harness = match GlassPiRuntime::spawn(&runtime.worktree, options) {
795        Ok(harness) => harness,
796        Err(error) => {
797            send_critical_worker_event(&events, WorkerEvent::Failed(id, error.to_string()));
798            return;
799        }
800    };
801    send_critical_worker_event(&events, WorkerEvent::Ready(id.clone()));
802    loop {
803        loop {
804            match commands.try_recv() {
805                Ok(WorkerCommand::Request(request)) => {
806                    let waits_for_agent = matches!(
807                        request,
808                        PiSessionRequest::Prompt { .. } | PiSessionRequest::FollowUp { .. }
809                    );
810                    match harness.start_request(request) {
811                        Ok(request_id) => send_critical_worker_event(
812                            &events,
813                            WorkerEvent::RequestStarted(id.clone(), request_id, waits_for_agent),
814                        ),
815                        Err(error) => {
816                            send_critical_worker_event(
817                                &events,
818                                WorkerEvent::Failed(id.clone(), error.to_string()),
819                            );
820                            return;
821                        }
822                    }
823                }
824                Ok(WorkerCommand::Shutdown) | Err(TryRecvError::Disconnected) => {
825                    send_critical_worker_event(&events, WorkerEvent::Stopped(id));
826                    return;
827                }
828                Err(TryRecvError::Empty) => break,
829            }
830        }
831        match harness.recv_event_timeout(Duration::from_millis(25)) {
832            Ok(Some(value)) => send_lossy_worker_event(
833                &events,
834                WorkerEvent::Pi(id.clone(), value),
835                &dropped_events,
836            ),
837            Ok(None) => {}
838            Err(error) => {
839                send_critical_worker_event(&events, WorkerEvent::Failed(id, error.to_string()));
840                return;
841            }
842        }
843    }
844}
845
846fn send_lossy_worker_event(
847    events: &SyncSender<WorkerEvent>,
848    event: WorkerEvent,
849    dropped_events: &AtomicU64,
850) {
851    if matches!(events.try_send(event), Err(TrySendError::Full(_))) {
852        dropped_events.fetch_add(1, Ordering::Relaxed);
853    }
854}
855
856fn send_critical_worker_event(events: &SyncSender<WorkerEvent>, event: WorkerEvent) {
857    let _ = events.send(event);
858}
859
860fn validate_spec(
861    root: &Path,
862    records: &BTreeMap<AgentId, AgentRecord>,
863    spec: &AgentSpec,
864) -> DevelopmentResult<()> {
865    validate_text("agent role", &spec.role, 128)?;
866    validate_text("agent task", &spec.task, MAX_PROMPT_BYTES)?;
867    if spec.dependencies.len() > MAX_AGENTS {
868        return Err(DevelopmentError::InvalidInput(
869            "agent dependency list is too large".into(),
870        ));
871    }
872    let mut unique = BTreeSet::new();
873    for dependency in &spec.dependencies {
874        if !unique.insert(dependency) {
875            return Err(DevelopmentError::InvalidInput(format!(
876                "duplicate dependency {}",
877                dependency.as_str()
878            )));
879        }
880        if !records.contains_key(dependency) {
881            return Err(DevelopmentError::NotFound(format!(
882                "dependency agent {}",
883                dependency.as_str()
884            )));
885        }
886    }
887    if let Some(worktree) = &spec.worktree {
888        let worktree = std::fs::canonicalize(worktree)?;
889        if worktree == Path::new("/")
890            || (!worktree.starts_with(root) && !worktree.join(".git").exists())
891        {
892            return Err(DevelopmentError::PathOutsideWorkspace(worktree));
893        }
894    }
895    if spec.max_runtime_seconds == Some(0) || spec.max_events == Some(0) {
896        return Err(DevelopmentError::InvalidInput(
897            "agent budgets must be greater than zero".into(),
898        ));
899    }
900    Ok(())
901}
902
903fn validate_text(description: &str, text: &str, limit: usize) -> DevelopmentResult<()> {
904    if text.trim().is_empty() || text.len() > limit || text.contains('\0') {
905        return Err(DevelopmentError::InvalidInput(format!(
906            "{description} must contain 1..={limit} bytes without NUL"
907        )));
908    }
909    Ok(())
910}
911
912fn now_ms() -> u128 {
913    SystemTime::now()
914        .duration_since(UNIX_EPOCH)
915        .unwrap_or_default()
916        .as_millis()
917}
918
919#[cfg(test)]
920mod tests {
921    use super::*;
922    use std::sync::atomic::{AtomicU64, Ordering};
923
924    static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
925
926    fn test_root() -> PathBuf {
927        let sequence = NEXT_ROOT.fetch_add(1, Ordering::Relaxed);
928        let root = std::env::temp_dir().join(format!(
929            "glass-agent-registry-{}-{sequence}",
930            std::process::id()
931        ));
932        std::fs::create_dir_all(&root).unwrap();
933        root
934    }
935
936    #[test]
937    fn lossy_worker_queue_reports_dropped_events() {
938        let (sender, _receiver) = mpsc::sync_channel(1);
939        let id = AgentId::parse("agent-overflow").unwrap();
940        sender.send(WorkerEvent::Ready(id.clone())).unwrap();
941        let dropped = AtomicU64::new(0);
942        send_lossy_worker_event(
943            &sender,
944            WorkerEvent::Pi(id, serde_json::json!({"type":"delta"})),
945            &dropped,
946        );
947        assert_eq!(dropped.load(Ordering::Relaxed), 1);
948    }
949
950    fn insert_test_worker(
951        registry: &mut AgentRegistry,
952        id: AgentId,
953        command: SyncSender<WorkerCommand>,
954        worker: thread::JoinHandle<()>,
955    ) {
956        let spec = AgentSpec::new("lifecycle-test", "prove worker ownership");
957        let now = now_ms();
958        registry.records.insert(
959            id.clone(),
960            AgentRecord {
961                snapshot: AgentSnapshot {
962                    id,
963                    role: spec.role.clone(),
964                    task: spec.task.clone(),
965                    status: AgentStatus::Working,
966                    dependencies: Vec::new(),
967                    model: None,
968                    thinking: None,
969                    worktree: registry.root.clone(),
970                    unrestricted: false,
971                    created_at_ms: now,
972                    started_at_ms: Some(now),
973                    updated_at_ms: now,
974                    event_count: 0,
975                    dropped_event_count: 0,
976                    last_error: None,
977                    last_response_id: None,
978                    evidence: Vec::new(),
979                },
980                spec,
981                command: Some(command),
982                worker: Some(worker),
983                awaiting_agent_settle: false,
984                dropped_events: Arc::new(AtomicU64::new(0)),
985            },
986        );
987    }
988
989    #[test]
990    fn repeated_cancel_joins_every_owned_worker() {
991        let root = test_root();
992        let mut registry = AgentRegistry::new(&root).unwrap();
993        let exited = Arc::new(AtomicU64::new(0));
994        for sequence in 0..128 {
995            let id = AgentId::parse(format!("agent-test-{sequence}")).unwrap();
996            let (command, commands) = mpsc::sync_channel(COMMAND_CAPACITY);
997            let exits = Arc::clone(&exited);
998            let worker = thread::spawn(move || {
999                while let Ok(command) = commands.recv() {
1000                    if matches!(command, WorkerCommand::Shutdown) {
1001                        break;
1002                    }
1003                }
1004                exits.fetch_add(1, Ordering::SeqCst);
1005            });
1006            insert_test_worker(&mut registry, id.clone(), command, worker);
1007            registry.cancel(&id).unwrap();
1008            let record = registry.records.get(&id).unwrap();
1009            assert_eq!(record.snapshot.status, AgentStatus::Cancelled);
1010            assert!(record.worker.is_none());
1011        }
1012        assert_eq!(exited.load(Ordering::SeqCst), 128);
1013        drop(registry);
1014        std::fs::remove_dir_all(root).unwrap();
1015    }
1016
1017    #[test]
1018    fn worker_panic_becomes_attributed_agent_failure() {
1019        let root = test_root();
1020        let mut registry = AgentRegistry::new(&root).unwrap();
1021        let id = AgentId::parse("agent-panic-test").unwrap();
1022        let (command, _commands) = mpsc::sync_channel(COMMAND_CAPACITY);
1023        let worker = thread::spawn(|| panic!("intentional worker panic"));
1024        worker.thread().unpark();
1025        insert_test_worker(&mut registry, id.clone(), command, worker);
1026        for _ in 0..100 {
1027            registry.refresh().unwrap();
1028            if registry.records[&id].snapshot.status == AgentStatus::Failed {
1029                break;
1030            }
1031            thread::sleep(Duration::from_millis(5));
1032        }
1033        let snapshot = registry.snapshot(&id).unwrap();
1034        assert_eq!(snapshot.status, AgentStatus::Failed);
1035        assert!(snapshot.last_error.unwrap().contains(id.as_str()));
1036        assert!(registry.history(0).unwrap().iter().any(|event| {
1037            event.agent_id == id
1038                && event.kind == "workerPanicked"
1039                && event.payload["role"] == "lifecycle-test"
1040        }));
1041        drop(registry);
1042        std::fs::remove_dir_all(root).unwrap();
1043    }
1044
1045    fn wait_for_status(
1046        registry: &mut AgentRegistry,
1047        id: &AgentId,
1048        expected: AgentStatus,
1049    ) -> AgentSnapshot {
1050        for _ in 0..200 {
1051            let snapshot = registry.snapshot(id).unwrap();
1052            if snapshot.status == expected {
1053                return snapshot;
1054            }
1055            if snapshot.status == AgentStatus::Failed {
1056                panic!("agent failed: {:?}", snapshot.last_error);
1057            }
1058            thread::sleep(Duration::from_millis(25));
1059        }
1060        panic!("agent did not reach {expected:?}");
1061    }
1062
1063    #[test]
1064    fn independent_pi_sessions_schedule_dependencies_and_stream_state() {
1065        if std::process::Command::new("pi")
1066            .arg("--version")
1067            .output()
1068            .is_err()
1069        {
1070            return;
1071        }
1072        let root = test_root();
1073        let mut registry = AgentRegistry::new(&root).unwrap();
1074        let first = registry
1075            .create(AgentSpec::new("implementer", "inspect state"))
1076            .unwrap();
1077        let peer = registry
1078            .create(AgentSpec::new("researcher", "inspect independent state"))
1079            .unwrap();
1080        wait_for_status(&mut registry, &first, AgentStatus::Idle);
1081        wait_for_status(&mut registry, &peer, AgentStatus::Idle);
1082        registry.request(&first, PiSessionRequest::State).unwrap();
1083        registry.request(&peer, PiSessionRequest::State).unwrap();
1084        for _ in 0..200 {
1085            let snapshot = registry.snapshot(&first).unwrap();
1086            if snapshot
1087                .evidence
1088                .iter()
1089                .any(|event| event.get("type").and_then(Value::as_str) == Some("response"))
1090            {
1091                break;
1092            }
1093            thread::sleep(Duration::from_millis(25));
1094        }
1095        assert!(registry.snapshot(&first).unwrap().event_count > 0);
1096        for _ in 0..200 {
1097            if registry.snapshot(&peer).unwrap().event_count > 0 {
1098                break;
1099            }
1100            thread::sleep(Duration::from_millis(25));
1101        }
1102        assert!(registry.snapshot(&peer).unwrap().event_count > 0);
1103
1104        let mut dependent = AgentSpec::new("reviewer", "review evidence");
1105        dependent.dependencies.push(first.clone());
1106        let second = registry.create(dependent).unwrap();
1107        assert_eq!(
1108            registry.snapshot(&second).unwrap().status,
1109            AgentStatus::Queued
1110        );
1111        registry.complete(&first).unwrap();
1112        wait_for_status(&mut registry, &second, AgentStatus::Idle);
1113        assert_ne!(first, second);
1114        registry.cancel(&second).unwrap();
1115        registry.cancel(&peer).unwrap();
1116        assert_eq!(
1117            registry.snapshot(&second).unwrap().status,
1118            AgentStatus::Cancelled
1119        );
1120        drop(registry);
1121        std::fs::remove_dir_all(root).unwrap();
1122    }
1123
1124    #[test]
1125    fn invalid_agent_specs_and_failed_dependencies_fail_closed() {
1126        let root = test_root();
1127        let mut registry = AgentRegistry::new(&root).unwrap();
1128        assert!(registry.create(AgentSpec::new("", "task")).is_err());
1129        let first = registry.create(AgentSpec::new("lead", "task")).unwrap();
1130        let mut dependent = AgentSpec::new("review", "task");
1131        dependent.dependencies.push(first.clone());
1132        let second = registry.create(dependent).unwrap();
1133        registry.cancel(&first).unwrap();
1134        assert_eq!(
1135            registry.snapshot(&second).unwrap().status,
1136            AgentStatus::Failed
1137        );
1138        drop(registry);
1139        std::fs::remove_dir_all(root).unwrap();
1140    }
1141}