Skip to main content

glass_dev/
tasks.rs

1//! Autonomous, verified task DAG scheduling above resident Pi agents.
2
3use crate::agents::{AgentEvent, AgentId, AgentRegistry, AgentSnapshot, AgentSpec, AgentStatus};
4use crate::development::{DevelopmentError, DevelopmentResult};
5use serde::{Deserialize, Serialize};
6use serde_json::Value;
7use std::collections::{BTreeMap, BTreeSet};
8use std::path::{Path, PathBuf};
9use std::process::{Command, Stdio};
10use std::thread;
11use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
12
13const MAX_TASKS: usize = 128;
14const MAX_DEPENDENCIES: usize = 32;
15const MAX_TASK_EVIDENCE: usize = 128;
16const MAX_TASK_TEXT_BYTES: usize = 128 * 1024;
17const MAX_VERIFIER_COMMAND_BYTES: usize = 16 * 1024;
18
19#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
20pub struct TaskId(String);
21
22impl TaskId {
23    pub fn as_str(&self) -> &str {
24        &self.0
25    }
26
27    pub fn parse(value: impl Into<String>) -> DevelopmentResult<Self> {
28        let value = value.into();
29        if !value.starts_with("task-")
30            || value.len() > 64
31            || value.chars().any(|character| character.is_control())
32        {
33            return Err(DevelopmentError::InvalidInput("invalid task id".into()));
34        }
35        Ok(Self(value))
36    }
37}
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
40#[serde(rename_all = "camelCase")]
41pub enum TaskState {
42    Queued,
43    Ready,
44    Running,
45    Waiting,
46    Verifying,
47    Succeeded,
48    Failed,
49    Cancelled,
50    Paused,
51    Blocked,
52}
53
54impl TaskState {
55    fn terminal(self) -> bool {
56        matches!(self, Self::Succeeded | Self::Failed | Self::Cancelled)
57    }
58}
59
60#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
61#[serde(rename_all = "camelCase")]
62pub struct TaskBudget {
63    pub max_runtime_seconds: u64,
64    pub max_events: u64,
65    pub max_tokens: Option<u64>,
66}
67
68impl Default for TaskBudget {
69    fn default() -> Self {
70        Self {
71            max_runtime_seconds: 3_600,
72            max_events: 10_000,
73            max_tokens: None,
74        }
75    }
76}
77
78#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
79#[serde(rename_all = "camelCase")]
80pub struct RetryPolicy {
81    pub max_retries: u32,
82    pub backoff_seconds: u64,
83}
84
85impl Default for RetryPolicy {
86    fn default() -> Self {
87        Self {
88            max_retries: 0,
89            backoff_seconds: 1,
90        }
91    }
92}
93
94#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
95#[serde(tag = "kind", rename_all = "camelCase")]
96pub enum VerificationRequirement {
97    #[default]
98    Settled,
99    Command {
100        command: String,
101        expected_exit: i32,
102        timeout_seconds: u64,
103    },
104    LspDiagnostics {
105        max_errors: u64,
106    },
107    BrowserWorkflow {
108        assertion: String,
109    },
110    SemanticRegression {
111        baseline: String,
112        maximum_regressions: u64,
113    },
114    DebuggerAssertion {
115        session: String,
116        expression: String,
117    },
118    GitChange {
119        require_changes: bool,
120        require_clean: bool,
121    },
122    TrustedCustom {
123        name: String,
124    },
125    All {
126        requirements: Vec<VerificationRequirement>,
127    },
128}
129
130#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
131#[serde(rename_all = "camelCase")]
132pub struct TaskEvidence {
133    pub kind: String,
134    pub actor: String,
135    pub source: String,
136    pub passed: Option<bool>,
137    pub observed_at_ms: u128,
138    pub details: Value,
139}
140
141#[derive(Debug, Clone, Serialize, Deserialize)]
142#[serde(rename_all = "camelCase")]
143pub struct TaskSpec {
144    pub title: String,
145    pub goal: String,
146    pub prompt: String,
147    #[serde(default = "default_role")]
148    pub role: String,
149    #[serde(default)]
150    pub dependencies: Vec<TaskId>,
151    pub worktree: Option<PathBuf>,
152    pub model: Option<String>,
153    pub thinking: Option<String>,
154    #[serde(default)]
155    pub unrestricted: bool,
156    #[serde(default)]
157    pub budget: TaskBudget,
158    #[serde(default)]
159    pub verification: VerificationRequirement,
160    #[serde(default)]
161    pub retry: RetryPolicy,
162}
163
164fn default_role() -> String {
165    "developer".into()
166}
167
168impl TaskSpec {
169    pub fn new(title: impl Into<String>, prompt: impl Into<String>) -> Self {
170        let title = title.into();
171        Self {
172            goal: title.clone(),
173            title,
174            prompt: prompt.into(),
175            role: default_role(),
176            dependencies: Vec::new(),
177            worktree: None,
178            model: None,
179            thinking: None,
180            unrestricted: false,
181            budget: TaskBudget::default(),
182            verification: VerificationRequirement::default(),
183            retry: RetryPolicy::default(),
184        }
185    }
186}
187
188#[derive(Debug, Clone, Serialize, Deserialize)]
189#[serde(rename_all = "camelCase")]
190pub struct TaskSnapshot {
191    pub id: TaskId,
192    pub title: String,
193    pub goal: String,
194    pub prompt: String,
195    pub role: String,
196    pub dependencies: Vec<TaskId>,
197    pub assigned_agent: Option<AgentId>,
198    pub worktree: PathBuf,
199    pub model: Option<String>,
200    pub thinking: Option<String>,
201    pub unrestricted: bool,
202    pub budget: TaskBudget,
203    pub verification: VerificationRequirement,
204    pub retry: RetryPolicy,
205    pub state: TaskState,
206    pub attempt: u32,
207    pub observed_tokens: u64,
208    pub created_at_ms: u128,
209    pub started_at_ms: Option<u128>,
210    pub completed_at_ms: Option<u128>,
211    pub updated_at_ms: u128,
212    pub last_error: Option<String>,
213    pub evidence: Vec<TaskEvidence>,
214    pub blocked_override: bool,
215}
216
217struct TaskRecord {
218    snapshot: TaskSnapshot,
219    next_retry_at_ms: Option<u128>,
220}
221
222pub trait TaskAgentBackend {
223    fn refresh_agents(&mut self) -> DevelopmentResult<()>;
224    fn create_agent(&mut self, spec: AgentSpec) -> DevelopmentResult<AgentId>;
225    fn agent_snapshots(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>>;
226    fn agent_events(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>>;
227    fn prompt_agent(&mut self, id: &AgentId, prompt: String) -> DevelopmentResult<()>;
228    fn cancel_agent(&mut self, id: &AgentId) -> DevelopmentResult<()>;
229    fn complete_agent(&mut self, id: &AgentId) -> DevelopmentResult<()>;
230}
231
232impl TaskAgentBackend for AgentRegistry {
233    fn refresh_agents(&mut self) -> DevelopmentResult<()> {
234        self.refresh()
235    }
236
237    fn create_agent(&mut self, spec: AgentSpec) -> DevelopmentResult<AgentId> {
238        self.create(spec)
239    }
240
241    fn agent_snapshots(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>> {
242        self.list()
243    }
244
245    fn agent_events(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>> {
246        self.history(since)
247    }
248
249    fn prompt_agent(&mut self, id: &AgentId, prompt: String) -> DevelopmentResult<()> {
250        self.prompt(id, prompt)
251    }
252
253    fn cancel_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
254        self.cancel(id)
255    }
256
257    fn complete_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
258        self.complete(id)
259    }
260}
261
262pub struct TaskScheduler {
263    root: PathBuf,
264    tasks: BTreeMap<TaskId, TaskRecord>,
265    next_task: u64,
266    agent_event_cursor: u64,
267}
268
269impl TaskScheduler {
270    pub fn new(root: impl AsRef<Path>) -> DevelopmentResult<Self> {
271        Ok(Self {
272            root: std::fs::canonicalize(root)?,
273            tasks: BTreeMap::new(),
274            next_task: 1,
275            agent_event_cursor: 0,
276        })
277    }
278
279    pub fn create<B: TaskAgentBackend>(
280        &mut self,
281        agents: &mut B,
282        spec: TaskSpec,
283    ) -> DevelopmentResult<TaskId> {
284        self.refresh(agents)?;
285        self.validate_spec(&spec)?;
286        if self.tasks.len() >= MAX_TASKS {
287            return Err(DevelopmentError::Conflict(format!(
288                "task scheduler reached its {MAX_TASKS} task limit"
289            )));
290        }
291        let id = TaskId(format!("task-{:04}", self.next_task));
292        self.next_task = self
293            .next_task
294            .checked_add(1)
295            .ok_or_else(|| DevelopmentError::Conflict("task identifier space exhausted".into()))?;
296        let now = now_ms();
297        self.tasks.insert(
298            id.clone(),
299            TaskRecord {
300                snapshot: TaskSnapshot {
301                    id: id.clone(),
302                    title: spec.title,
303                    goal: spec.goal,
304                    prompt: spec.prompt,
305                    role: spec.role,
306                    dependencies: spec.dependencies,
307                    assigned_agent: None,
308                    worktree: spec.worktree.unwrap_or_else(|| self.root.clone()),
309                    model: spec.model,
310                    thinking: spec.thinking,
311                    unrestricted: spec.unrestricted,
312                    budget: spec.budget,
313                    verification: spec.verification,
314                    retry: spec.retry,
315                    state: TaskState::Queued,
316                    attempt: 0,
317                    observed_tokens: 0,
318                    created_at_ms: now,
319                    started_at_ms: None,
320                    completed_at_ms: None,
321                    updated_at_ms: now,
322                    last_error: None,
323                    evidence: Vec::new(),
324                    blocked_override: false,
325                },
326                next_retry_at_ms: None,
327            },
328        );
329        self.schedule_ready(agents)?;
330        Ok(id)
331    }
332
333    pub fn refresh<B: TaskAgentBackend>(&mut self, agents: &mut B) -> DevelopmentResult<()> {
334        agents.refresh_agents()?;
335        let snapshots = agents.agent_snapshots()?;
336        let events = agents.agent_events(self.agent_event_cursor)?;
337        for event in events {
338            self.agent_event_cursor = self.agent_event_cursor.max(event.sequence);
339            self.apply_agent_event(agents, &event)?;
340        }
341        self.apply_agent_failures(&snapshots);
342        self.enforce_task_budgets(agents)?;
343        self.verify_pending(agents)?;
344        self.propagate_blocked();
345        self.schedule_ready(agents)
346    }
347
348    pub fn list<B: TaskAgentBackend>(
349        &mut self,
350        agents: &mut B,
351    ) -> DevelopmentResult<Vec<TaskSnapshot>> {
352        self.refresh(agents)?;
353        Ok(self
354            .tasks
355            .values()
356            .map(|record| record.snapshot.clone())
357            .collect())
358    }
359
360    pub fn snapshot<B: TaskAgentBackend>(
361        &mut self,
362        agents: &mut B,
363        id: &TaskId,
364    ) -> DevelopmentResult<TaskSnapshot> {
365        self.refresh(agents)?;
366        self.tasks
367            .get(id)
368            .map(|record| record.snapshot.clone())
369            .ok_or_else(|| DevelopmentError::NotFound(format!("task {}", id.as_str())))
370    }
371
372    pub fn submit_evidence(
373        &mut self,
374        id: &TaskId,
375        kind: impl Into<String>,
376        actor: impl Into<String>,
377        source: impl Into<String>,
378        passed: bool,
379        details: Value,
380    ) -> DevelopmentResult<()> {
381        let record = self.record_mut(id)?;
382        push_evidence(
383            record,
384            TaskEvidence {
385                kind: kind.into(),
386                actor: actor.into(),
387                source: source.into(),
388                passed: Some(passed),
389                observed_at_ms: now_ms(),
390                details,
391            },
392        );
393        if record.snapshot.state == TaskState::Waiting {
394            record.snapshot.state = TaskState::Verifying;
395        }
396        Ok(())
397    }
398
399    pub fn pause<B: TaskAgentBackend>(
400        &mut self,
401        agents: &mut B,
402        id: &TaskId,
403    ) -> DevelopmentResult<()> {
404        self.refresh(agents)?;
405        let agent = self.record(id)?.snapshot.assigned_agent.clone();
406        if let Some(agent) = agent {
407            agents.cancel_agent(&agent)?;
408        }
409        let record = self.record_mut(id)?;
410        if record.snapshot.state.terminal() {
411            return Err(DevelopmentError::Conflict(
412                "terminal task cannot be paused".into(),
413            ));
414        }
415        record.snapshot.assigned_agent = None;
416        record.snapshot.state = TaskState::Paused;
417        record.snapshot.updated_at_ms = now_ms();
418        Ok(())
419    }
420
421    pub fn resume<B: TaskAgentBackend>(
422        &mut self,
423        agents: &mut B,
424        id: &TaskId,
425    ) -> DevelopmentResult<()> {
426        let record = self.record_mut(id)?;
427        if record.snapshot.state != TaskState::Paused {
428            return Err(DevelopmentError::Conflict("task is not paused".into()));
429        }
430        record.snapshot.state = TaskState::Queued;
431        record.snapshot.updated_at_ms = now_ms();
432        self.schedule_ready(agents)
433    }
434
435    pub fn cancel<B: TaskAgentBackend>(
436        &mut self,
437        agents: &mut B,
438        id: &TaskId,
439    ) -> DevelopmentResult<()> {
440        self.refresh(agents)?;
441        let agent = self.record(id)?.snapshot.assigned_agent.clone();
442        if let Some(agent) = agent {
443            agents.cancel_agent(&agent)?;
444        }
445        let record = self.record_mut(id)?;
446        if !record.snapshot.state.terminal() {
447            record.snapshot.state = TaskState::Cancelled;
448            record.snapshot.completed_at_ms = Some(now_ms());
449            record.snapshot.updated_at_ms = now_ms();
450        }
451        self.propagate_blocked();
452        Ok(())
453    }
454
455    pub fn retry<B: TaskAgentBackend>(
456        &mut self,
457        agents: &mut B,
458        id: &TaskId,
459    ) -> DevelopmentResult<()> {
460        let record = self.record_mut(id)?;
461        if !matches!(
462            record.snapshot.state,
463            TaskState::Failed | TaskState::Blocked
464        ) {
465            return Err(DevelopmentError::Conflict(
466                "only failed or blocked tasks can be retried".into(),
467            ));
468        }
469        record.snapshot.state = TaskState::Queued;
470        record.snapshot.assigned_agent = None;
471        record.snapshot.completed_at_ms = None;
472        record.snapshot.last_error = None;
473        record.snapshot.updated_at_ms = now_ms();
474        record.next_retry_at_ms = None;
475        self.schedule_ready(agents)
476    }
477
478    pub fn reassign<B: TaskAgentBackend>(
479        &mut self,
480        agents: &mut B,
481        id: &TaskId,
482        role: String,
483        model: Option<String>,
484        thinking: Option<String>,
485    ) -> DevelopmentResult<()> {
486        validate_text("task role", &role)?;
487        let agent = self.record(id)?.snapshot.assigned_agent.clone();
488        if let Some(agent) = agent {
489            agents.cancel_agent(&agent)?;
490        }
491        let record = self.record_mut(id)?;
492        if record.snapshot.state.terminal() {
493            return Err(DevelopmentError::Conflict(
494                "terminal task cannot be reassigned".into(),
495            ));
496        }
497        record.snapshot.role = role;
498        record.snapshot.model = model;
499        record.snapshot.thinking = thinking;
500        record.snapshot.assigned_agent = None;
501        record.snapshot.state = TaskState::Queued;
502        record.snapshot.updated_at_ms = now_ms();
503        self.schedule_ready(agents)
504    }
505
506    pub fn override_blocked<B: TaskAgentBackend>(
507        &mut self,
508        agents: &mut B,
509        id: &TaskId,
510    ) -> DevelopmentResult<()> {
511        let record = self.record_mut(id)?;
512        if record.snapshot.state != TaskState::Blocked {
513            return Err(DevelopmentError::Conflict("task is not blocked".into()));
514        }
515        record.snapshot.blocked_override = true;
516        record.snapshot.state = TaskState::Queued;
517        record.snapshot.last_error = None;
518        record.snapshot.updated_at_ms = now_ms();
519        self.schedule_ready(agents)
520    }
521
522    fn schedule_ready<B: TaskAgentBackend>(&mut self, agents: &mut B) -> DevelopmentResult<()> {
523        let now = now_ms();
524        let ready = self
525            .tasks
526            .iter()
527            .filter(|(_, record)| record.snapshot.state == TaskState::Queued)
528            .filter(|(_, record)| record.next_retry_at_ms.is_none_or(|at| at <= now))
529            .filter(|(_, record)| {
530                record.snapshot.blocked_override
531                    || record.snapshot.dependencies.iter().all(|dependency| {
532                        self.tasks.get(dependency).is_some_and(|dependency| {
533                            dependency.snapshot.state == TaskState::Succeeded
534                        })
535                    })
536            })
537            .map(|(id, _)| id.clone())
538            .collect::<Vec<_>>();
539        for id in ready {
540            let record = self.record(&id)?;
541            let mut spec = AgentSpec::new(
542                record.snapshot.role.clone(),
543                format!("task {}: {}", id.as_str(), record.snapshot.title),
544            );
545            spec.model.clone_from(&record.snapshot.model);
546            spec.thinking.clone_from(&record.snapshot.thinking);
547            spec.worktree = Some(record.snapshot.worktree.clone());
548            spec.unrestricted = record.snapshot.unrestricted;
549            spec.max_runtime_seconds = Some(record.snapshot.budget.max_runtime_seconds);
550            spec.max_events = Some(record.snapshot.budget.max_events);
551            let agent = agents.create_agent(spec)?;
552            let record = self.record_mut(&id)?;
553            record.snapshot.assigned_agent = Some(agent);
554            record.snapshot.state = TaskState::Ready;
555            record.snapshot.attempt = record.snapshot.attempt.saturating_add(1);
556            record.snapshot.updated_at_ms = now_ms();
557            record.next_retry_at_ms = None;
558        }
559        Ok(())
560    }
561
562    fn apply_agent_event<B: TaskAgentBackend>(
563        &mut self,
564        agents: &mut B,
565        event: &AgentEvent,
566    ) -> DevelopmentResult<()> {
567        let Some(task_id) = self.task_for_agent(&event.agent_id) else {
568            return Ok(());
569        };
570        let state = self.record(&task_id)?.snapshot.state;
571        match event.kind.as_str() {
572            "ready" if state == TaskState::Ready => {
573                let prompt = self.record(&task_id)?.snapshot.prompt.clone();
574                agents.prompt_agent(&event.agent_id, prompt)?;
575                let record = self.record_mut(&task_id)?;
576                record.snapshot.state = TaskState::Running;
577                record.snapshot.started_at_ms.get_or_insert_with(now_ms);
578                record.snapshot.updated_at_ms = now_ms();
579            }
580            "agent_settled" if state == TaskState::Running => {
581                let record = self.record_mut(&task_id)?;
582                record.snapshot.state = TaskState::Verifying;
583                record.snapshot.updated_at_ms = now_ms();
584                push_evidence(
585                    record,
586                    TaskEvidence {
587                        kind: "agent.settled".into(),
588                        actor: event.agent_id.as_str().into(),
589                        source: "pi-agent-session".into(),
590                        passed: Some(true),
591                        observed_at_ms: event.timestamp_ms,
592                        details: event.payload.clone(),
593                    },
594                );
595            }
596            kind => {
597                let record = self.record_mut(&task_id)?;
598                if kind == "message_end" {
599                    record.snapshot.observed_tokens = record
600                        .snapshot
601                        .observed_tokens
602                        .saturating_add(event_tokens(&event.payload));
603                }
604                push_evidence(
605                    record,
606                    TaskEvidence {
607                        kind: format!("agent.{kind}"),
608                        actor: event.agent_id.as_str().into(),
609                        source: "pi-agent-session".into(),
610                        passed: None,
611                        observed_at_ms: event.timestamp_ms,
612                        details: event.payload.clone(),
613                    },
614                );
615            }
616        }
617        Ok(())
618    }
619
620    fn apply_agent_failures(&mut self, snapshots: &[AgentSnapshot]) {
621        for snapshot in snapshots {
622            if !matches!(
623                snapshot.status,
624                AgentStatus::Failed | AgentStatus::Cancelled
625            ) {
626                continue;
627            }
628            let Some(id) = self.task_for_agent(&snapshot.id) else {
629                continue;
630            };
631            let record = self.tasks.get_mut(&id).expect("task exists");
632            if !record.snapshot.state.terminal() {
633                record.snapshot.state = TaskState::Failed;
634                record.snapshot.last_error = snapshot
635                    .last_error
636                    .clone()
637                    .or_else(|| Some("assigned agent stopped before verification".into()));
638                record.snapshot.completed_at_ms = Some(now_ms());
639                record.snapshot.updated_at_ms = now_ms();
640            }
641        }
642    }
643
644    fn verify_pending<B: TaskAgentBackend>(&mut self, agents: &mut B) -> DevelopmentResult<()> {
645        let pending = self
646            .tasks
647            .iter()
648            .filter(|(_, record)| record.snapshot.state == TaskState::Verifying)
649            .map(|(id, _)| id.clone())
650            .collect::<Vec<_>>();
651        for id in pending {
652            let (requirement, worktree, evidence) = {
653                let record = self.record(&id)?;
654                (
655                    record.snapshot.verification.clone(),
656                    record.snapshot.worktree.clone(),
657                    record.snapshot.evidence.clone(),
658                )
659            };
660            match verify(&requirement, &worktree, &evidence)? {
661                VerificationOutcome::Passed(new_evidence) => {
662                    let agent = self.record(&id)?.snapshot.assigned_agent.clone();
663                    if let Some(agent) = agent {
664                        agents.complete_agent(&agent)?;
665                    }
666                    let record = self.record_mut(&id)?;
667                    for evidence in new_evidence {
668                        push_evidence(record, evidence);
669                    }
670                    record.snapshot.state = TaskState::Succeeded;
671                    record.snapshot.completed_at_ms = Some(now_ms());
672                    record.snapshot.updated_at_ms = now_ms();
673                    record.snapshot.last_error = None;
674                }
675                VerificationOutcome::Waiting => {
676                    let record = self.record_mut(&id)?;
677                    record.snapshot.state = TaskState::Waiting;
678                    record.snapshot.updated_at_ms = now_ms();
679                }
680                VerificationOutcome::Failed(error, new_evidence) => {
681                    let (agent, retry, attempt) = {
682                        let record = self.record(&id)?;
683                        (
684                            record.snapshot.assigned_agent.clone(),
685                            record.snapshot.retry.clone(),
686                            record.snapshot.attempt,
687                        )
688                    };
689                    if let Some(agent) = agent {
690                        agents.cancel_agent(&agent)?;
691                    }
692                    let record = self.record_mut(&id)?;
693                    for evidence in new_evidence {
694                        push_evidence(record, evidence);
695                    }
696                    record.snapshot.assigned_agent = None;
697                    record.snapshot.last_error = Some(error);
698                    record.snapshot.updated_at_ms = now_ms();
699                    if attempt <= retry.max_retries {
700                        record.snapshot.state = TaskState::Queued;
701                        record.next_retry_at_ms = Some(
702                            now_ms().saturating_add(u128::from(retry.backoff_seconds) * 1_000),
703                        );
704                    } else {
705                        record.snapshot.state = TaskState::Failed;
706                        record.snapshot.completed_at_ms = Some(now_ms());
707                    }
708                }
709            }
710        }
711        Ok(())
712    }
713
714    fn enforce_task_budgets<B: TaskAgentBackend>(
715        &mut self,
716        agents: &mut B,
717    ) -> DevelopmentResult<()> {
718        let now = now_ms();
719        let exceeded = self
720            .tasks
721            .iter()
722            .filter(|(_, record)| {
723                matches!(
724                    record.snapshot.state,
725                    TaskState::Running | TaskState::Waiting
726                )
727            })
728            .filter(|(_, record)| {
729                record.snapshot.started_at_ms.is_some_and(|started| {
730                    now.saturating_sub(started)
731                        > u128::from(record.snapshot.budget.max_runtime_seconds) * 1_000
732                }) || record
733                    .snapshot
734                    .budget
735                    .max_tokens
736                    .is_some_and(|limit| record.snapshot.observed_tokens >= limit)
737            })
738            .map(|(id, _)| id.clone())
739            .collect::<Vec<_>>();
740        for id in exceeded {
741            if let Some(agent) = self.record(&id)?.snapshot.assigned_agent.clone() {
742                agents.cancel_agent(&agent)?;
743            }
744            let record = self.record_mut(&id)?;
745            record.snapshot.state = TaskState::Failed;
746            record.snapshot.last_error = Some("task runtime or token budget exceeded".into());
747            record.snapshot.completed_at_ms = Some(now);
748            record.snapshot.updated_at_ms = now;
749        }
750        Ok(())
751    }
752
753    fn propagate_blocked(&mut self) {
754        loop {
755            let failed = self
756                .tasks
757                .iter()
758                .filter(|(_, record)| {
759                    matches!(
760                        record.snapshot.state,
761                        TaskState::Failed | TaskState::Cancelled | TaskState::Blocked
762                    )
763                })
764                .map(|(id, _)| id.clone())
765                .collect::<BTreeSet<_>>();
766            let blocked = self
767                .tasks
768                .iter()
769                .filter(|(_, record)| {
770                    record.snapshot.state == TaskState::Queued && !record.snapshot.blocked_override
771                })
772                .filter(|(_, record)| {
773                    record
774                        .snapshot
775                        .dependencies
776                        .iter()
777                        .any(|dependency| failed.contains(dependency))
778                })
779                .map(|(id, _)| id.clone())
780                .collect::<Vec<_>>();
781            if blocked.is_empty() {
782                break;
783            }
784            for id in blocked {
785                let record = self.tasks.get_mut(&id).expect("task exists");
786                record.snapshot.state = TaskState::Blocked;
787                record.snapshot.last_error = Some("a prerequisite task did not succeed".into());
788                record.snapshot.updated_at_ms = now_ms();
789            }
790        }
791    }
792
793    fn validate_spec(&self, spec: &TaskSpec) -> DevelopmentResult<()> {
794        for (label, value) in [
795            ("task title", spec.title.as_str()),
796            ("task goal", spec.goal.as_str()),
797            ("task prompt", spec.prompt.as_str()),
798            ("task role", spec.role.as_str()),
799        ] {
800            validate_text(label, value)?;
801        }
802        if spec.dependencies.len() > MAX_DEPENDENCIES {
803            return Err(DevelopmentError::InvalidInput(
804                "task has too many dependencies".into(),
805            ));
806        }
807        let mut unique = BTreeSet::new();
808        for dependency in &spec.dependencies {
809            if !unique.insert(dependency) {
810                return Err(DevelopmentError::InvalidInput(format!(
811                    "duplicate task dependency {}",
812                    dependency.as_str()
813                )));
814            }
815            if !self.tasks.contains_key(dependency) {
816                return Err(DevelopmentError::NotFound(format!(
817                    "task dependency {}",
818                    dependency.as_str()
819                )));
820            }
821        }
822        if spec.budget.max_runtime_seconds == 0 || spec.budget.max_events == 0 {
823            return Err(DevelopmentError::InvalidInput(
824                "task budgets must be positive".into(),
825            ));
826        }
827        validate_verification(&spec.verification, 0)?;
828        if let Some(worktree) = &spec.worktree {
829            let worktree = std::fs::canonicalize(worktree)?;
830            if worktree == Path::new("/")
831                || (!worktree.starts_with(&self.root) && !worktree.join(".git").exists())
832            {
833                return Err(DevelopmentError::PathOutsideWorkspace(worktree));
834            }
835        }
836        Ok(())
837    }
838
839    fn task_for_agent(&self, agent: &AgentId) -> Option<TaskId> {
840        self.tasks
841            .iter()
842            .find(|(_, record)| record.snapshot.assigned_agent.as_ref() == Some(agent))
843            .map(|(id, _)| id.clone())
844    }
845
846    fn record(&self, id: &TaskId) -> DevelopmentResult<&TaskRecord> {
847        self.tasks
848            .get(id)
849            .ok_or_else(|| DevelopmentError::NotFound(format!("task {}", id.as_str())))
850    }
851
852    fn record_mut(&mut self, id: &TaskId) -> DevelopmentResult<&mut TaskRecord> {
853        self.tasks
854            .get_mut(id)
855            .ok_or_else(|| DevelopmentError::NotFound(format!("task {}", id.as_str())))
856    }
857}
858
859enum VerificationOutcome {
860    Passed(Vec<TaskEvidence>),
861    Waiting,
862    Failed(String, Vec<TaskEvidence>),
863}
864
865fn verify(
866    requirement: &VerificationRequirement,
867    worktree: &Path,
868    evidence: &[TaskEvidence],
869) -> DevelopmentResult<VerificationOutcome> {
870    match requirement {
871        VerificationRequirement::Settled => Ok(VerificationOutcome::Passed(Vec::new())),
872        VerificationRequirement::Command {
873            command,
874            expected_exit,
875            timeout_seconds,
876        } => verify_command(worktree, command, *expected_exit, *timeout_seconds),
877        VerificationRequirement::All { requirements } => {
878            let mut collected = Vec::new();
879            for requirement in requirements {
880                match verify(requirement, worktree, evidence)? {
881                    VerificationOutcome::Passed(mut current) => collected.append(&mut current),
882                    VerificationOutcome::Waiting => return Ok(VerificationOutcome::Waiting),
883                    VerificationOutcome::Failed(error, mut current) => {
884                        collected.append(&mut current);
885                        return Ok(VerificationOutcome::Failed(error, collected));
886                    }
887                }
888            }
889            Ok(VerificationOutcome::Passed(collected))
890        }
891        requirement => {
892            let kind = verification_kind(requirement);
893            match evidence.iter().rev().find(|item| item.kind == kind) {
894                Some(item) if evidence_satisfies(requirement, item) => {
895                    Ok(VerificationOutcome::Passed(Vec::new()))
896                }
897                Some(item) if item.passed == Some(false) => Ok(VerificationOutcome::Failed(
898                    format!("{kind} verification failed"),
899                    Vec::new(),
900                )),
901                _ => Ok(VerificationOutcome::Waiting),
902            }
903        }
904    }
905}
906
907fn evidence_satisfies(requirement: &VerificationRequirement, evidence: &TaskEvidence) -> bool {
908    if evidence.passed != Some(true) {
909        return false;
910    }
911    match requirement {
912        VerificationRequirement::LspDiagnostics { max_errors } => evidence
913            .details
914            .get("errors")
915            .and_then(Value::as_u64)
916            .is_some_and(|errors| errors <= *max_errors),
917        VerificationRequirement::SemanticRegression {
918            maximum_regressions,
919            ..
920        } => evidence
921            .details
922            .get("regressions")
923            .and_then(Value::as_u64)
924            .is_some_and(|regressions| regressions <= *maximum_regressions),
925        VerificationRequirement::GitChange {
926            require_changes,
927            require_clean,
928        } => {
929            evidence.details.get("hasChanges").and_then(Value::as_bool) == Some(*require_changes)
930                && (!*require_clean
931                    || evidence.details.get("clean").and_then(Value::as_bool) == Some(true))
932        }
933        _ => true,
934    }
935}
936
937fn verify_command(
938    worktree: &Path,
939    command: &str,
940    expected_exit: i32,
941    timeout_seconds: u64,
942) -> DevelopmentResult<VerificationOutcome> {
943    let mut process = if cfg!(windows) {
944        let mut command_process = Command::new("cmd.exe");
945        command_process.args(["/d", "/s", "/c", command]);
946        command_process
947    } else {
948        let mut command_process = Command::new("sh");
949        command_process.args(["-lc", command]);
950        command_process
951    };
952    let mut child = process
953        .current_dir(worktree)
954        .stdin(Stdio::null())
955        .stdout(Stdio::null())
956        .stderr(Stdio::null())
957        .spawn()?;
958    let deadline = Instant::now() + Duration::from_secs(timeout_seconds);
959    loop {
960        if let Some(status) = child.try_wait()? {
961            let actual = status.code().unwrap_or(-1);
962            let passed = actual == expected_exit;
963            let observed = TaskEvidence {
964                kind: "command".into(),
965                actor: "glass-task-verifier".into(),
966                source: "trusted-command".into(),
967                passed: Some(passed),
968                observed_at_ms: now_ms(),
969                details: serde_json::json!({
970                    "command": command,
971                    "expectedExit": expected_exit,
972                    "actualExit": actual,
973                }),
974            };
975            return Ok(if passed {
976                VerificationOutcome::Passed(vec![observed])
977            } else {
978                VerificationOutcome::Failed(
979                    format!("verification command exited {actual}, expected {expected_exit}"),
980                    vec![observed],
981                )
982            });
983        }
984        if Instant::now() >= deadline {
985            child.kill()?;
986            child.wait()?;
987            return Ok(VerificationOutcome::Failed(
988                "verification command timed out".into(),
989                vec![TaskEvidence {
990                    kind: "command".into(),
991                    actor: "glass-task-verifier".into(),
992                    source: "trusted-command".into(),
993                    passed: Some(false),
994                    observed_at_ms: now_ms(),
995                    details: serde_json::json!({"command":command,"timedOut":true}),
996                }],
997            ));
998        }
999        thread::sleep(Duration::from_millis(10));
1000    }
1001}
1002
1003fn verification_kind(requirement: &VerificationRequirement) -> &'static str {
1004    match requirement {
1005        VerificationRequirement::LspDiagnostics { .. } => "lspDiagnostics",
1006        VerificationRequirement::BrowserWorkflow { .. } => "browserWorkflow",
1007        VerificationRequirement::SemanticRegression { .. } => "semanticRegression",
1008        VerificationRequirement::DebuggerAssertion { .. } => "debuggerAssertion",
1009        VerificationRequirement::GitChange { .. } => "gitChange",
1010        VerificationRequirement::TrustedCustom { .. } => "trustedCustom",
1011        _ => "unknown",
1012    }
1013}
1014
1015fn event_tokens(value: &Value) -> u64 {
1016    value
1017        .pointer("/message/usage/totalTokens")
1018        .or_else(|| value.pointer("/usage/totalTokens"))
1019        .and_then(Value::as_u64)
1020        .or_else(|| {
1021            let usage = value
1022                .pointer("/message/usage")
1023                .or_else(|| value.get("usage"))?;
1024            Some(
1025                ["input", "output", "cacheRead", "cacheWrite"]
1026                    .iter()
1027                    .filter_map(|name| usage.get(name).and_then(Value::as_u64))
1028                    .fold(0_u64, u64::saturating_add),
1029            )
1030        })
1031        .unwrap_or(0)
1032}
1033
1034fn validate_verification(
1035    requirement: &VerificationRequirement,
1036    depth: usize,
1037) -> DevelopmentResult<()> {
1038    if depth > 8 {
1039        return Err(DevelopmentError::InvalidInput(
1040            "task verification nesting exceeds 8".into(),
1041        ));
1042    }
1043    match requirement {
1044        VerificationRequirement::Command {
1045            command,
1046            timeout_seconds,
1047            ..
1048        } => {
1049            if command.is_empty()
1050                || command.len() > MAX_VERIFIER_COMMAND_BYTES
1051                || command.contains('\0')
1052                || *timeout_seconds == 0
1053                || *timeout_seconds > 600
1054            {
1055                return Err(DevelopmentError::InvalidInput(
1056                    "invalid task verification command".into(),
1057                ));
1058            }
1059        }
1060        VerificationRequirement::All { requirements } => {
1061            if requirements.is_empty() || requirements.len() > 16 {
1062                return Err(DevelopmentError::InvalidInput(
1063                    "task verification group must contain 1..=16 requirements".into(),
1064                ));
1065            }
1066            for requirement in requirements {
1067                validate_verification(requirement, depth + 1)?;
1068            }
1069        }
1070        _ => {}
1071    }
1072    Ok(())
1073}
1074
1075fn push_evidence(record: &mut TaskRecord, evidence: TaskEvidence) {
1076    if record.snapshot.evidence.len() == MAX_TASK_EVIDENCE {
1077        record.snapshot.evidence.remove(0);
1078    }
1079    record.snapshot.evidence.push(evidence);
1080    record.snapshot.updated_at_ms = now_ms();
1081}
1082
1083fn validate_text(label: &str, value: &str) -> DevelopmentResult<()> {
1084    if value.is_empty() || value.len() > MAX_TASK_TEXT_BYTES || value.contains('\0') {
1085        return Err(DevelopmentError::InvalidInput(format!(
1086            "{label} must contain 1..={MAX_TASK_TEXT_BYTES} bytes without NUL"
1087        )));
1088    }
1089    Ok(())
1090}
1091
1092fn now_ms() -> u128 {
1093    SystemTime::now()
1094        .duration_since(UNIX_EPOCH)
1095        .unwrap_or_default()
1096        .as_millis()
1097}
1098
1099#[cfg(test)]
1100mod tests {
1101    use super::*;
1102    use std::sync::atomic::{AtomicU64, Ordering};
1103
1104    static NEXT_TEST_ROOT: AtomicU64 = AtomicU64::new(1);
1105
1106    #[derive(Default)]
1107    struct FakeAgents {
1108        next: u64,
1109        next_event: u64,
1110        snapshots: BTreeMap<AgentId, AgentSnapshot>,
1111        events: Vec<AgentEvent>,
1112        prompts: Vec<(AgentId, String)>,
1113    }
1114
1115    impl FakeAgents {
1116        fn emit(&mut self, id: &AgentId, kind: &str, payload: Value) {
1117            self.next_event += 1;
1118            self.events.push(AgentEvent {
1119                sequence: self.next_event,
1120                agent_id: id.clone(),
1121                timestamp_ms: now_ms(),
1122                kind: kind.into(),
1123                payload,
1124            });
1125        }
1126
1127        fn snapshot(id: AgentId, spec: AgentSpec) -> AgentSnapshot {
1128            AgentSnapshot {
1129                id,
1130                role: spec.role,
1131                task: spec.task,
1132                status: AgentStatus::Idle,
1133                dependencies: Vec::new(),
1134                model: spec.model,
1135                thinking: spec.thinking,
1136                worktree: spec.worktree.unwrap(),
1137                unrestricted: spec.unrestricted,
1138                created_at_ms: now_ms(),
1139                started_at_ms: Some(now_ms()),
1140                updated_at_ms: now_ms(),
1141                event_count: 0,
1142                dropped_event_count: 0,
1143                last_error: None,
1144                last_response_id: None,
1145                evidence: Vec::new(),
1146            }
1147        }
1148    }
1149
1150    impl TaskAgentBackend for FakeAgents {
1151        fn refresh_agents(&mut self) -> DevelopmentResult<()> {
1152            Ok(())
1153        }
1154
1155        fn create_agent(&mut self, spec: AgentSpec) -> DevelopmentResult<AgentId> {
1156            self.next += 1;
1157            let id = AgentId::parse(format!("agent-{:04}", self.next))?;
1158            self.snapshots
1159                .insert(id.clone(), Self::snapshot(id.clone(), spec));
1160            self.emit(&id, "ready", Value::Null);
1161            Ok(id)
1162        }
1163
1164        fn agent_snapshots(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>> {
1165            Ok(self.snapshots.values().cloned().collect())
1166        }
1167
1168        fn agent_events(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>> {
1169            Ok(self
1170                .events
1171                .iter()
1172                .filter(|event| event.sequence > since)
1173                .cloned()
1174                .collect())
1175        }
1176
1177        fn prompt_agent(&mut self, id: &AgentId, prompt: String) -> DevelopmentResult<()> {
1178            self.prompts.push((id.clone(), prompt));
1179            self.emit(id, "requestStarted", Value::String("request-1".into()));
1180            self.emit(id, "agent_settled", Value::Null);
1181            Ok(())
1182        }
1183
1184        fn cancel_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
1185            self.snapshots.get_mut(id).unwrap().status = AgentStatus::Cancelled;
1186            Ok(())
1187        }
1188
1189        fn complete_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
1190            self.snapshots.get_mut(id).unwrap().status = AgentStatus::Completed;
1191            Ok(())
1192        }
1193    }
1194
1195    fn scheduler() -> (PathBuf, TaskScheduler, FakeAgents) {
1196        let root = std::env::temp_dir().join(format!(
1197            "glass-task-scheduler-test-{}-{}",
1198            std::process::id(),
1199            NEXT_TEST_ROOT.fetch_add(1, Ordering::Relaxed)
1200        ));
1201        std::fs::create_dir_all(&root).unwrap();
1202        let scheduler = TaskScheduler::new(&root).unwrap();
1203        (root, scheduler, FakeAgents::default())
1204    }
1205
1206    fn settle(scheduler: &mut TaskScheduler, agents: &mut FakeAgents) {
1207        scheduler.refresh(agents).unwrap();
1208        scheduler.refresh(agents).unwrap();
1209    }
1210
1211    #[test]
1212    fn tasks_dispatch_prompts_verify_and_wake_dag_dependents() {
1213        let (root, mut scheduler, mut agents) = scheduler();
1214        let first = scheduler
1215            .create(&mut agents, TaskSpec::new("investigate", "inspect failure"))
1216            .unwrap();
1217        settle(&mut scheduler, &mut agents);
1218        assert_eq!(
1219            scheduler.snapshot(&mut agents, &first).unwrap().state,
1220            TaskState::Succeeded
1221        );
1222        assert_eq!(agents.prompts[0].1, "inspect failure");
1223
1224        let mut dependent = TaskSpec::new("repair", "apply repair");
1225        dependent.dependencies.push(first);
1226        let dependent = scheduler.create(&mut agents, dependent).unwrap();
1227        settle(&mut scheduler, &mut agents);
1228        assert_eq!(
1229            scheduler.snapshot(&mut agents, &dependent).unwrap().state,
1230            TaskState::Succeeded
1231        );
1232        assert_eq!(agents.prompts[1].1, "apply repair");
1233        std::fs::remove_dir_all(root).unwrap();
1234    }
1235
1236    #[test]
1237    fn eight_ready_tasks_dispatch_before_integration_dependency_wakes() {
1238        let (root, mut scheduler, mut agents) = scheduler();
1239        let mut leaves = Vec::new();
1240        for index in 0..8 {
1241            leaves.push(
1242                scheduler
1243                    .create(
1244                        &mut agents,
1245                        TaskSpec::new(format!("worker-{index}"), format!("inspect shard {index}")),
1246                    )
1247                    .unwrap(),
1248            );
1249        }
1250        assert_eq!(scheduler.tasks.len(), 8);
1251
1252        let mut integration = TaskSpec::new("integration", "verify all shards");
1253        integration.dependencies = leaves.clone();
1254        let integration = scheduler.create(&mut agents, integration).unwrap();
1255        assert_eq!(scheduler.tasks.len(), 9);
1256        settle(&mut scheduler, &mut agents);
1257        settle(&mut scheduler, &mut agents);
1258
1259        for leaf in leaves {
1260            assert_eq!(
1261                scheduler.snapshot(&mut agents, &leaf).unwrap().state,
1262                TaskState::Succeeded
1263            );
1264        }
1265        assert_eq!(
1266            scheduler.snapshot(&mut agents, &integration).unwrap().state,
1267            TaskState::Succeeded
1268        );
1269        assert_eq!(agents.prompts.len(), 9);
1270        std::fs::remove_dir_all(root).unwrap();
1271    }
1272
1273    #[test]
1274    fn verification_failure_retries_then_blocks_descendants() {
1275        let (root, mut scheduler, mut agents) = scheduler();
1276        let mut failing = TaskSpec::new("verify", "produce candidate");
1277        failing.verification = VerificationRequirement::Command {
1278            command: "exit 7".into(),
1279            expected_exit: 0,
1280            timeout_seconds: 2,
1281        };
1282        failing.retry = RetryPolicy {
1283            max_retries: 1,
1284            backoff_seconds: 0,
1285        };
1286        let failing = scheduler.create(&mut agents, failing).unwrap();
1287        let mut child = TaskSpec::new("integration", "integrate candidate");
1288        child.dependencies.push(failing.clone());
1289        let child = scheduler.create(&mut agents, child).unwrap();
1290        for _ in 0..6 {
1291            scheduler.refresh(&mut agents).unwrap();
1292        }
1293        let failed = scheduler.snapshot(&mut agents, &failing).unwrap();
1294        assert_eq!(failed.state, TaskState::Failed);
1295        assert_eq!(failed.attempt, 2);
1296        assert_eq!(
1297            scheduler.snapshot(&mut agents, &child).unwrap().state,
1298            TaskState::Blocked
1299        );
1300        assert_eq!(agents.prompts.len(), 2);
1301        std::fs::remove_dir_all(root).unwrap();
1302    }
1303
1304    #[test]
1305    fn external_verification_evidence_is_proof_not_agent_claim() {
1306        let (root, mut scheduler, mut agents) = scheduler();
1307        let mut spec = TaskSpec::new("diagnostics", "repair diagnostics");
1308        spec.verification = VerificationRequirement::LspDiagnostics { max_errors: 0 };
1309        let id = scheduler.create(&mut agents, spec).unwrap();
1310        settle(&mut scheduler, &mut agents);
1311        assert_eq!(
1312            scheduler.snapshot(&mut agents, &id).unwrap().state,
1313            TaskState::Waiting
1314        );
1315        scheduler
1316            .submit_evidence(
1317                &id,
1318                "lspDiagnostics",
1319                "resident-lsp",
1320                "task-test",
1321                true,
1322                serde_json::json!({"errors":0,"source":"resident-lsp"}),
1323            )
1324            .unwrap();
1325        scheduler.refresh(&mut agents).unwrap();
1326        assert_eq!(
1327            scheduler.snapshot(&mut agents, &id).unwrap().state,
1328            TaskState::Succeeded
1329        );
1330        std::fs::remove_dir_all(root).unwrap();
1331    }
1332}