1use crate::agents::{AgentEvent, AgentId, AgentRegistry, AgentSnapshot, AgentSpec, AgentStatus};
4use crate::development::{DevelopmentError, DevelopmentResult};
5use serde::{Deserialize, Serialize};
6use serde_json::Value;
7use std::collections::{BTreeMap, BTreeSet};
8use std::path::{Path, PathBuf};
9use std::process::{Command, Stdio};
10use std::thread;
11use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
12
13const MAX_TASKS: usize = 128;
14const MAX_DEPENDENCIES: usize = 32;
15const MAX_TASK_EVIDENCE: usize = 128;
16const MAX_TASK_TEXT_BYTES: usize = 128 * 1024;
17const MAX_VERIFIER_COMMAND_BYTES: usize = 16 * 1024;
18
19#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
20pub struct TaskId(String);
21
22impl TaskId {
23 pub fn as_str(&self) -> &str {
24 &self.0
25 }
26
27 pub fn parse(value: impl Into<String>) -> DevelopmentResult<Self> {
28 let value = value.into();
29 if !value.starts_with("task-")
30 || value.len() > 64
31 || value.chars().any(|character| character.is_control())
32 {
33 return Err(DevelopmentError::InvalidInput("invalid task id".into()));
34 }
35 Ok(Self(value))
36 }
37}
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
40#[serde(rename_all = "camelCase")]
41pub enum TaskState {
42 Queued,
43 Ready,
44 Running,
45 Waiting,
46 Verifying,
47 Succeeded,
48 Failed,
49 Cancelled,
50 Paused,
51 Blocked,
52}
53
54impl TaskState {
55 fn terminal(self) -> bool {
56 matches!(self, Self::Succeeded | Self::Failed | Self::Cancelled)
57 }
58}
59
60#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
61#[serde(rename_all = "camelCase")]
62pub struct TaskBudget {
63 pub max_runtime_seconds: u64,
64 pub max_events: u64,
65 pub max_tokens: Option<u64>,
66}
67
68impl Default for TaskBudget {
69 fn default() -> Self {
70 Self {
71 max_runtime_seconds: 3_600,
72 max_events: 10_000,
73 max_tokens: None,
74 }
75 }
76}
77
78#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
79#[serde(rename_all = "camelCase")]
80pub struct RetryPolicy {
81 pub max_retries: u32,
82 pub backoff_seconds: u64,
83}
84
85impl Default for RetryPolicy {
86 fn default() -> Self {
87 Self {
88 max_retries: 0,
89 backoff_seconds: 1,
90 }
91 }
92}
93
94#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
95#[serde(tag = "kind", rename_all = "camelCase")]
96pub enum VerificationRequirement {
97 #[default]
98 Settled,
99 Command {
100 command: String,
101 expected_exit: i32,
102 timeout_seconds: u64,
103 },
104 LspDiagnostics {
105 max_errors: u64,
106 },
107 BrowserWorkflow {
108 assertion: String,
109 },
110 SemanticRegression {
111 baseline: String,
112 maximum_regressions: u64,
113 },
114 DebuggerAssertion {
115 session: String,
116 expression: String,
117 },
118 GitChange {
119 require_changes: bool,
120 require_clean: bool,
121 },
122 TrustedCustom {
123 name: String,
124 },
125 All {
126 requirements: Vec<VerificationRequirement>,
127 },
128}
129
130#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
131#[serde(rename_all = "camelCase")]
132pub struct TaskEvidence {
133 pub kind: String,
134 pub actor: String,
135 pub source: String,
136 pub passed: Option<bool>,
137 pub observed_at_ms: u128,
138 pub details: Value,
139}
140
141#[derive(Debug, Clone, Serialize, Deserialize)]
142#[serde(rename_all = "camelCase")]
143pub struct TaskSpec {
144 pub title: String,
145 pub goal: String,
146 pub prompt: String,
147 #[serde(default = "default_role")]
148 pub role: String,
149 #[serde(default)]
150 pub dependencies: Vec<TaskId>,
151 pub worktree: Option<PathBuf>,
152 pub model: Option<String>,
153 pub thinking: Option<String>,
154 #[serde(default)]
155 pub unrestricted: bool,
156 #[serde(default)]
157 pub budget: TaskBudget,
158 #[serde(default)]
159 pub verification: VerificationRequirement,
160 #[serde(default)]
161 pub retry: RetryPolicy,
162}
163
164fn default_role() -> String {
165 "developer".into()
166}
167
168impl TaskSpec {
169 pub fn new(title: impl Into<String>, prompt: impl Into<String>) -> Self {
170 let title = title.into();
171 Self {
172 goal: title.clone(),
173 title,
174 prompt: prompt.into(),
175 role: default_role(),
176 dependencies: Vec::new(),
177 worktree: None,
178 model: None,
179 thinking: None,
180 unrestricted: false,
181 budget: TaskBudget::default(),
182 verification: VerificationRequirement::default(),
183 retry: RetryPolicy::default(),
184 }
185 }
186}
187
188#[derive(Debug, Clone, Serialize, Deserialize)]
189#[serde(rename_all = "camelCase")]
190pub struct TaskSnapshot {
191 pub id: TaskId,
192 pub title: String,
193 pub goal: String,
194 pub prompt: String,
195 pub role: String,
196 pub dependencies: Vec<TaskId>,
197 pub assigned_agent: Option<AgentId>,
198 pub worktree: PathBuf,
199 pub model: Option<String>,
200 pub thinking: Option<String>,
201 pub unrestricted: bool,
202 pub budget: TaskBudget,
203 pub verification: VerificationRequirement,
204 pub retry: RetryPolicy,
205 pub state: TaskState,
206 pub attempt: u32,
207 pub observed_tokens: u64,
208 pub created_at_ms: u128,
209 pub started_at_ms: Option<u128>,
210 pub completed_at_ms: Option<u128>,
211 pub updated_at_ms: u128,
212 pub last_error: Option<String>,
213 pub evidence: Vec<TaskEvidence>,
214 pub blocked_override: bool,
215}
216
217struct TaskRecord {
218 snapshot: TaskSnapshot,
219 next_retry_at_ms: Option<u128>,
220}
221
222pub trait TaskAgentBackend {
223 fn refresh_agents(&mut self) -> DevelopmentResult<()>;
224 fn create_agent(&mut self, spec: AgentSpec) -> DevelopmentResult<AgentId>;
225 fn agent_snapshots(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>>;
226 fn agent_events(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>>;
227 fn prompt_agent(&mut self, id: &AgentId, prompt: String) -> DevelopmentResult<()>;
228 fn cancel_agent(&mut self, id: &AgentId) -> DevelopmentResult<()>;
229 fn complete_agent(&mut self, id: &AgentId) -> DevelopmentResult<()>;
230}
231
232impl TaskAgentBackend for AgentRegistry {
233 fn refresh_agents(&mut self) -> DevelopmentResult<()> {
234 self.refresh()
235 }
236
237 fn create_agent(&mut self, spec: AgentSpec) -> DevelopmentResult<AgentId> {
238 self.create(spec)
239 }
240
241 fn agent_snapshots(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>> {
242 self.list()
243 }
244
245 fn agent_events(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>> {
246 self.history(since)
247 }
248
249 fn prompt_agent(&mut self, id: &AgentId, prompt: String) -> DevelopmentResult<()> {
250 self.prompt(id, prompt)
251 }
252
253 fn cancel_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
254 self.cancel(id)
255 }
256
257 fn complete_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
258 self.complete(id)
259 }
260}
261
262pub struct TaskScheduler {
263 root: PathBuf,
264 tasks: BTreeMap<TaskId, TaskRecord>,
265 next_task: u64,
266 agent_event_cursor: u64,
267}
268
269impl TaskScheduler {
270 pub fn new(root: impl AsRef<Path>) -> DevelopmentResult<Self> {
271 Ok(Self {
272 root: std::fs::canonicalize(root)?,
273 tasks: BTreeMap::new(),
274 next_task: 1,
275 agent_event_cursor: 0,
276 })
277 }
278
279 pub fn create<B: TaskAgentBackend>(
280 &mut self,
281 agents: &mut B,
282 spec: TaskSpec,
283 ) -> DevelopmentResult<TaskId> {
284 self.refresh(agents)?;
285 self.validate_spec(&spec)?;
286 if self.tasks.len() >= MAX_TASKS {
287 return Err(DevelopmentError::Conflict(format!(
288 "task scheduler reached its {MAX_TASKS} task limit"
289 )));
290 }
291 let id = TaskId(format!("task-{:04}", self.next_task));
292 self.next_task = self
293 .next_task
294 .checked_add(1)
295 .ok_or_else(|| DevelopmentError::Conflict("task identifier space exhausted".into()))?;
296 let now = now_ms();
297 self.tasks.insert(
298 id.clone(),
299 TaskRecord {
300 snapshot: TaskSnapshot {
301 id: id.clone(),
302 title: spec.title,
303 goal: spec.goal,
304 prompt: spec.prompt,
305 role: spec.role,
306 dependencies: spec.dependencies,
307 assigned_agent: None,
308 worktree: spec.worktree.unwrap_or_else(|| self.root.clone()),
309 model: spec.model,
310 thinking: spec.thinking,
311 unrestricted: spec.unrestricted,
312 budget: spec.budget,
313 verification: spec.verification,
314 retry: spec.retry,
315 state: TaskState::Queued,
316 attempt: 0,
317 observed_tokens: 0,
318 created_at_ms: now,
319 started_at_ms: None,
320 completed_at_ms: None,
321 updated_at_ms: now,
322 last_error: None,
323 evidence: Vec::new(),
324 blocked_override: false,
325 },
326 next_retry_at_ms: None,
327 },
328 );
329 self.schedule_ready(agents)?;
330 Ok(id)
331 }
332
333 pub fn refresh<B: TaskAgentBackend>(&mut self, agents: &mut B) -> DevelopmentResult<()> {
334 agents.refresh_agents()?;
335 let snapshots = agents.agent_snapshots()?;
336 let events = agents.agent_events(self.agent_event_cursor)?;
337 for event in events {
338 self.agent_event_cursor = self.agent_event_cursor.max(event.sequence);
339 self.apply_agent_event(agents, &event)?;
340 }
341 self.apply_agent_failures(&snapshots);
342 self.enforce_task_budgets(agents)?;
343 self.verify_pending(agents)?;
344 self.propagate_blocked();
345 self.schedule_ready(agents)
346 }
347
348 pub fn list<B: TaskAgentBackend>(
349 &mut self,
350 agents: &mut B,
351 ) -> DevelopmentResult<Vec<TaskSnapshot>> {
352 self.refresh(agents)?;
353 Ok(self
354 .tasks
355 .values()
356 .map(|record| record.snapshot.clone())
357 .collect())
358 }
359
360 pub fn snapshot<B: TaskAgentBackend>(
361 &mut self,
362 agents: &mut B,
363 id: &TaskId,
364 ) -> DevelopmentResult<TaskSnapshot> {
365 self.refresh(agents)?;
366 self.tasks
367 .get(id)
368 .map(|record| record.snapshot.clone())
369 .ok_or_else(|| DevelopmentError::NotFound(format!("task {}", id.as_str())))
370 }
371
372 pub fn submit_evidence(
373 &mut self,
374 id: &TaskId,
375 kind: impl Into<String>,
376 actor: impl Into<String>,
377 source: impl Into<String>,
378 passed: bool,
379 details: Value,
380 ) -> DevelopmentResult<()> {
381 let record = self.record_mut(id)?;
382 push_evidence(
383 record,
384 TaskEvidence {
385 kind: kind.into(),
386 actor: actor.into(),
387 source: source.into(),
388 passed: Some(passed),
389 observed_at_ms: now_ms(),
390 details,
391 },
392 );
393 if record.snapshot.state == TaskState::Waiting {
394 record.snapshot.state = TaskState::Verifying;
395 }
396 Ok(())
397 }
398
399 pub fn pause<B: TaskAgentBackend>(
400 &mut self,
401 agents: &mut B,
402 id: &TaskId,
403 ) -> DevelopmentResult<()> {
404 self.refresh(agents)?;
405 let agent = self.record(id)?.snapshot.assigned_agent.clone();
406 if let Some(agent) = agent {
407 agents.cancel_agent(&agent)?;
408 }
409 let record = self.record_mut(id)?;
410 if record.snapshot.state.terminal() {
411 return Err(DevelopmentError::Conflict(
412 "terminal task cannot be paused".into(),
413 ));
414 }
415 record.snapshot.assigned_agent = None;
416 record.snapshot.state = TaskState::Paused;
417 record.snapshot.updated_at_ms = now_ms();
418 Ok(())
419 }
420
421 pub fn resume<B: TaskAgentBackend>(
422 &mut self,
423 agents: &mut B,
424 id: &TaskId,
425 ) -> DevelopmentResult<()> {
426 let record = self.record_mut(id)?;
427 if record.snapshot.state != TaskState::Paused {
428 return Err(DevelopmentError::Conflict("task is not paused".into()));
429 }
430 record.snapshot.state = TaskState::Queued;
431 record.snapshot.updated_at_ms = now_ms();
432 self.schedule_ready(agents)
433 }
434
435 pub fn cancel<B: TaskAgentBackend>(
436 &mut self,
437 agents: &mut B,
438 id: &TaskId,
439 ) -> DevelopmentResult<()> {
440 self.refresh(agents)?;
441 let agent = self.record(id)?.snapshot.assigned_agent.clone();
442 if let Some(agent) = agent {
443 agents.cancel_agent(&agent)?;
444 }
445 let record = self.record_mut(id)?;
446 if !record.snapshot.state.terminal() {
447 record.snapshot.state = TaskState::Cancelled;
448 record.snapshot.completed_at_ms = Some(now_ms());
449 record.snapshot.updated_at_ms = now_ms();
450 }
451 self.propagate_blocked();
452 Ok(())
453 }
454
455 pub fn retry<B: TaskAgentBackend>(
456 &mut self,
457 agents: &mut B,
458 id: &TaskId,
459 ) -> DevelopmentResult<()> {
460 let record = self.record_mut(id)?;
461 if !matches!(
462 record.snapshot.state,
463 TaskState::Failed | TaskState::Blocked
464 ) {
465 return Err(DevelopmentError::Conflict(
466 "only failed or blocked tasks can be retried".into(),
467 ));
468 }
469 record.snapshot.state = TaskState::Queued;
470 record.snapshot.assigned_agent = None;
471 record.snapshot.completed_at_ms = None;
472 record.snapshot.last_error = None;
473 record.snapshot.updated_at_ms = now_ms();
474 record.next_retry_at_ms = None;
475 self.schedule_ready(agents)
476 }
477
478 pub fn reassign<B: TaskAgentBackend>(
479 &mut self,
480 agents: &mut B,
481 id: &TaskId,
482 role: String,
483 model: Option<String>,
484 thinking: Option<String>,
485 ) -> DevelopmentResult<()> {
486 validate_text("task role", &role)?;
487 let agent = self.record(id)?.snapshot.assigned_agent.clone();
488 if let Some(agent) = agent {
489 agents.cancel_agent(&agent)?;
490 }
491 let record = self.record_mut(id)?;
492 if record.snapshot.state.terminal() {
493 return Err(DevelopmentError::Conflict(
494 "terminal task cannot be reassigned".into(),
495 ));
496 }
497 record.snapshot.role = role;
498 record.snapshot.model = model;
499 record.snapshot.thinking = thinking;
500 record.snapshot.assigned_agent = None;
501 record.snapshot.state = TaskState::Queued;
502 record.snapshot.updated_at_ms = now_ms();
503 self.schedule_ready(agents)
504 }
505
506 pub fn override_blocked<B: TaskAgentBackend>(
507 &mut self,
508 agents: &mut B,
509 id: &TaskId,
510 ) -> DevelopmentResult<()> {
511 let record = self.record_mut(id)?;
512 if record.snapshot.state != TaskState::Blocked {
513 return Err(DevelopmentError::Conflict("task is not blocked".into()));
514 }
515 record.snapshot.blocked_override = true;
516 record.snapshot.state = TaskState::Queued;
517 record.snapshot.last_error = None;
518 record.snapshot.updated_at_ms = now_ms();
519 self.schedule_ready(agents)
520 }
521
522 fn schedule_ready<B: TaskAgentBackend>(&mut self, agents: &mut B) -> DevelopmentResult<()> {
523 let now = now_ms();
524 let ready = self
525 .tasks
526 .iter()
527 .filter(|(_, record)| record.snapshot.state == TaskState::Queued)
528 .filter(|(_, record)| record.next_retry_at_ms.is_none_or(|at| at <= now))
529 .filter(|(_, record)| {
530 record.snapshot.blocked_override
531 || record.snapshot.dependencies.iter().all(|dependency| {
532 self.tasks.get(dependency).is_some_and(|dependency| {
533 dependency.snapshot.state == TaskState::Succeeded
534 })
535 })
536 })
537 .map(|(id, _)| id.clone())
538 .collect::<Vec<_>>();
539 for id in ready {
540 let record = self.record(&id)?;
541 let mut spec = AgentSpec::new(
542 record.snapshot.role.clone(),
543 format!("task {}: {}", id.as_str(), record.snapshot.title),
544 );
545 spec.model.clone_from(&record.snapshot.model);
546 spec.thinking.clone_from(&record.snapshot.thinking);
547 spec.worktree = Some(record.snapshot.worktree.clone());
548 spec.unrestricted = record.snapshot.unrestricted;
549 spec.max_runtime_seconds = Some(record.snapshot.budget.max_runtime_seconds);
550 spec.max_events = Some(record.snapshot.budget.max_events);
551 let agent = agents.create_agent(spec)?;
552 let record = self.record_mut(&id)?;
553 record.snapshot.assigned_agent = Some(agent);
554 record.snapshot.state = TaskState::Ready;
555 record.snapshot.attempt = record.snapshot.attempt.saturating_add(1);
556 record.snapshot.updated_at_ms = now_ms();
557 record.next_retry_at_ms = None;
558 }
559 Ok(())
560 }
561
562 fn apply_agent_event<B: TaskAgentBackend>(
563 &mut self,
564 agents: &mut B,
565 event: &AgentEvent,
566 ) -> DevelopmentResult<()> {
567 let Some(task_id) = self.task_for_agent(&event.agent_id) else {
568 return Ok(());
569 };
570 let state = self.record(&task_id)?.snapshot.state;
571 match event.kind.as_str() {
572 "ready" if state == TaskState::Ready => {
573 let prompt = self.record(&task_id)?.snapshot.prompt.clone();
574 agents.prompt_agent(&event.agent_id, prompt)?;
575 let record = self.record_mut(&task_id)?;
576 record.snapshot.state = TaskState::Running;
577 record.snapshot.started_at_ms.get_or_insert_with(now_ms);
578 record.snapshot.updated_at_ms = now_ms();
579 }
580 "agent_settled" if state == TaskState::Running => {
581 let record = self.record_mut(&task_id)?;
582 record.snapshot.state = TaskState::Verifying;
583 record.snapshot.updated_at_ms = now_ms();
584 push_evidence(
585 record,
586 TaskEvidence {
587 kind: "agent.settled".into(),
588 actor: event.agent_id.as_str().into(),
589 source: "pi-agent-session".into(),
590 passed: Some(true),
591 observed_at_ms: event.timestamp_ms,
592 details: event.payload.clone(),
593 },
594 );
595 }
596 kind => {
597 let record = self.record_mut(&task_id)?;
598 if kind == "message_end" {
599 record.snapshot.observed_tokens = record
600 .snapshot
601 .observed_tokens
602 .saturating_add(event_tokens(&event.payload));
603 }
604 push_evidence(
605 record,
606 TaskEvidence {
607 kind: format!("agent.{kind}"),
608 actor: event.agent_id.as_str().into(),
609 source: "pi-agent-session".into(),
610 passed: None,
611 observed_at_ms: event.timestamp_ms,
612 details: event.payload.clone(),
613 },
614 );
615 }
616 }
617 Ok(())
618 }
619
620 fn apply_agent_failures(&mut self, snapshots: &[AgentSnapshot]) {
621 for snapshot in snapshots {
622 if !matches!(
623 snapshot.status,
624 AgentStatus::Failed | AgentStatus::Cancelled
625 ) {
626 continue;
627 }
628 let Some(id) = self.task_for_agent(&snapshot.id) else {
629 continue;
630 };
631 let record = self.tasks.get_mut(&id).expect("task exists");
632 if !record.snapshot.state.terminal() {
633 record.snapshot.state = TaskState::Failed;
634 record.snapshot.last_error = snapshot
635 .last_error
636 .clone()
637 .or_else(|| Some("assigned agent stopped before verification".into()));
638 record.snapshot.completed_at_ms = Some(now_ms());
639 record.snapshot.updated_at_ms = now_ms();
640 }
641 }
642 }
643
644 fn verify_pending<B: TaskAgentBackend>(&mut self, agents: &mut B) -> DevelopmentResult<()> {
645 let pending = self
646 .tasks
647 .iter()
648 .filter(|(_, record)| record.snapshot.state == TaskState::Verifying)
649 .map(|(id, _)| id.clone())
650 .collect::<Vec<_>>();
651 for id in pending {
652 let (requirement, worktree, evidence) = {
653 let record = self.record(&id)?;
654 (
655 record.snapshot.verification.clone(),
656 record.snapshot.worktree.clone(),
657 record.snapshot.evidence.clone(),
658 )
659 };
660 match verify(&requirement, &worktree, &evidence)? {
661 VerificationOutcome::Passed(new_evidence) => {
662 let agent = self.record(&id)?.snapshot.assigned_agent.clone();
663 if let Some(agent) = agent {
664 agents.complete_agent(&agent)?;
665 }
666 let record = self.record_mut(&id)?;
667 for evidence in new_evidence {
668 push_evidence(record, evidence);
669 }
670 record.snapshot.state = TaskState::Succeeded;
671 record.snapshot.completed_at_ms = Some(now_ms());
672 record.snapshot.updated_at_ms = now_ms();
673 record.snapshot.last_error = None;
674 }
675 VerificationOutcome::Waiting => {
676 let record = self.record_mut(&id)?;
677 record.snapshot.state = TaskState::Waiting;
678 record.snapshot.updated_at_ms = now_ms();
679 }
680 VerificationOutcome::Failed(error, new_evidence) => {
681 let (agent, retry, attempt) = {
682 let record = self.record(&id)?;
683 (
684 record.snapshot.assigned_agent.clone(),
685 record.snapshot.retry.clone(),
686 record.snapshot.attempt,
687 )
688 };
689 if let Some(agent) = agent {
690 agents.cancel_agent(&agent)?;
691 }
692 let record = self.record_mut(&id)?;
693 for evidence in new_evidence {
694 push_evidence(record, evidence);
695 }
696 record.snapshot.assigned_agent = None;
697 record.snapshot.last_error = Some(error);
698 record.snapshot.updated_at_ms = now_ms();
699 if attempt <= retry.max_retries {
700 record.snapshot.state = TaskState::Queued;
701 record.next_retry_at_ms = Some(
702 now_ms().saturating_add(u128::from(retry.backoff_seconds) * 1_000),
703 );
704 } else {
705 record.snapshot.state = TaskState::Failed;
706 record.snapshot.completed_at_ms = Some(now_ms());
707 }
708 }
709 }
710 }
711 Ok(())
712 }
713
714 fn enforce_task_budgets<B: TaskAgentBackend>(
715 &mut self,
716 agents: &mut B,
717 ) -> DevelopmentResult<()> {
718 let now = now_ms();
719 let exceeded = self
720 .tasks
721 .iter()
722 .filter(|(_, record)| {
723 matches!(
724 record.snapshot.state,
725 TaskState::Running | TaskState::Waiting
726 )
727 })
728 .filter(|(_, record)| {
729 record.snapshot.started_at_ms.is_some_and(|started| {
730 now.saturating_sub(started)
731 > u128::from(record.snapshot.budget.max_runtime_seconds) * 1_000
732 }) || record
733 .snapshot
734 .budget
735 .max_tokens
736 .is_some_and(|limit| record.snapshot.observed_tokens >= limit)
737 })
738 .map(|(id, _)| id.clone())
739 .collect::<Vec<_>>();
740 for id in exceeded {
741 if let Some(agent) = self.record(&id)?.snapshot.assigned_agent.clone() {
742 agents.cancel_agent(&agent)?;
743 }
744 let record = self.record_mut(&id)?;
745 record.snapshot.state = TaskState::Failed;
746 record.snapshot.last_error = Some("task runtime or token budget exceeded".into());
747 record.snapshot.completed_at_ms = Some(now);
748 record.snapshot.updated_at_ms = now;
749 }
750 Ok(())
751 }
752
753 fn propagate_blocked(&mut self) {
754 loop {
755 let failed = self
756 .tasks
757 .iter()
758 .filter(|(_, record)| {
759 matches!(
760 record.snapshot.state,
761 TaskState::Failed | TaskState::Cancelled | TaskState::Blocked
762 )
763 })
764 .map(|(id, _)| id.clone())
765 .collect::<BTreeSet<_>>();
766 let blocked = self
767 .tasks
768 .iter()
769 .filter(|(_, record)| {
770 record.snapshot.state == TaskState::Queued && !record.snapshot.blocked_override
771 })
772 .filter(|(_, record)| {
773 record
774 .snapshot
775 .dependencies
776 .iter()
777 .any(|dependency| failed.contains(dependency))
778 })
779 .map(|(id, _)| id.clone())
780 .collect::<Vec<_>>();
781 if blocked.is_empty() {
782 break;
783 }
784 for id in blocked {
785 let record = self.tasks.get_mut(&id).expect("task exists");
786 record.snapshot.state = TaskState::Blocked;
787 record.snapshot.last_error = Some("a prerequisite task did not succeed".into());
788 record.snapshot.updated_at_ms = now_ms();
789 }
790 }
791 }
792
793 fn validate_spec(&self, spec: &TaskSpec) -> DevelopmentResult<()> {
794 for (label, value) in [
795 ("task title", spec.title.as_str()),
796 ("task goal", spec.goal.as_str()),
797 ("task prompt", spec.prompt.as_str()),
798 ("task role", spec.role.as_str()),
799 ] {
800 validate_text(label, value)?;
801 }
802 if spec.dependencies.len() > MAX_DEPENDENCIES {
803 return Err(DevelopmentError::InvalidInput(
804 "task has too many dependencies".into(),
805 ));
806 }
807 let mut unique = BTreeSet::new();
808 for dependency in &spec.dependencies {
809 if !unique.insert(dependency) {
810 return Err(DevelopmentError::InvalidInput(format!(
811 "duplicate task dependency {}",
812 dependency.as_str()
813 )));
814 }
815 if !self.tasks.contains_key(dependency) {
816 return Err(DevelopmentError::NotFound(format!(
817 "task dependency {}",
818 dependency.as_str()
819 )));
820 }
821 }
822 if spec.budget.max_runtime_seconds == 0 || spec.budget.max_events == 0 {
823 return Err(DevelopmentError::InvalidInput(
824 "task budgets must be positive".into(),
825 ));
826 }
827 validate_verification(&spec.verification, 0)?;
828 if let Some(worktree) = &spec.worktree {
829 let worktree = std::fs::canonicalize(worktree)?;
830 if worktree == Path::new("/")
831 || (!worktree.starts_with(&self.root) && !worktree.join(".git").exists())
832 {
833 return Err(DevelopmentError::PathOutsideWorkspace(worktree));
834 }
835 }
836 Ok(())
837 }
838
839 fn task_for_agent(&self, agent: &AgentId) -> Option<TaskId> {
840 self.tasks
841 .iter()
842 .find(|(_, record)| record.snapshot.assigned_agent.as_ref() == Some(agent))
843 .map(|(id, _)| id.clone())
844 }
845
846 fn record(&self, id: &TaskId) -> DevelopmentResult<&TaskRecord> {
847 self.tasks
848 .get(id)
849 .ok_or_else(|| DevelopmentError::NotFound(format!("task {}", id.as_str())))
850 }
851
852 fn record_mut(&mut self, id: &TaskId) -> DevelopmentResult<&mut TaskRecord> {
853 self.tasks
854 .get_mut(id)
855 .ok_or_else(|| DevelopmentError::NotFound(format!("task {}", id.as_str())))
856 }
857}
858
859enum VerificationOutcome {
860 Passed(Vec<TaskEvidence>),
861 Waiting,
862 Failed(String, Vec<TaskEvidence>),
863}
864
865fn verify(
866 requirement: &VerificationRequirement,
867 worktree: &Path,
868 evidence: &[TaskEvidence],
869) -> DevelopmentResult<VerificationOutcome> {
870 match requirement {
871 VerificationRequirement::Settled => Ok(VerificationOutcome::Passed(Vec::new())),
872 VerificationRequirement::Command {
873 command,
874 expected_exit,
875 timeout_seconds,
876 } => verify_command(worktree, command, *expected_exit, *timeout_seconds),
877 VerificationRequirement::All { requirements } => {
878 let mut collected = Vec::new();
879 for requirement in requirements {
880 match verify(requirement, worktree, evidence)? {
881 VerificationOutcome::Passed(mut current) => collected.append(&mut current),
882 VerificationOutcome::Waiting => return Ok(VerificationOutcome::Waiting),
883 VerificationOutcome::Failed(error, mut current) => {
884 collected.append(&mut current);
885 return Ok(VerificationOutcome::Failed(error, collected));
886 }
887 }
888 }
889 Ok(VerificationOutcome::Passed(collected))
890 }
891 requirement => {
892 let kind = verification_kind(requirement);
893 match evidence.iter().rev().find(|item| item.kind == kind) {
894 Some(item) if evidence_satisfies(requirement, item) => {
895 Ok(VerificationOutcome::Passed(Vec::new()))
896 }
897 Some(item) if item.passed == Some(false) => Ok(VerificationOutcome::Failed(
898 format!("{kind} verification failed"),
899 Vec::new(),
900 )),
901 _ => Ok(VerificationOutcome::Waiting),
902 }
903 }
904 }
905}
906
907fn evidence_satisfies(requirement: &VerificationRequirement, evidence: &TaskEvidence) -> bool {
908 if evidence.passed != Some(true) {
909 return false;
910 }
911 match requirement {
912 VerificationRequirement::LspDiagnostics { max_errors } => evidence
913 .details
914 .get("errors")
915 .and_then(Value::as_u64)
916 .is_some_and(|errors| errors <= *max_errors),
917 VerificationRequirement::SemanticRegression {
918 maximum_regressions,
919 ..
920 } => evidence
921 .details
922 .get("regressions")
923 .and_then(Value::as_u64)
924 .is_some_and(|regressions| regressions <= *maximum_regressions),
925 VerificationRequirement::GitChange {
926 require_changes,
927 require_clean,
928 } => {
929 evidence.details.get("hasChanges").and_then(Value::as_bool) == Some(*require_changes)
930 && (!*require_clean
931 || evidence.details.get("clean").and_then(Value::as_bool) == Some(true))
932 }
933 _ => true,
934 }
935}
936
937fn verify_command(
938 worktree: &Path,
939 command: &str,
940 expected_exit: i32,
941 timeout_seconds: u64,
942) -> DevelopmentResult<VerificationOutcome> {
943 let mut process = if cfg!(windows) {
944 let mut command_process = Command::new("cmd.exe");
945 command_process.args(["/d", "/s", "/c", command]);
946 command_process
947 } else {
948 let mut command_process = Command::new("sh");
949 command_process.args(["-lc", command]);
950 command_process
951 };
952 let mut child = process
953 .current_dir(worktree)
954 .stdin(Stdio::null())
955 .stdout(Stdio::null())
956 .stderr(Stdio::null())
957 .spawn()?;
958 let deadline = Instant::now() + Duration::from_secs(timeout_seconds);
959 loop {
960 if let Some(status) = child.try_wait()? {
961 let actual = status.code().unwrap_or(-1);
962 let passed = actual == expected_exit;
963 let observed = TaskEvidence {
964 kind: "command".into(),
965 actor: "glass-task-verifier".into(),
966 source: "trusted-command".into(),
967 passed: Some(passed),
968 observed_at_ms: now_ms(),
969 details: serde_json::json!({
970 "command": command,
971 "expectedExit": expected_exit,
972 "actualExit": actual,
973 }),
974 };
975 return Ok(if passed {
976 VerificationOutcome::Passed(vec![observed])
977 } else {
978 VerificationOutcome::Failed(
979 format!("verification command exited {actual}, expected {expected_exit}"),
980 vec![observed],
981 )
982 });
983 }
984 if Instant::now() >= deadline {
985 child.kill()?;
986 child.wait()?;
987 return Ok(VerificationOutcome::Failed(
988 "verification command timed out".into(),
989 vec![TaskEvidence {
990 kind: "command".into(),
991 actor: "glass-task-verifier".into(),
992 source: "trusted-command".into(),
993 passed: Some(false),
994 observed_at_ms: now_ms(),
995 details: serde_json::json!({"command":command,"timedOut":true}),
996 }],
997 ));
998 }
999 thread::sleep(Duration::from_millis(10));
1000 }
1001}
1002
1003fn verification_kind(requirement: &VerificationRequirement) -> &'static str {
1004 match requirement {
1005 VerificationRequirement::LspDiagnostics { .. } => "lspDiagnostics",
1006 VerificationRequirement::BrowserWorkflow { .. } => "browserWorkflow",
1007 VerificationRequirement::SemanticRegression { .. } => "semanticRegression",
1008 VerificationRequirement::DebuggerAssertion { .. } => "debuggerAssertion",
1009 VerificationRequirement::GitChange { .. } => "gitChange",
1010 VerificationRequirement::TrustedCustom { .. } => "trustedCustom",
1011 _ => "unknown",
1012 }
1013}
1014
1015fn event_tokens(value: &Value) -> u64 {
1016 value
1017 .pointer("/message/usage/totalTokens")
1018 .or_else(|| value.pointer("/usage/totalTokens"))
1019 .and_then(Value::as_u64)
1020 .or_else(|| {
1021 let usage = value
1022 .pointer("/message/usage")
1023 .or_else(|| value.get("usage"))?;
1024 Some(
1025 ["input", "output", "cacheRead", "cacheWrite"]
1026 .iter()
1027 .filter_map(|name| usage.get(name).and_then(Value::as_u64))
1028 .fold(0_u64, u64::saturating_add),
1029 )
1030 })
1031 .unwrap_or(0)
1032}
1033
1034fn validate_verification(
1035 requirement: &VerificationRequirement,
1036 depth: usize,
1037) -> DevelopmentResult<()> {
1038 if depth > 8 {
1039 return Err(DevelopmentError::InvalidInput(
1040 "task verification nesting exceeds 8".into(),
1041 ));
1042 }
1043 match requirement {
1044 VerificationRequirement::Command {
1045 command,
1046 timeout_seconds,
1047 ..
1048 } => {
1049 if command.is_empty()
1050 || command.len() > MAX_VERIFIER_COMMAND_BYTES
1051 || command.contains('\0')
1052 || *timeout_seconds == 0
1053 || *timeout_seconds > 600
1054 {
1055 return Err(DevelopmentError::InvalidInput(
1056 "invalid task verification command".into(),
1057 ));
1058 }
1059 }
1060 VerificationRequirement::All { requirements } => {
1061 if requirements.is_empty() || requirements.len() > 16 {
1062 return Err(DevelopmentError::InvalidInput(
1063 "task verification group must contain 1..=16 requirements".into(),
1064 ));
1065 }
1066 for requirement in requirements {
1067 validate_verification(requirement, depth + 1)?;
1068 }
1069 }
1070 _ => {}
1071 }
1072 Ok(())
1073}
1074
1075fn push_evidence(record: &mut TaskRecord, evidence: TaskEvidence) {
1076 if record.snapshot.evidence.len() == MAX_TASK_EVIDENCE {
1077 record.snapshot.evidence.remove(0);
1078 }
1079 record.snapshot.evidence.push(evidence);
1080 record.snapshot.updated_at_ms = now_ms();
1081}
1082
1083fn validate_text(label: &str, value: &str) -> DevelopmentResult<()> {
1084 if value.is_empty() || value.len() > MAX_TASK_TEXT_BYTES || value.contains('\0') {
1085 return Err(DevelopmentError::InvalidInput(format!(
1086 "{label} must contain 1..={MAX_TASK_TEXT_BYTES} bytes without NUL"
1087 )));
1088 }
1089 Ok(())
1090}
1091
1092fn now_ms() -> u128 {
1093 SystemTime::now()
1094 .duration_since(UNIX_EPOCH)
1095 .unwrap_or_default()
1096 .as_millis()
1097}
1098
1099#[cfg(test)]
1100mod tests {
1101 use super::*;
1102 use std::sync::atomic::{AtomicU64, Ordering};
1103
1104 static NEXT_TEST_ROOT: AtomicU64 = AtomicU64::new(1);
1105
1106 #[derive(Default)]
1107 struct FakeAgents {
1108 next: u64,
1109 next_event: u64,
1110 snapshots: BTreeMap<AgentId, AgentSnapshot>,
1111 events: Vec<AgentEvent>,
1112 prompts: Vec<(AgentId, String)>,
1113 }
1114
1115 impl FakeAgents {
1116 fn emit(&mut self, id: &AgentId, kind: &str, payload: Value) {
1117 self.next_event += 1;
1118 self.events.push(AgentEvent {
1119 sequence: self.next_event,
1120 agent_id: id.clone(),
1121 timestamp_ms: now_ms(),
1122 kind: kind.into(),
1123 payload,
1124 });
1125 }
1126
1127 fn snapshot(id: AgentId, spec: AgentSpec) -> AgentSnapshot {
1128 AgentSnapshot {
1129 id,
1130 role: spec.role,
1131 task: spec.task,
1132 status: AgentStatus::Idle,
1133 dependencies: Vec::new(),
1134 model: spec.model,
1135 thinking: spec.thinking,
1136 worktree: spec.worktree.unwrap(),
1137 unrestricted: spec.unrestricted,
1138 created_at_ms: now_ms(),
1139 started_at_ms: Some(now_ms()),
1140 updated_at_ms: now_ms(),
1141 event_count: 0,
1142 dropped_event_count: 0,
1143 last_error: None,
1144 last_response_id: None,
1145 evidence: Vec::new(),
1146 }
1147 }
1148 }
1149
1150 impl TaskAgentBackend for FakeAgents {
1151 fn refresh_agents(&mut self) -> DevelopmentResult<()> {
1152 Ok(())
1153 }
1154
1155 fn create_agent(&mut self, spec: AgentSpec) -> DevelopmentResult<AgentId> {
1156 self.next += 1;
1157 let id = AgentId::parse(format!("agent-{:04}", self.next))?;
1158 self.snapshots
1159 .insert(id.clone(), Self::snapshot(id.clone(), spec));
1160 self.emit(&id, "ready", Value::Null);
1161 Ok(id)
1162 }
1163
1164 fn agent_snapshots(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>> {
1165 Ok(self.snapshots.values().cloned().collect())
1166 }
1167
1168 fn agent_events(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>> {
1169 Ok(self
1170 .events
1171 .iter()
1172 .filter(|event| event.sequence > since)
1173 .cloned()
1174 .collect())
1175 }
1176
1177 fn prompt_agent(&mut self, id: &AgentId, prompt: String) -> DevelopmentResult<()> {
1178 self.prompts.push((id.clone(), prompt));
1179 self.emit(id, "requestStarted", Value::String("request-1".into()));
1180 self.emit(id, "agent_settled", Value::Null);
1181 Ok(())
1182 }
1183
1184 fn cancel_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
1185 self.snapshots.get_mut(id).unwrap().status = AgentStatus::Cancelled;
1186 Ok(())
1187 }
1188
1189 fn complete_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
1190 self.snapshots.get_mut(id).unwrap().status = AgentStatus::Completed;
1191 Ok(())
1192 }
1193 }
1194
1195 fn scheduler() -> (PathBuf, TaskScheduler, FakeAgents) {
1196 let root = std::env::temp_dir().join(format!(
1197 "glass-task-scheduler-test-{}-{}",
1198 std::process::id(),
1199 NEXT_TEST_ROOT.fetch_add(1, Ordering::Relaxed)
1200 ));
1201 std::fs::create_dir_all(&root).unwrap();
1202 let scheduler = TaskScheduler::new(&root).unwrap();
1203 (root, scheduler, FakeAgents::default())
1204 }
1205
1206 fn settle(scheduler: &mut TaskScheduler, agents: &mut FakeAgents) {
1207 scheduler.refresh(agents).unwrap();
1208 scheduler.refresh(agents).unwrap();
1209 }
1210
1211 #[test]
1212 fn tasks_dispatch_prompts_verify_and_wake_dag_dependents() {
1213 let (root, mut scheduler, mut agents) = scheduler();
1214 let first = scheduler
1215 .create(&mut agents, TaskSpec::new("investigate", "inspect failure"))
1216 .unwrap();
1217 settle(&mut scheduler, &mut agents);
1218 assert_eq!(
1219 scheduler.snapshot(&mut agents, &first).unwrap().state,
1220 TaskState::Succeeded
1221 );
1222 assert_eq!(agents.prompts[0].1, "inspect failure");
1223
1224 let mut dependent = TaskSpec::new("repair", "apply repair");
1225 dependent.dependencies.push(first);
1226 let dependent = scheduler.create(&mut agents, dependent).unwrap();
1227 settle(&mut scheduler, &mut agents);
1228 assert_eq!(
1229 scheduler.snapshot(&mut agents, &dependent).unwrap().state,
1230 TaskState::Succeeded
1231 );
1232 assert_eq!(agents.prompts[1].1, "apply repair");
1233 std::fs::remove_dir_all(root).unwrap();
1234 }
1235
1236 #[test]
1237 fn eight_ready_tasks_dispatch_before_integration_dependency_wakes() {
1238 let (root, mut scheduler, mut agents) = scheduler();
1239 let mut leaves = Vec::new();
1240 for index in 0..8 {
1241 leaves.push(
1242 scheduler
1243 .create(
1244 &mut agents,
1245 TaskSpec::new(format!("worker-{index}"), format!("inspect shard {index}")),
1246 )
1247 .unwrap(),
1248 );
1249 }
1250 assert_eq!(scheduler.tasks.len(), 8);
1251
1252 let mut integration = TaskSpec::new("integration", "verify all shards");
1253 integration.dependencies = leaves.clone();
1254 let integration = scheduler.create(&mut agents, integration).unwrap();
1255 assert_eq!(scheduler.tasks.len(), 9);
1256 settle(&mut scheduler, &mut agents);
1257 settle(&mut scheduler, &mut agents);
1258
1259 for leaf in leaves {
1260 assert_eq!(
1261 scheduler.snapshot(&mut agents, &leaf).unwrap().state,
1262 TaskState::Succeeded
1263 );
1264 }
1265 assert_eq!(
1266 scheduler.snapshot(&mut agents, &integration).unwrap().state,
1267 TaskState::Succeeded
1268 );
1269 assert_eq!(agents.prompts.len(), 9);
1270 std::fs::remove_dir_all(root).unwrap();
1271 }
1272
1273 #[test]
1274 fn verification_failure_retries_then_blocks_descendants() {
1275 let (root, mut scheduler, mut agents) = scheduler();
1276 let mut failing = TaskSpec::new("verify", "produce candidate");
1277 failing.verification = VerificationRequirement::Command {
1278 command: "exit 7".into(),
1279 expected_exit: 0,
1280 timeout_seconds: 2,
1281 };
1282 failing.retry = RetryPolicy {
1283 max_retries: 1,
1284 backoff_seconds: 0,
1285 };
1286 let failing = scheduler.create(&mut agents, failing).unwrap();
1287 let mut child = TaskSpec::new("integration", "integrate candidate");
1288 child.dependencies.push(failing.clone());
1289 let child = scheduler.create(&mut agents, child).unwrap();
1290 for _ in 0..6 {
1291 scheduler.refresh(&mut agents).unwrap();
1292 }
1293 let failed = scheduler.snapshot(&mut agents, &failing).unwrap();
1294 assert_eq!(failed.state, TaskState::Failed);
1295 assert_eq!(failed.attempt, 2);
1296 assert_eq!(
1297 scheduler.snapshot(&mut agents, &child).unwrap().state,
1298 TaskState::Blocked
1299 );
1300 assert_eq!(agents.prompts.len(), 2);
1301 std::fs::remove_dir_all(root).unwrap();
1302 }
1303
1304 #[test]
1305 fn external_verification_evidence_is_proof_not_agent_claim() {
1306 let (root, mut scheduler, mut agents) = scheduler();
1307 let mut spec = TaskSpec::new("diagnostics", "repair diagnostics");
1308 spec.verification = VerificationRequirement::LspDiagnostics { max_errors: 0 };
1309 let id = scheduler.create(&mut agents, spec).unwrap();
1310 settle(&mut scheduler, &mut agents);
1311 assert_eq!(
1312 scheduler.snapshot(&mut agents, &id).unwrap().state,
1313 TaskState::Waiting
1314 );
1315 scheduler
1316 .submit_evidence(
1317 &id,
1318 "lspDiagnostics",
1319 "resident-lsp",
1320 "task-test",
1321 true,
1322 serde_json::json!({"errors":0,"source":"resident-lsp"}),
1323 )
1324 .unwrap();
1325 scheduler.refresh(&mut agents).unwrap();
1326 assert_eq!(
1327 scheduler.snapshot(&mut agents, &id).unwrap().state,
1328 TaskState::Succeeded
1329 );
1330 std::fs::remove_dir_all(root).unwrap();
1331 }
1332}