1use crate::agents::{AgentEvent, AgentId, AgentRegistry, AgentSnapshot, AgentSpec, AgentStatus};
4use crate::development::{DevelopmentError, DevelopmentResult};
5use serde::{Deserialize, Serialize};
6use serde_json::Value;
7use std::collections::{BTreeMap, BTreeSet};
8use std::path::{Path, PathBuf};
9use std::process::{Command, Stdio};
10use std::thread;
11use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
12
13const MAX_TASKS: usize = 128;
14const MAX_DEPENDENCIES: usize = 32;
15const MAX_TASK_EVIDENCE: usize = 128;
16const MAX_TASK_TEXT_BYTES: usize = 128 * 1024;
17const MAX_VERIFIER_COMMAND_BYTES: usize = 16 * 1024;
18
19#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
20pub struct TaskId(String);
21
22impl TaskId {
23 pub fn as_str(&self) -> &str {
24 &self.0
25 }
26
27 pub fn parse(value: impl Into<String>) -> DevelopmentResult<Self> {
28 let value = value.into();
29 if !value.starts_with("task-")
30 || value.len() > 64
31 || value.chars().any(|character| character.is_control())
32 {
33 return Err(DevelopmentError::InvalidInput("invalid task id".into()));
34 }
35 Ok(Self(value))
36 }
37}
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
40#[serde(rename_all = "camelCase")]
41pub enum TaskState {
42 Queued,
43 Ready,
44 Running,
45 Waiting,
46 Verifying,
47 Succeeded,
48 Failed,
49 Cancelled,
50 Paused,
51 Blocked,
52}
53
54impl TaskState {
55 fn terminal(self) -> bool {
56 matches!(self, Self::Succeeded | Self::Failed | Self::Cancelled)
57 }
58}
59
60#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
61#[serde(rename_all = "camelCase")]
62pub struct TaskBudget {
63 pub max_runtime_seconds: u64,
64 pub max_events: u64,
65 pub max_tokens: Option<u64>,
66}
67
68impl Default for TaskBudget {
69 fn default() -> Self {
70 Self {
71 max_runtime_seconds: 3_600,
72 max_events: 10_000,
73 max_tokens: None,
74 }
75 }
76}
77
78#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
79#[serde(rename_all = "camelCase")]
80pub struct RetryPolicy {
81 pub max_retries: u32,
82 pub backoff_seconds: u64,
83}
84
85impl Default for RetryPolicy {
86 fn default() -> Self {
87 Self {
88 max_retries: 0,
89 backoff_seconds: 1,
90 }
91 }
92}
93
94#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
95#[serde(tag = "kind", rename_all = "camelCase")]
96pub enum VerificationRequirement {
97 #[default]
99 Inferred,
100 Settled,
102 Command {
103 command: String,
104 expected_exit: i32,
105 timeout_seconds: u64,
106 },
107 LspDiagnostics {
108 max_errors: u64,
109 },
110 BrowserWorkflow {
111 assertion: String,
112 },
113 SemanticRegression {
114 baseline: String,
115 maximum_regressions: u64,
116 },
117 DebuggerAssertion {
118 session: String,
119 expression: String,
120 },
121 GitChange {
122 require_changes: bool,
123 require_clean: bool,
124 },
125 TrustedCustom {
126 name: String,
127 },
128 All {
129 requirements: Vec<VerificationRequirement>,
130 },
131}
132
133#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
134#[serde(rename_all = "camelCase")]
135pub struct TaskEvidence {
136 pub kind: String,
137 pub actor: String,
138 pub source: String,
139 pub passed: Option<bool>,
140 pub observed_at_ms: u128,
141 pub details: Value,
142}
143
144#[derive(Debug, Clone, Serialize, Deserialize)]
145#[serde(rename_all = "camelCase")]
146pub struct TaskSpec {
147 pub title: String,
148 pub goal: String,
149 pub prompt: String,
150 #[serde(default = "default_role")]
151 pub role: String,
152 #[serde(default)]
153 pub dependencies: Vec<TaskId>,
154 pub worktree: Option<PathBuf>,
155 pub model: Option<String>,
156 pub thinking: Option<String>,
157 #[serde(default)]
158 pub unrestricted: bool,
159 #[serde(default)]
160 pub budget: TaskBudget,
161 #[serde(default)]
162 pub verification: VerificationRequirement,
163 #[serde(default)]
164 pub retry: RetryPolicy,
165}
166
167fn default_role() -> String {
168 "developer".into()
169}
170
171impl TaskSpec {
172 pub fn new(title: impl Into<String>, prompt: impl Into<String>) -> Self {
173 let title = title.into();
174 Self {
175 goal: title.clone(),
176 title,
177 prompt: prompt.into(),
178 role: default_role(),
179 dependencies: Vec::new(),
180 worktree: None,
181 model: None,
182 thinking: None,
183 unrestricted: false,
184 budget: TaskBudget::default(),
185 verification: VerificationRequirement::default(),
186 retry: RetryPolicy::default(),
187 }
188 }
189}
190
191#[derive(Debug, Clone, Serialize, Deserialize)]
192#[serde(rename_all = "camelCase")]
193pub struct TaskSnapshot {
194 pub id: TaskId,
195 pub title: String,
196 pub goal: String,
197 pub prompt: String,
198 pub role: String,
199 pub dependencies: Vec<TaskId>,
200 pub assigned_agent: Option<AgentId>,
201 pub worktree: PathBuf,
202 pub model: Option<String>,
203 pub thinking: Option<String>,
204 pub unrestricted: bool,
205 pub budget: TaskBudget,
206 pub verification: VerificationRequirement,
207 pub retry: RetryPolicy,
208 pub state: TaskState,
209 pub attempt: u32,
210 pub observed_tokens: u64,
211 pub created_at_ms: u128,
212 pub started_at_ms: Option<u128>,
213 pub completed_at_ms: Option<u128>,
214 pub updated_at_ms: u128,
215 pub last_error: Option<String>,
216 pub evidence: Vec<TaskEvidence>,
217 pub blocked_override: bool,
218}
219
220struct TaskRecord {
221 snapshot: TaskSnapshot,
222 next_retry_at_ms: Option<u128>,
223}
224
225pub trait TaskAgentBackend {
226 fn refresh_agents(&mut self) -> DevelopmentResult<()>;
227 fn create_agent(&mut self, spec: AgentSpec) -> DevelopmentResult<AgentId>;
228 fn agent_snapshots(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>>;
229 fn agent_events(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>>;
230 fn prompt_agent(&mut self, id: &AgentId, prompt: String) -> DevelopmentResult<()>;
231 fn cancel_agent(&mut self, id: &AgentId) -> DevelopmentResult<()>;
232 fn complete_agent(&mut self, id: &AgentId) -> DevelopmentResult<()>;
233}
234
235impl TaskAgentBackend for AgentRegistry {
236 fn refresh_agents(&mut self) -> DevelopmentResult<()> {
237 self.refresh()
238 }
239
240 fn create_agent(&mut self, spec: AgentSpec) -> DevelopmentResult<AgentId> {
241 self.create(spec)
242 }
243
244 fn agent_snapshots(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>> {
245 self.list()
246 }
247
248 fn agent_events(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>> {
249 self.history(since)
250 }
251
252 fn prompt_agent(&mut self, id: &AgentId, prompt: String) -> DevelopmentResult<()> {
253 self.prompt(id, prompt)
254 }
255
256 fn cancel_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
257 self.cancel(id)
258 }
259
260 fn complete_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
261 self.complete(id)
262 }
263}
264
265pub struct TaskScheduler {
266 root: PathBuf,
267 tasks: BTreeMap<TaskId, TaskRecord>,
268 next_task: u64,
269 agent_event_cursor: u64,
270}
271
272impl TaskScheduler {
273 pub fn new(root: impl AsRef<Path>) -> DevelopmentResult<Self> {
274 Ok(Self {
275 root: std::fs::canonicalize(root)?,
276 tasks: BTreeMap::new(),
277 next_task: 1,
278 agent_event_cursor: 0,
279 })
280 }
281
282 pub fn create<B: TaskAgentBackend>(
283 &mut self,
284 agents: &mut B,
285 mut spec: TaskSpec,
286 ) -> DevelopmentResult<TaskId> {
287 self.refresh(agents)?;
288 if spec.verification == VerificationRequirement::Inferred {
289 let worktree = spec.worktree.as_deref().unwrap_or(&self.root);
290 spec.verification = infer_verification(worktree);
291 }
292 self.validate_spec(&spec)?;
293 if self.tasks.len() >= MAX_TASKS {
294 return Err(DevelopmentError::Conflict(format!(
295 "task scheduler reached its {MAX_TASKS} task limit"
296 )));
297 }
298 let id = TaskId(format!("task-{:04}", self.next_task));
299 self.next_task = self
300 .next_task
301 .checked_add(1)
302 .ok_or_else(|| DevelopmentError::Conflict("task identifier space exhausted".into()))?;
303 let now = now_ms();
304 self.tasks.insert(
305 id.clone(),
306 TaskRecord {
307 snapshot: TaskSnapshot {
308 id: id.clone(),
309 title: spec.title,
310 goal: spec.goal,
311 prompt: spec.prompt,
312 role: spec.role,
313 dependencies: spec.dependencies,
314 assigned_agent: None,
315 worktree: spec.worktree.unwrap_or_else(|| self.root.clone()),
316 model: spec.model,
317 thinking: spec.thinking,
318 unrestricted: spec.unrestricted,
319 budget: spec.budget,
320 verification: spec.verification,
321 retry: spec.retry,
322 state: TaskState::Queued,
323 attempt: 0,
324 observed_tokens: 0,
325 created_at_ms: now,
326 started_at_ms: None,
327 completed_at_ms: None,
328 updated_at_ms: now,
329 last_error: None,
330 evidence: Vec::new(),
331 blocked_override: false,
332 },
333 next_retry_at_ms: None,
334 },
335 );
336 self.schedule_ready(agents)?;
337 Ok(id)
338 }
339
340 pub fn refresh<B: TaskAgentBackend>(&mut self, agents: &mut B) -> DevelopmentResult<()> {
341 agents.refresh_agents()?;
342 let snapshots = agents.agent_snapshots()?;
343 let events = agents.agent_events(self.agent_event_cursor)?;
344 for event in events {
345 self.agent_event_cursor = self.agent_event_cursor.max(event.sequence);
346 self.apply_agent_event(agents, &event)?;
347 }
348 self.apply_agent_failures(&snapshots);
349 self.enforce_task_budgets(agents)?;
350 self.verify_pending(agents)?;
351 self.propagate_blocked();
352 self.schedule_ready(agents)
353 }
354
355 pub fn list<B: TaskAgentBackend>(
356 &mut self,
357 agents: &mut B,
358 ) -> DevelopmentResult<Vec<TaskSnapshot>> {
359 self.refresh(agents)?;
360 Ok(self
361 .tasks
362 .values()
363 .map(|record| record.snapshot.clone())
364 .collect())
365 }
366
367 pub fn snapshot<B: TaskAgentBackend>(
368 &mut self,
369 agents: &mut B,
370 id: &TaskId,
371 ) -> DevelopmentResult<TaskSnapshot> {
372 self.refresh(agents)?;
373 self.tasks
374 .get(id)
375 .map(|record| record.snapshot.clone())
376 .ok_or_else(|| DevelopmentError::NotFound(format!("task {}", id.as_str())))
377 }
378
379 pub fn submit_evidence(
380 &mut self,
381 id: &TaskId,
382 kind: impl Into<String>,
383 actor: impl Into<String>,
384 source: impl Into<String>,
385 passed: bool,
386 details: Value,
387 ) -> DevelopmentResult<()> {
388 let record = self.record_mut(id)?;
389 push_evidence(
390 record,
391 TaskEvidence {
392 kind: kind.into(),
393 actor: actor.into(),
394 source: source.into(),
395 passed: Some(passed),
396 observed_at_ms: now_ms(),
397 details,
398 },
399 );
400 if record.snapshot.state == TaskState::Waiting {
401 record.snapshot.state = TaskState::Verifying;
402 }
403 Ok(())
404 }
405
406 pub fn pause<B: TaskAgentBackend>(
407 &mut self,
408 agents: &mut B,
409 id: &TaskId,
410 ) -> DevelopmentResult<()> {
411 self.refresh(agents)?;
412 let agent = self.record(id)?.snapshot.assigned_agent.clone();
413 if let Some(agent) = agent {
414 agents.cancel_agent(&agent)?;
415 }
416 let record = self.record_mut(id)?;
417 if record.snapshot.state.terminal() {
418 return Err(DevelopmentError::Conflict(
419 "terminal task cannot be paused".into(),
420 ));
421 }
422 record.snapshot.assigned_agent = None;
423 record.snapshot.state = TaskState::Paused;
424 record.snapshot.updated_at_ms = now_ms();
425 Ok(())
426 }
427
428 pub fn resume<B: TaskAgentBackend>(
429 &mut self,
430 agents: &mut B,
431 id: &TaskId,
432 ) -> DevelopmentResult<()> {
433 let record = self.record_mut(id)?;
434 if record.snapshot.state != TaskState::Paused {
435 return Err(DevelopmentError::Conflict("task is not paused".into()));
436 }
437 record.snapshot.state = TaskState::Queued;
438 record.snapshot.updated_at_ms = now_ms();
439 self.schedule_ready(agents)
440 }
441
442 pub fn cancel<B: TaskAgentBackend>(
443 &mut self,
444 agents: &mut B,
445 id: &TaskId,
446 ) -> DevelopmentResult<()> {
447 self.refresh(agents)?;
448 let agent = self.record(id)?.snapshot.assigned_agent.clone();
449 if let Some(agent) = agent {
450 agents.cancel_agent(&agent)?;
451 }
452 let record = self.record_mut(id)?;
453 if !record.snapshot.state.terminal() {
454 record.snapshot.state = TaskState::Cancelled;
455 record.snapshot.completed_at_ms = Some(now_ms());
456 record.snapshot.updated_at_ms = now_ms();
457 }
458 self.propagate_blocked();
459 Ok(())
460 }
461
462 pub fn retry<B: TaskAgentBackend>(
463 &mut self,
464 agents: &mut B,
465 id: &TaskId,
466 ) -> DevelopmentResult<()> {
467 let record = self.record_mut(id)?;
468 if !matches!(
469 record.snapshot.state,
470 TaskState::Failed | TaskState::Blocked
471 ) {
472 return Err(DevelopmentError::Conflict(
473 "only failed or blocked tasks can be retried".into(),
474 ));
475 }
476 record.snapshot.state = TaskState::Queued;
477 record.snapshot.assigned_agent = None;
478 record.snapshot.completed_at_ms = None;
479 record.snapshot.last_error = None;
480 record.snapshot.updated_at_ms = now_ms();
481 record.next_retry_at_ms = None;
482 self.schedule_ready(agents)
483 }
484
485 pub fn reassign<B: TaskAgentBackend>(
486 &mut self,
487 agents: &mut B,
488 id: &TaskId,
489 role: String,
490 model: Option<String>,
491 thinking: Option<String>,
492 ) -> DevelopmentResult<()> {
493 validate_text("task role", &role)?;
494 let agent = self.record(id)?.snapshot.assigned_agent.clone();
495 if let Some(agent) = agent {
496 agents.cancel_agent(&agent)?;
497 }
498 let record = self.record_mut(id)?;
499 if record.snapshot.state.terminal() {
500 return Err(DevelopmentError::Conflict(
501 "terminal task cannot be reassigned".into(),
502 ));
503 }
504 record.snapshot.role = role;
505 record.snapshot.model = model;
506 record.snapshot.thinking = thinking;
507 record.snapshot.assigned_agent = None;
508 record.snapshot.state = TaskState::Queued;
509 record.snapshot.updated_at_ms = now_ms();
510 self.schedule_ready(agents)
511 }
512
513 pub fn override_blocked<B: TaskAgentBackend>(
514 &mut self,
515 agents: &mut B,
516 id: &TaskId,
517 ) -> DevelopmentResult<()> {
518 let record = self.record_mut(id)?;
519 if record.snapshot.state != TaskState::Blocked {
520 return Err(DevelopmentError::Conflict("task is not blocked".into()));
521 }
522 record.snapshot.blocked_override = true;
523 record.snapshot.state = TaskState::Queued;
524 record.snapshot.last_error = None;
525 record.snapshot.updated_at_ms = now_ms();
526 self.schedule_ready(agents)
527 }
528
529 fn schedule_ready<B: TaskAgentBackend>(&mut self, agents: &mut B) -> DevelopmentResult<()> {
530 let now = now_ms();
531 let ready = self
532 .tasks
533 .iter()
534 .filter(|(_, record)| record.snapshot.state == TaskState::Queued)
535 .filter(|(_, record)| record.next_retry_at_ms.is_none_or(|at| at <= now))
536 .filter(|(_, record)| {
537 record.snapshot.blocked_override
538 || record.snapshot.dependencies.iter().all(|dependency| {
539 self.tasks.get(dependency).is_some_and(|dependency| {
540 dependency.snapshot.state == TaskState::Succeeded
541 })
542 })
543 })
544 .map(|(id, _)| id.clone())
545 .collect::<Vec<_>>();
546 for id in ready {
547 let record = self.record(&id)?;
548 let mut spec = AgentSpec::new(
549 record.snapshot.role.clone(),
550 format!("task {}: {}", id.as_str(), record.snapshot.title),
551 );
552 spec.model.clone_from(&record.snapshot.model);
553 spec.thinking.clone_from(&record.snapshot.thinking);
554 spec.worktree = Some(record.snapshot.worktree.clone());
555 spec.unrestricted = record.snapshot.unrestricted;
556 spec.max_runtime_seconds = Some(record.snapshot.budget.max_runtime_seconds);
557 spec.max_events = Some(record.snapshot.budget.max_events);
558 let agent = agents.create_agent(spec)?;
559 let record = self.record_mut(&id)?;
560 record.snapshot.assigned_agent = Some(agent);
561 record.snapshot.state = TaskState::Ready;
562 record.snapshot.attempt = record.snapshot.attempt.saturating_add(1);
563 record.snapshot.updated_at_ms = now_ms();
564 record.next_retry_at_ms = None;
565 }
566 Ok(())
567 }
568
569 fn apply_agent_event<B: TaskAgentBackend>(
570 &mut self,
571 agents: &mut B,
572 event: &AgentEvent,
573 ) -> DevelopmentResult<()> {
574 let Some(task_id) = self.task_for_agent(&event.agent_id) else {
575 return Ok(());
576 };
577 let state = self.record(&task_id)?.snapshot.state;
578 match event.kind.as_str() {
579 "ready" if state == TaskState::Ready => {
580 let prompt = self.record(&task_id)?.snapshot.prompt.clone();
581 agents.prompt_agent(&event.agent_id, prompt)?;
582 let record = self.record_mut(&task_id)?;
583 record.snapshot.state = TaskState::Running;
584 record.snapshot.started_at_ms.get_or_insert_with(now_ms);
585 record.snapshot.updated_at_ms = now_ms();
586 }
587 "agent_settled" if state == TaskState::Running => {
588 let record = self.record_mut(&task_id)?;
589 record.snapshot.state = TaskState::Verifying;
590 record.snapshot.updated_at_ms = now_ms();
591 push_evidence(
592 record,
593 TaskEvidence {
594 kind: "agent.settled".into(),
595 actor: event.agent_id.as_str().into(),
596 source: "pi-agent-session".into(),
597 passed: Some(true),
598 observed_at_ms: event.timestamp_ms,
599 details: event.payload.clone(),
600 },
601 );
602 }
603 kind => {
604 let record = self.record_mut(&task_id)?;
605 if kind == "message_end" {
606 record.snapshot.observed_tokens = record
607 .snapshot
608 .observed_tokens
609 .saturating_add(event_tokens(&event.payload));
610 }
611 push_evidence(
612 record,
613 TaskEvidence {
614 kind: format!("agent.{kind}"),
615 actor: event.agent_id.as_str().into(),
616 source: "pi-agent-session".into(),
617 passed: None,
618 observed_at_ms: event.timestamp_ms,
619 details: event.payload.clone(),
620 },
621 );
622 }
623 }
624 Ok(())
625 }
626
627 fn apply_agent_failures(&mut self, snapshots: &[AgentSnapshot]) {
628 for snapshot in snapshots {
629 if !matches!(
630 snapshot.status,
631 AgentStatus::Failed | AgentStatus::Cancelled
632 ) {
633 continue;
634 }
635 let Some(id) = self.task_for_agent(&snapshot.id) else {
636 continue;
637 };
638 let record = self.tasks.get_mut(&id).expect("task exists");
639 if !record.snapshot.state.terminal() {
640 record.snapshot.state = TaskState::Failed;
641 record.snapshot.last_error = snapshot
642 .last_error
643 .clone()
644 .or_else(|| Some("assigned agent stopped before verification".into()));
645 record.snapshot.completed_at_ms = Some(now_ms());
646 record.snapshot.updated_at_ms = now_ms();
647 }
648 }
649 }
650
651 fn verify_pending<B: TaskAgentBackend>(&mut self, agents: &mut B) -> DevelopmentResult<()> {
652 let pending = self
653 .tasks
654 .iter()
655 .filter(|(_, record)| record.snapshot.state == TaskState::Verifying)
656 .map(|(id, _)| id.clone())
657 .collect::<Vec<_>>();
658 for id in pending {
659 let (requirement, worktree, evidence) = {
660 let record = self.record(&id)?;
661 (
662 record.snapshot.verification.clone(),
663 record.snapshot.worktree.clone(),
664 record.snapshot.evidence.clone(),
665 )
666 };
667 match verify(&requirement, &worktree, &evidence)? {
668 VerificationOutcome::Passed(new_evidence) => {
669 let agent = self.record(&id)?.snapshot.assigned_agent.clone();
670 if let Some(agent) = agent {
671 agents.complete_agent(&agent)?;
672 }
673 let record = self.record_mut(&id)?;
674 for evidence in new_evidence {
675 push_evidence(record, evidence);
676 }
677 record.snapshot.state = TaskState::Succeeded;
678 record.snapshot.completed_at_ms = Some(now_ms());
679 record.snapshot.updated_at_ms = now_ms();
680 record.snapshot.last_error = None;
681 }
682 VerificationOutcome::Waiting => {
683 let record = self.record_mut(&id)?;
684 record.snapshot.state = TaskState::Waiting;
685 record.snapshot.updated_at_ms = now_ms();
686 }
687 VerificationOutcome::Failed(error, new_evidence) => {
688 let (agent, retry, attempt) = {
689 let record = self.record(&id)?;
690 (
691 record.snapshot.assigned_agent.clone(),
692 record.snapshot.retry.clone(),
693 record.snapshot.attempt,
694 )
695 };
696 if let Some(agent) = agent {
697 agents.cancel_agent(&agent)?;
698 }
699 let record = self.record_mut(&id)?;
700 for evidence in new_evidence {
701 push_evidence(record, evidence);
702 }
703 record.snapshot.assigned_agent = None;
704 record.snapshot.last_error = Some(error);
705 record.snapshot.updated_at_ms = now_ms();
706 if attempt <= retry.max_retries {
707 record.snapshot.state = TaskState::Queued;
708 record.next_retry_at_ms = Some(
709 now_ms().saturating_add(u128::from(retry.backoff_seconds) * 1_000),
710 );
711 } else {
712 record.snapshot.state = TaskState::Failed;
713 record.snapshot.completed_at_ms = Some(now_ms());
714 }
715 }
716 }
717 }
718 Ok(())
719 }
720
721 fn enforce_task_budgets<B: TaskAgentBackend>(
722 &mut self,
723 agents: &mut B,
724 ) -> DevelopmentResult<()> {
725 let now = now_ms();
726 let exceeded = self
727 .tasks
728 .iter()
729 .filter(|(_, record)| {
730 matches!(
731 record.snapshot.state,
732 TaskState::Running | TaskState::Waiting
733 )
734 })
735 .filter(|(_, record)| {
736 record.snapshot.started_at_ms.is_some_and(|started| {
737 now.saturating_sub(started)
738 > u128::from(record.snapshot.budget.max_runtime_seconds) * 1_000
739 }) || record
740 .snapshot
741 .budget
742 .max_tokens
743 .is_some_and(|limit| record.snapshot.observed_tokens >= limit)
744 })
745 .map(|(id, _)| id.clone())
746 .collect::<Vec<_>>();
747 for id in exceeded {
748 if let Some(agent) = self.record(&id)?.snapshot.assigned_agent.clone() {
749 agents.cancel_agent(&agent)?;
750 }
751 let record = self.record_mut(&id)?;
752 record.snapshot.state = TaskState::Failed;
753 record.snapshot.last_error = Some("task runtime or token budget exceeded".into());
754 record.snapshot.completed_at_ms = Some(now);
755 record.snapshot.updated_at_ms = now;
756 }
757 Ok(())
758 }
759
760 fn propagate_blocked(&mut self) {
761 loop {
762 let failed = self
763 .tasks
764 .iter()
765 .filter(|(_, record)| {
766 matches!(
767 record.snapshot.state,
768 TaskState::Failed | TaskState::Cancelled | TaskState::Blocked
769 )
770 })
771 .map(|(id, _)| id.clone())
772 .collect::<BTreeSet<_>>();
773 let blocked = self
774 .tasks
775 .iter()
776 .filter(|(_, record)| {
777 record.snapshot.state == TaskState::Queued && !record.snapshot.blocked_override
778 })
779 .filter(|(_, record)| {
780 record
781 .snapshot
782 .dependencies
783 .iter()
784 .any(|dependency| failed.contains(dependency))
785 })
786 .map(|(id, _)| id.clone())
787 .collect::<Vec<_>>();
788 if blocked.is_empty() {
789 break;
790 }
791 for id in blocked {
792 let record = self.tasks.get_mut(&id).expect("task exists");
793 record.snapshot.state = TaskState::Blocked;
794 record.snapshot.last_error = Some("a prerequisite task did not succeed".into());
795 record.snapshot.updated_at_ms = now_ms();
796 }
797 }
798 }
799
800 fn validate_spec(&self, spec: &TaskSpec) -> DevelopmentResult<()> {
801 for (label, value) in [
802 ("task title", spec.title.as_str()),
803 ("task goal", spec.goal.as_str()),
804 ("task prompt", spec.prompt.as_str()),
805 ("task role", spec.role.as_str()),
806 ] {
807 validate_text(label, value)?;
808 }
809 if spec.dependencies.len() > MAX_DEPENDENCIES {
810 return Err(DevelopmentError::InvalidInput(
811 "task has too many dependencies".into(),
812 ));
813 }
814 let mut unique = BTreeSet::new();
815 for dependency in &spec.dependencies {
816 if !unique.insert(dependency) {
817 return Err(DevelopmentError::InvalidInput(format!(
818 "duplicate task dependency {}",
819 dependency.as_str()
820 )));
821 }
822 if !self.tasks.contains_key(dependency) {
823 return Err(DevelopmentError::NotFound(format!(
824 "task dependency {}",
825 dependency.as_str()
826 )));
827 }
828 }
829 if spec.budget.max_runtime_seconds == 0 || spec.budget.max_events == 0 {
830 return Err(DevelopmentError::InvalidInput(
831 "task budgets must be positive".into(),
832 ));
833 }
834 validate_verification(&spec.verification, 0)?;
835 if let Some(worktree) = &spec.worktree {
836 let worktree = std::fs::canonicalize(worktree)?;
837 if worktree == Path::new("/")
838 || (!worktree.starts_with(&self.root) && !worktree.join(".git").exists())
839 {
840 return Err(DevelopmentError::PathOutsideWorkspace(worktree));
841 }
842 }
843 Ok(())
844 }
845
846 fn task_for_agent(&self, agent: &AgentId) -> Option<TaskId> {
847 self.tasks
848 .iter()
849 .find(|(_, record)| record.snapshot.assigned_agent.as_ref() == Some(agent))
850 .map(|(id, _)| id.clone())
851 }
852
853 fn record(&self, id: &TaskId) -> DevelopmentResult<&TaskRecord> {
854 self.tasks
855 .get(id)
856 .ok_or_else(|| DevelopmentError::NotFound(format!("task {}", id.as_str())))
857 }
858
859 fn record_mut(&mut self, id: &TaskId) -> DevelopmentResult<&mut TaskRecord> {
860 self.tasks
861 .get_mut(id)
862 .ok_or_else(|| DevelopmentError::NotFound(format!("task {}", id.as_str())))
863 }
864}
865
866enum VerificationOutcome {
867 Passed(Vec<TaskEvidence>),
868 Waiting,
869 Failed(String, Vec<TaskEvidence>),
870}
871
872fn verify(
873 requirement: &VerificationRequirement,
874 worktree: &Path,
875 evidence: &[TaskEvidence],
876) -> DevelopmentResult<VerificationOutcome> {
877 match requirement {
878 VerificationRequirement::Inferred => Err(DevelopmentError::Conflict(
879 "inferred task verification was not resolved at creation".into(),
880 )),
881 VerificationRequirement::Settled => Ok(VerificationOutcome::Passed(Vec::new())),
882 VerificationRequirement::Command {
883 command,
884 expected_exit,
885 timeout_seconds,
886 } => verify_command(worktree, command, *expected_exit, *timeout_seconds),
887 VerificationRequirement::All { requirements } => {
888 let mut collected = Vec::new();
889 for requirement in requirements {
890 match verify(requirement, worktree, evidence)? {
891 VerificationOutcome::Passed(mut current) => collected.append(&mut current),
892 VerificationOutcome::Waiting => return Ok(VerificationOutcome::Waiting),
893 VerificationOutcome::Failed(error, mut current) => {
894 collected.append(&mut current);
895 return Ok(VerificationOutcome::Failed(error, collected));
896 }
897 }
898 }
899 Ok(VerificationOutcome::Passed(collected))
900 }
901 requirement => {
902 let kind = verification_kind(requirement);
903 match evidence.iter().rev().find(|item| item.kind == kind) {
904 Some(item) if evidence_satisfies(requirement, item) => {
905 Ok(VerificationOutcome::Passed(Vec::new()))
906 }
907 Some(item) if item.passed == Some(false) => Ok(VerificationOutcome::Failed(
908 format!("{kind} verification failed"),
909 Vec::new(),
910 )),
911 _ => Ok(VerificationOutcome::Waiting),
912 }
913 }
914 }
915}
916
917fn evidence_satisfies(requirement: &VerificationRequirement, evidence: &TaskEvidence) -> bool {
918 if evidence.passed != Some(true) {
919 return false;
920 }
921 match requirement {
922 VerificationRequirement::LspDiagnostics { max_errors } => evidence
923 .details
924 .get("errors")
925 .and_then(Value::as_u64)
926 .is_some_and(|errors| errors <= *max_errors),
927 VerificationRequirement::SemanticRegression {
928 maximum_regressions,
929 ..
930 } => evidence
931 .details
932 .get("regressions")
933 .and_then(Value::as_u64)
934 .is_some_and(|regressions| regressions <= *maximum_regressions),
935 VerificationRequirement::GitChange {
936 require_changes,
937 require_clean,
938 } => {
939 evidence.details.get("hasChanges").and_then(Value::as_bool) == Some(*require_changes)
940 && (!*require_clean
941 || evidence.details.get("clean").and_then(Value::as_bool) == Some(true))
942 }
943 _ => true,
944 }
945}
946
947fn verify_command(
948 worktree: &Path,
949 command: &str,
950 expected_exit: i32,
951 timeout_seconds: u64,
952) -> DevelopmentResult<VerificationOutcome> {
953 let mut process = if cfg!(windows) {
954 let mut command_process = Command::new("cmd.exe");
955 command_process.args(["/d", "/s", "/c", command]);
956 command_process
957 } else {
958 let mut command_process = Command::new("sh");
959 command_process.args(["-lc", command]);
960 command_process
961 };
962 let mut child = process
963 .current_dir(worktree)
964 .stdin(Stdio::null())
965 .stdout(Stdio::null())
966 .stderr(Stdio::null())
967 .spawn()?;
968 let deadline = Instant::now() + Duration::from_secs(timeout_seconds);
969 loop {
970 if let Some(status) = child.try_wait()? {
971 let actual = status.code().unwrap_or(-1);
972 let passed = actual == expected_exit;
973 let observed = TaskEvidence {
974 kind: "command".into(),
975 actor: "glass-task-verifier".into(),
976 source: "trusted-command".into(),
977 passed: Some(passed),
978 observed_at_ms: now_ms(),
979 details: serde_json::json!({
980 "command": command,
981 "expectedExit": expected_exit,
982 "actualExit": actual,
983 }),
984 };
985 return Ok(if passed {
986 VerificationOutcome::Passed(vec![observed])
987 } else {
988 VerificationOutcome::Failed(
989 format!("verification command exited {actual}, expected {expected_exit}"),
990 vec![observed],
991 )
992 });
993 }
994 if Instant::now() >= deadline {
995 child.kill()?;
996 child.wait()?;
997 return Ok(VerificationOutcome::Failed(
998 "verification command timed out".into(),
999 vec![TaskEvidence {
1000 kind: "command".into(),
1001 actor: "glass-task-verifier".into(),
1002 source: "trusted-command".into(),
1003 passed: Some(false),
1004 observed_at_ms: now_ms(),
1005 details: serde_json::json!({"command":command,"timedOut":true}),
1006 }],
1007 ));
1008 }
1009 thread::sleep(Duration::from_millis(10));
1010 }
1011}
1012
1013fn verification_kind(requirement: &VerificationRequirement) -> &'static str {
1014 match requirement {
1015 VerificationRequirement::LspDiagnostics { .. } => "lspDiagnostics",
1016 VerificationRequirement::BrowserWorkflow { .. } => "browserWorkflow",
1017 VerificationRequirement::SemanticRegression { .. } => "semanticRegression",
1018 VerificationRequirement::DebuggerAssertion { .. } => "debuggerAssertion",
1019 VerificationRequirement::GitChange { .. } => "gitChange",
1020 VerificationRequirement::TrustedCustom { .. } => "trustedCustom",
1021 _ => "unknown",
1022 }
1023}
1024
1025fn infer_verification(worktree: &Path) -> VerificationRequirement {
1026 let command = if worktree.join("Cargo.lock").is_file() {
1027 Some("cargo test --workspace --all-targets --locked")
1028 } else if worktree.join("Cargo.toml").is_file() {
1029 Some("cargo test --workspace --all-targets")
1030 } else if worktree.join("package.json").is_file() {
1031 Some("npm test")
1032 } else if worktree.join("pyproject.toml").is_file() || worktree.join("pytest.ini").is_file() {
1033 Some("python -m pytest")
1034 } else if worktree.join("go.mod").is_file() {
1035 Some("go test ./...")
1036 } else {
1037 None
1038 };
1039 command.map_or(
1040 VerificationRequirement::GitChange {
1041 require_changes: true,
1042 require_clean: false,
1043 },
1044 |command| VerificationRequirement::Command {
1045 command: command.into(),
1046 expected_exit: 0,
1047 timeout_seconds: 600,
1048 },
1049 )
1050}
1051
1052fn event_tokens(value: &Value) -> u64 {
1053 value
1054 .pointer("/message/usage/totalTokens")
1055 .or_else(|| value.pointer("/usage/totalTokens"))
1056 .and_then(Value::as_u64)
1057 .or_else(|| {
1058 let usage = value
1059 .pointer("/message/usage")
1060 .or_else(|| value.get("usage"))?;
1061 Some(
1062 ["input", "output", "cacheRead", "cacheWrite"]
1063 .iter()
1064 .filter_map(|name| usage.get(name).and_then(Value::as_u64))
1065 .fold(0_u64, u64::saturating_add),
1066 )
1067 })
1068 .unwrap_or(0)
1069}
1070
1071fn validate_verification(
1072 requirement: &VerificationRequirement,
1073 depth: usize,
1074) -> DevelopmentResult<()> {
1075 if depth > 8 {
1076 return Err(DevelopmentError::InvalidInput(
1077 "task verification nesting exceeds 8".into(),
1078 ));
1079 }
1080 match requirement {
1081 VerificationRequirement::Command {
1082 command,
1083 timeout_seconds,
1084 ..
1085 } => {
1086 if command.is_empty()
1087 || command.len() > MAX_VERIFIER_COMMAND_BYTES
1088 || command.contains('\0')
1089 || *timeout_seconds == 0
1090 || *timeout_seconds > 600
1091 {
1092 return Err(DevelopmentError::InvalidInput(
1093 "invalid task verification command".into(),
1094 ));
1095 }
1096 }
1097 VerificationRequirement::All { requirements } => {
1098 if requirements.is_empty() || requirements.len() > 16 {
1099 return Err(DevelopmentError::InvalidInput(
1100 "task verification group must contain 1..=16 requirements".into(),
1101 ));
1102 }
1103 for requirement in requirements {
1104 validate_verification(requirement, depth + 1)?;
1105 }
1106 }
1107 _ => {}
1108 }
1109 Ok(())
1110}
1111
1112fn push_evidence(record: &mut TaskRecord, evidence: TaskEvidence) {
1113 if record.snapshot.evidence.len() == MAX_TASK_EVIDENCE {
1114 record.snapshot.evidence.remove(0);
1115 }
1116 record.snapshot.evidence.push(evidence);
1117 record.snapshot.updated_at_ms = now_ms();
1118}
1119
1120fn validate_text(label: &str, value: &str) -> DevelopmentResult<()> {
1121 if value.is_empty() || value.len() > MAX_TASK_TEXT_BYTES || value.contains('\0') {
1122 return Err(DevelopmentError::InvalidInput(format!(
1123 "{label} must contain 1..={MAX_TASK_TEXT_BYTES} bytes without NUL"
1124 )));
1125 }
1126 Ok(())
1127}
1128
1129fn now_ms() -> u128 {
1130 SystemTime::now()
1131 .duration_since(UNIX_EPOCH)
1132 .unwrap_or_default()
1133 .as_millis()
1134}
1135
1136#[cfg(test)]
1137mod tests {
1138 use super::*;
1139 use std::sync::atomic::{AtomicU64, Ordering};
1140
1141 static NEXT_TEST_ROOT: AtomicU64 = AtomicU64::new(1);
1142
1143 #[derive(Default)]
1144 struct FakeAgents {
1145 next: u64,
1146 next_event: u64,
1147 snapshots: BTreeMap<AgentId, AgentSnapshot>,
1148 events: Vec<AgentEvent>,
1149 prompts: Vec<(AgentId, String)>,
1150 }
1151
1152 impl FakeAgents {
1153 fn emit(&mut self, id: &AgentId, kind: &str, payload: Value) {
1154 self.next_event += 1;
1155 self.events.push(AgentEvent {
1156 sequence: self.next_event,
1157 agent_id: id.clone(),
1158 timestamp_ms: now_ms(),
1159 kind: kind.into(),
1160 payload,
1161 });
1162 }
1163
1164 fn snapshot(id: AgentId, spec: AgentSpec) -> AgentSnapshot {
1165 AgentSnapshot {
1166 id,
1167 role: spec.role,
1168 task: spec.task,
1169 status: AgentStatus::Idle,
1170 dependencies: Vec::new(),
1171 model: spec.model,
1172 thinking: spec.thinking,
1173 worktree: spec.worktree.unwrap(),
1174 unrestricted: spec.unrestricted,
1175 created_at_ms: now_ms(),
1176 started_at_ms: Some(now_ms()),
1177 updated_at_ms: now_ms(),
1178 event_count: 0,
1179 dropped_event_count: 0,
1180 last_error: None,
1181 last_response_id: None,
1182 evidence: Vec::new(),
1183 }
1184 }
1185 }
1186
1187 impl TaskAgentBackend for FakeAgents {
1188 fn refresh_agents(&mut self) -> DevelopmentResult<()> {
1189 Ok(())
1190 }
1191
1192 fn create_agent(&mut self, spec: AgentSpec) -> DevelopmentResult<AgentId> {
1193 self.next += 1;
1194 let id = AgentId::parse(format!("agent-{:04}", self.next))?;
1195 self.snapshots
1196 .insert(id.clone(), Self::snapshot(id.clone(), spec));
1197 self.emit(&id, "ready", Value::Null);
1198 Ok(id)
1199 }
1200
1201 fn agent_snapshots(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>> {
1202 Ok(self.snapshots.values().cloned().collect())
1203 }
1204
1205 fn agent_events(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>> {
1206 Ok(self
1207 .events
1208 .iter()
1209 .filter(|event| event.sequence > since)
1210 .cloned()
1211 .collect())
1212 }
1213
1214 fn prompt_agent(&mut self, id: &AgentId, prompt: String) -> DevelopmentResult<()> {
1215 self.prompts.push((id.clone(), prompt));
1216 self.emit(id, "requestStarted", Value::String("request-1".into()));
1217 self.emit(id, "agent_settled", Value::Null);
1218 Ok(())
1219 }
1220
1221 fn cancel_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
1222 self.snapshots.get_mut(id).unwrap().status = AgentStatus::Cancelled;
1223 Ok(())
1224 }
1225
1226 fn complete_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
1227 self.snapshots.get_mut(id).unwrap().status = AgentStatus::Completed;
1228 Ok(())
1229 }
1230 }
1231
1232 fn scheduler() -> (PathBuf, TaskScheduler, FakeAgents) {
1233 let root = std::env::temp_dir().join(format!(
1234 "glass-task-scheduler-test-{}-{}",
1235 std::process::id(),
1236 NEXT_TEST_ROOT.fetch_add(1, Ordering::Relaxed)
1237 ));
1238 std::fs::create_dir_all(&root).unwrap();
1239 let scheduler = TaskScheduler::new(&root).unwrap();
1240 (root, scheduler, FakeAgents::default())
1241 }
1242
1243 fn settle(scheduler: &mut TaskScheduler, agents: &mut FakeAgents) {
1244 scheduler.refresh(agents).unwrap();
1245 scheduler.refresh(agents).unwrap();
1246 }
1247
1248 fn settle_only(title: impl Into<String>, prompt: impl Into<String>) -> TaskSpec {
1249 let mut spec = TaskSpec::new(title, prompt);
1250 spec.verification = VerificationRequirement::Settled;
1251 spec
1252 }
1253
1254 #[test]
1255 fn tasks_dispatch_prompts_verify_and_wake_dag_dependents() {
1256 let (root, mut scheduler, mut agents) = scheduler();
1257 let first = scheduler
1258 .create(&mut agents, settle_only("investigate", "inspect failure"))
1259 .unwrap();
1260 settle(&mut scheduler, &mut agents);
1261 assert_eq!(
1262 scheduler.snapshot(&mut agents, &first).unwrap().state,
1263 TaskState::Succeeded
1264 );
1265 assert_eq!(agents.prompts[0].1, "inspect failure");
1266
1267 let mut dependent = settle_only("repair", "apply repair");
1268 dependent.dependencies.push(first);
1269 let dependent = scheduler.create(&mut agents, dependent).unwrap();
1270 settle(&mut scheduler, &mut agents);
1271 assert_eq!(
1272 scheduler.snapshot(&mut agents, &dependent).unwrap().state,
1273 TaskState::Succeeded
1274 );
1275 assert_eq!(agents.prompts[1].1, "apply repair");
1276 std::fs::remove_dir_all(root).unwrap();
1277 }
1278
1279 #[test]
1280 fn eight_ready_tasks_dispatch_before_integration_dependency_wakes() {
1281 let (root, mut scheduler, mut agents) = scheduler();
1282 let mut leaves = Vec::new();
1283 for index in 0..8 {
1284 leaves.push(
1285 scheduler
1286 .create(
1287 &mut agents,
1288 settle_only(format!("worker-{index}"), format!("inspect shard {index}")),
1289 )
1290 .unwrap(),
1291 );
1292 }
1293 assert_eq!(scheduler.tasks.len(), 8);
1294
1295 let mut integration = settle_only("integration", "verify all shards");
1296 integration.dependencies = leaves.clone();
1297 let integration = scheduler.create(&mut agents, integration).unwrap();
1298 assert_eq!(scheduler.tasks.len(), 9);
1299 settle(&mut scheduler, &mut agents);
1300 settle(&mut scheduler, &mut agents);
1301
1302 for leaf in leaves {
1303 assert_eq!(
1304 scheduler.snapshot(&mut agents, &leaf).unwrap().state,
1305 TaskState::Succeeded
1306 );
1307 }
1308 assert_eq!(
1309 scheduler.snapshot(&mut agents, &integration).unwrap().state,
1310 TaskState::Succeeded
1311 );
1312 assert_eq!(agents.prompts.len(), 9);
1313 std::fs::remove_dir_all(root).unwrap();
1314 }
1315
1316 #[test]
1317 fn verification_failure_retries_then_blocks_descendants() {
1318 let (root, mut scheduler, mut agents) = scheduler();
1319 let mut failing = TaskSpec::new("verify", "produce candidate");
1320 failing.verification = VerificationRequirement::Command {
1321 command: "exit 7".into(),
1322 expected_exit: 0,
1323 timeout_seconds: 2,
1324 };
1325 failing.retry = RetryPolicy {
1326 max_retries: 1,
1327 backoff_seconds: 0,
1328 };
1329 let failing = scheduler.create(&mut agents, failing).unwrap();
1330 let mut child = TaskSpec::new("integration", "integrate candidate");
1331 child.dependencies.push(failing.clone());
1332 let child = scheduler.create(&mut agents, child).unwrap();
1333 for _ in 0..6 {
1334 scheduler.refresh(&mut agents).unwrap();
1335 }
1336 let failed = scheduler.snapshot(&mut agents, &failing).unwrap();
1337 assert_eq!(failed.state, TaskState::Failed);
1338 assert_eq!(failed.attempt, 2);
1339 assert_eq!(
1340 scheduler.snapshot(&mut agents, &child).unwrap().state,
1341 TaskState::Blocked
1342 );
1343 assert_eq!(agents.prompts.len(), 2);
1344 std::fs::remove_dir_all(root).unwrap();
1345 }
1346
1347 #[test]
1348 fn external_verification_evidence_is_proof_not_agent_claim() {
1349 let (root, mut scheduler, mut agents) = scheduler();
1350 let mut spec = TaskSpec::new("diagnostics", "repair diagnostics");
1351 spec.verification = VerificationRequirement::LspDiagnostics { max_errors: 0 };
1352 let id = scheduler.create(&mut agents, spec).unwrap();
1353 settle(&mut scheduler, &mut agents);
1354 assert_eq!(
1355 scheduler.snapshot(&mut agents, &id).unwrap().state,
1356 TaskState::Waiting
1357 );
1358 scheduler
1359 .submit_evidence(
1360 &id,
1361 "lspDiagnostics",
1362 "resident-lsp",
1363 "task-test",
1364 true,
1365 serde_json::json!({"errors":0,"source":"resident-lsp"}),
1366 )
1367 .unwrap();
1368 scheduler.refresh(&mut agents).unwrap();
1369 assert_eq!(
1370 scheduler.snapshot(&mut agents, &id).unwrap().state,
1371 TaskState::Succeeded
1372 );
1373 std::fs::remove_dir_all(root).unwrap();
1374 }
1375
1376 #[test]
1377 fn implementation_tasks_infer_project_verification_and_settle_only_is_explicit() {
1378 let (root, mut scheduler, mut agents) = scheduler();
1379 std::fs::write(
1380 root.join("Cargo.toml"),
1381 "[package]\nname='verification-fixture'\nversion='0.1.0'\n",
1382 )
1383 .unwrap();
1384 let inferred = scheduler
1385 .create(
1386 &mut agents,
1387 TaskSpec::new("implement repair", "change the implementation"),
1388 )
1389 .unwrap();
1390 assert!(matches!(
1391 scheduler.tasks[&inferred].snapshot.verification,
1392 VerificationRequirement::Command { ref command, .. }
1393 if command == "cargo test --workspace --all-targets"
1394 ));
1395
1396 let research = scheduler
1397 .create(
1398 &mut agents,
1399 settle_only("research alternatives", "compare approaches"),
1400 )
1401 .unwrap();
1402 assert_eq!(
1403 scheduler.tasks[&research].snapshot.verification,
1404 VerificationRequirement::Settled
1405 );
1406 std::fs::remove_dir_all(root).unwrap();
1407 }
1408}