1use crate::agents::{AgentEvent, AgentId, AgentRegistry, AgentSnapshot, AgentSpec, AgentStatus};
4use crate::development::{DevelopmentError, DevelopmentResult};
5use serde::{Deserialize, Serialize};
6use serde_json::Value;
7use std::collections::{BTreeMap, BTreeSet};
8use std::path::{Path, PathBuf};
9use std::process::{Command, Stdio};
10use std::thread;
11use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
12
13const MAX_TASKS: usize = 128;
14const MAX_DEPENDENCIES: usize = 32;
15const MAX_TASK_EVIDENCE: usize = 128;
16const MAX_TASK_TEXT_BYTES: usize = 128 * 1024;
17const MAX_VERIFIER_COMMAND_BYTES: usize = 16 * 1024;
18
19#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
20pub struct TaskId(String);
21
22impl TaskId {
23 pub fn as_str(&self) -> &str {
24 &self.0
25 }
26
27 pub fn parse(value: impl Into<String>) -> DevelopmentResult<Self> {
28 let value = value.into();
29 if !value.starts_with("task-")
30 || value.len() > 64
31 || value.chars().any(|character| character.is_control())
32 {
33 return Err(DevelopmentError::InvalidInput("invalid task id".into()));
34 }
35 Ok(Self(value))
36 }
37}
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
40#[serde(rename_all = "camelCase")]
41pub enum TaskState {
42 Queued,
43 Ready,
44 Running,
45 Waiting,
46 Verifying,
47 Succeeded,
48 Failed,
49 Cancelled,
50 Paused,
51 Blocked,
52}
53
54impl TaskState {
55 pub fn label(&self) -> &'static str {
57 match self {
58 Self::Queued => "queued",
59 Self::Ready => "ready",
60 Self::Running => "running",
61 Self::Waiting => "waiting",
62 Self::Verifying => "verifying",
63 Self::Succeeded => "succeeded",
64 Self::Failed => "failed",
65 Self::Cancelled => "cancelled",
66 Self::Paused => "paused",
67 Self::Blocked => "blocked",
68 }
69 }
70}
71
72impl TaskState {
73 fn terminal(self) -> bool {
74 matches!(self, Self::Succeeded | Self::Failed | Self::Cancelled)
75 }
76}
77
78#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
79#[serde(rename_all = "camelCase")]
80pub struct TaskBudget {
81 pub max_runtime_seconds: u64,
82 pub max_events: u64,
83 pub max_tokens: Option<u64>,
84}
85
86impl Default for TaskBudget {
87 fn default() -> Self {
88 Self {
89 max_runtime_seconds: 3_600,
90 max_events: 10_000,
91 max_tokens: None,
92 }
93 }
94}
95
96#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
97#[serde(rename_all = "camelCase")]
98pub struct RetryPolicy {
99 pub max_retries: u32,
100 pub backoff_seconds: u64,
101}
102
103impl Default for RetryPolicy {
104 fn default() -> Self {
105 Self {
106 max_retries: 0,
107 backoff_seconds: 1,
108 }
109 }
110}
111
112#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
113#[serde(tag = "kind", rename_all = "camelCase")]
114pub enum VerificationRequirement {
115 #[default]
117 Inferred,
118 Settled,
120 Command {
121 command: String,
122 expected_exit: i32,
123 timeout_seconds: u64,
124 },
125 LspDiagnostics {
126 max_errors: u64,
127 },
128 BrowserWorkflow {
129 assertion: String,
130 },
131 SemanticRegression {
132 baseline: String,
133 maximum_regressions: u64,
134 },
135 DebuggerAssertion {
136 session: String,
137 expression: String,
138 },
139 GitChange {
140 require_changes: bool,
141 require_clean: bool,
142 },
143 TrustedCustom {
144 name: String,
145 },
146 All {
147 requirements: Vec<VerificationRequirement>,
148 },
149}
150
151#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
152#[serde(rename_all = "camelCase")]
153pub struct TaskEvidence {
154 pub kind: String,
155 pub actor: String,
156 pub source: String,
157 pub passed: Option<bool>,
158 pub observed_at_ms: u128,
159 pub details: Value,
160}
161
162#[derive(Debug, Clone, Serialize, Deserialize)]
163#[serde(rename_all = "camelCase")]
164pub struct TaskSpec {
165 pub title: String,
166 pub goal: String,
167 pub prompt: String,
168 #[serde(default = "default_role")]
169 pub role: String,
170 #[serde(default)]
171 pub dependencies: Vec<TaskId>,
172 pub worktree: Option<PathBuf>,
173 pub model: Option<String>,
174 pub thinking: Option<String>,
175 #[serde(default)]
176 pub unrestricted: bool,
177 #[serde(default)]
178 pub budget: TaskBudget,
179 #[serde(default)]
180 pub verification: VerificationRequirement,
181 #[serde(default)]
182 pub retry: RetryPolicy,
183}
184
185fn default_role() -> String {
186 "developer".into()
187}
188
189impl TaskSpec {
190 pub fn new(title: impl Into<String>, prompt: impl Into<String>) -> Self {
191 let title = title.into();
192 Self {
193 goal: title.clone(),
194 title,
195 prompt: prompt.into(),
196 role: default_role(),
197 dependencies: Vec::new(),
198 worktree: None,
199 model: None,
200 thinking: None,
201 unrestricted: false,
202 budget: TaskBudget::default(),
203 verification: VerificationRequirement::default(),
204 retry: RetryPolicy::default(),
205 }
206 }
207}
208
209#[derive(Debug, Clone, Serialize, Deserialize)]
210#[serde(rename_all = "camelCase")]
211pub struct TaskSnapshot {
212 pub id: TaskId,
213 pub title: String,
214 pub goal: String,
215 pub prompt: String,
216 pub role: String,
217 pub dependencies: Vec<TaskId>,
218 pub assigned_agent: Option<AgentId>,
219 pub worktree: PathBuf,
220 pub model: Option<String>,
221 pub thinking: Option<String>,
222 pub unrestricted: bool,
223 pub budget: TaskBudget,
224 pub verification: VerificationRequirement,
225 pub retry: RetryPolicy,
226 pub state: TaskState,
227 pub attempt: u32,
228 pub observed_tokens: u64,
229 pub created_at_ms: u128,
230 pub started_at_ms: Option<u128>,
231 pub completed_at_ms: Option<u128>,
232 pub updated_at_ms: u128,
233 pub last_error: Option<String>,
234 pub evidence: Vec<TaskEvidence>,
235 pub blocked_override: bool,
236}
237
238struct TaskRecord {
239 snapshot: TaskSnapshot,
240 next_retry_at_ms: Option<u128>,
241}
242
243pub trait TaskAgentBackend {
244 fn refresh_agents(&mut self) -> DevelopmentResult<()>;
245 fn create_agent(&mut self, spec: AgentSpec) -> DevelopmentResult<AgentId>;
246 fn agent_snapshots(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>>;
247 fn agent_events(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>>;
248 fn prompt_agent(&mut self, id: &AgentId, prompt: String) -> DevelopmentResult<()>;
249 fn cancel_agent(&mut self, id: &AgentId) -> DevelopmentResult<()>;
250 fn complete_agent(&mut self, id: &AgentId) -> DevelopmentResult<()>;
251}
252
253impl TaskAgentBackend for AgentRegistry {
254 fn refresh_agents(&mut self) -> DevelopmentResult<()> {
255 self.refresh()
256 }
257
258 fn create_agent(&mut self, spec: AgentSpec) -> DevelopmentResult<AgentId> {
259 self.create(spec)
260 }
261
262 fn agent_snapshots(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>> {
263 self.list()
264 }
265
266 fn agent_events(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>> {
267 self.history(since)
268 }
269
270 fn prompt_agent(&mut self, id: &AgentId, prompt: String) -> DevelopmentResult<()> {
271 self.prompt(id, prompt)
272 }
273
274 fn cancel_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
275 self.cancel(id)
276 }
277
278 fn complete_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
279 self.complete(id)
280 }
281}
282
283pub struct TaskScheduler {
284 root: PathBuf,
285 tasks: BTreeMap<TaskId, TaskRecord>,
286 next_task: u64,
287 agent_event_cursor: u64,
288}
289
290impl TaskScheduler {
291 pub fn new(root: impl AsRef<Path>) -> DevelopmentResult<Self> {
292 Ok(Self {
293 root: std::fs::canonicalize(root)?,
294 tasks: BTreeMap::new(),
295 next_task: 1,
296 agent_event_cursor: 0,
297 })
298 }
299
300 pub fn create<B: TaskAgentBackend>(
301 &mut self,
302 agents: &mut B,
303 mut spec: TaskSpec,
304 ) -> DevelopmentResult<TaskId> {
305 self.refresh(agents)?;
306 if spec.verification == VerificationRequirement::Inferred {
307 let worktree = spec.worktree.as_deref().unwrap_or(&self.root);
308 spec.verification = infer_verification(worktree);
309 }
310 self.validate_spec(&spec)?;
311 if self.tasks.len() >= MAX_TASKS {
312 return Err(DevelopmentError::Conflict(format!(
313 "task scheduler reached its {MAX_TASKS} task limit"
314 )));
315 }
316 let id = TaskId(format!("task-{:04}", self.next_task));
317 self.next_task = self
318 .next_task
319 .checked_add(1)
320 .ok_or_else(|| DevelopmentError::Conflict("task identifier space exhausted".into()))?;
321 let now = now_ms();
322 self.tasks.insert(
323 id.clone(),
324 TaskRecord {
325 snapshot: TaskSnapshot {
326 id: id.clone(),
327 title: spec.title,
328 goal: spec.goal,
329 prompt: spec.prompt,
330 role: spec.role,
331 dependencies: spec.dependencies,
332 assigned_agent: None,
333 worktree: spec.worktree.unwrap_or_else(|| self.root.clone()),
334 model: spec.model,
335 thinking: spec.thinking,
336 unrestricted: spec.unrestricted,
337 budget: spec.budget,
338 verification: spec.verification,
339 retry: spec.retry,
340 state: TaskState::Queued,
341 attempt: 0,
342 observed_tokens: 0,
343 created_at_ms: now,
344 started_at_ms: None,
345 completed_at_ms: None,
346 updated_at_ms: now,
347 last_error: None,
348 evidence: Vec::new(),
349 blocked_override: false,
350 },
351 next_retry_at_ms: None,
352 },
353 );
354 self.schedule_ready(agents)?;
355 Ok(id)
356 }
357
358 pub fn refresh<B: TaskAgentBackend>(&mut self, agents: &mut B) -> DevelopmentResult<()> {
359 agents.refresh_agents()?;
360 let snapshots = agents.agent_snapshots()?;
361 let events = agents.agent_events(self.agent_event_cursor)?;
362 for event in events {
363 self.agent_event_cursor = self.agent_event_cursor.max(event.sequence);
364 self.apply_agent_event(agents, &event)?;
365 }
366 self.apply_agent_failures(&snapshots);
367 self.enforce_task_budgets(agents)?;
368 self.verify_pending(agents)?;
369 self.propagate_blocked();
370 self.schedule_ready(agents)
371 }
372
373 pub fn list<B: TaskAgentBackend>(
374 &mut self,
375 agents: &mut B,
376 ) -> DevelopmentResult<Vec<TaskSnapshot>> {
377 self.refresh(agents)?;
378 Ok(self
379 .tasks
380 .values()
381 .map(|record| record.snapshot.clone())
382 .collect())
383 }
384
385 pub fn snapshot<B: TaskAgentBackend>(
386 &mut self,
387 agents: &mut B,
388 id: &TaskId,
389 ) -> DevelopmentResult<TaskSnapshot> {
390 self.refresh(agents)?;
391 self.tasks
392 .get(id)
393 .map(|record| record.snapshot.clone())
394 .ok_or_else(|| DevelopmentError::NotFound(format!("task {}", id.as_str())))
395 }
396
397 pub fn submit_evidence(
398 &mut self,
399 id: &TaskId,
400 kind: impl Into<String>,
401 actor: impl Into<String>,
402 source: impl Into<String>,
403 passed: bool,
404 details: Value,
405 ) -> DevelopmentResult<()> {
406 let record = self.record_mut(id)?;
407 push_evidence(
408 record,
409 TaskEvidence {
410 kind: kind.into(),
411 actor: actor.into(),
412 source: source.into(),
413 passed: Some(passed),
414 observed_at_ms: now_ms(),
415 details,
416 },
417 );
418 if record.snapshot.state == TaskState::Waiting {
419 record.snapshot.state = TaskState::Verifying;
420 }
421 Ok(())
422 }
423
424 pub fn pause<B: TaskAgentBackend>(
425 &mut self,
426 agents: &mut B,
427 id: &TaskId,
428 ) -> DevelopmentResult<()> {
429 self.refresh(agents)?;
430 let agent = self.record(id)?.snapshot.assigned_agent.clone();
431 if let Some(agent) = agent {
432 agents.cancel_agent(&agent)?;
433 }
434 let record = self.record_mut(id)?;
435 if record.snapshot.state.terminal() {
436 return Err(DevelopmentError::Conflict(
437 "terminal task cannot be paused".into(),
438 ));
439 }
440 record.snapshot.assigned_agent = None;
441 record.snapshot.state = TaskState::Paused;
442 record.snapshot.updated_at_ms = now_ms();
443 Ok(())
444 }
445
446 pub fn resume<B: TaskAgentBackend>(
447 &mut self,
448 agents: &mut B,
449 id: &TaskId,
450 ) -> DevelopmentResult<()> {
451 let record = self.record_mut(id)?;
452 if record.snapshot.state != TaskState::Paused {
453 return Err(DevelopmentError::Conflict("task is not paused".into()));
454 }
455 record.snapshot.state = TaskState::Queued;
456 record.snapshot.updated_at_ms = now_ms();
457 self.schedule_ready(agents)
458 }
459
460 pub fn cancel<B: TaskAgentBackend>(
461 &mut self,
462 agents: &mut B,
463 id: &TaskId,
464 ) -> DevelopmentResult<()> {
465 self.refresh(agents)?;
466 let agent = self.record(id)?.snapshot.assigned_agent.clone();
467 if let Some(agent) = agent {
468 agents.cancel_agent(&agent)?;
469 }
470 let record = self.record_mut(id)?;
471 if !record.snapshot.state.terminal() {
472 record.snapshot.state = TaskState::Cancelled;
473 record.snapshot.completed_at_ms = Some(now_ms());
474 record.snapshot.updated_at_ms = now_ms();
475 }
476 self.propagate_blocked();
477 Ok(())
478 }
479
480 pub fn retry<B: TaskAgentBackend>(
481 &mut self,
482 agents: &mut B,
483 id: &TaskId,
484 ) -> DevelopmentResult<()> {
485 let record = self.record_mut(id)?;
486 if !matches!(
487 record.snapshot.state,
488 TaskState::Failed | TaskState::Blocked
489 ) {
490 return Err(DevelopmentError::Conflict(
491 "only failed or blocked tasks can be retried".into(),
492 ));
493 }
494 record.snapshot.state = TaskState::Queued;
495 record.snapshot.assigned_agent = None;
496 record.snapshot.completed_at_ms = None;
497 record.snapshot.last_error = None;
498 record.snapshot.updated_at_ms = now_ms();
499 record.next_retry_at_ms = None;
500 self.schedule_ready(agents)
501 }
502
503 pub fn reassign<B: TaskAgentBackend>(
504 &mut self,
505 agents: &mut B,
506 id: &TaskId,
507 role: String,
508 model: Option<String>,
509 thinking: Option<String>,
510 ) -> DevelopmentResult<()> {
511 validate_text("task role", &role)?;
512 let agent = self.record(id)?.snapshot.assigned_agent.clone();
513 if let Some(agent) = agent {
514 agents.cancel_agent(&agent)?;
515 }
516 let record = self.record_mut(id)?;
517 if record.snapshot.state.terminal() {
518 return Err(DevelopmentError::Conflict(
519 "terminal task cannot be reassigned".into(),
520 ));
521 }
522 record.snapshot.role = role;
523 record.snapshot.model = model;
524 record.snapshot.thinking = thinking;
525 record.snapshot.assigned_agent = None;
526 record.snapshot.state = TaskState::Queued;
527 record.snapshot.updated_at_ms = now_ms();
528 self.schedule_ready(agents)
529 }
530
531 pub fn override_blocked<B: TaskAgentBackend>(
532 &mut self,
533 agents: &mut B,
534 id: &TaskId,
535 ) -> DevelopmentResult<()> {
536 let record = self.record_mut(id)?;
537 if record.snapshot.state != TaskState::Blocked {
538 return Err(DevelopmentError::Conflict("task is not blocked".into()));
539 }
540 record.snapshot.blocked_override = true;
541 record.snapshot.state = TaskState::Queued;
542 record.snapshot.last_error = None;
543 record.snapshot.updated_at_ms = now_ms();
544 self.schedule_ready(agents)
545 }
546
547 fn schedule_ready<B: TaskAgentBackend>(&mut self, agents: &mut B) -> DevelopmentResult<()> {
548 let now = now_ms();
549 let ready = self
550 .tasks
551 .iter()
552 .filter(|(_, record)| record.snapshot.state == TaskState::Queued)
553 .filter(|(_, record)| record.next_retry_at_ms.is_none_or(|at| at <= now))
554 .filter(|(_, record)| {
555 record.snapshot.blocked_override
556 || record.snapshot.dependencies.iter().all(|dependency| {
557 self.tasks.get(dependency).is_some_and(|dependency| {
558 dependency.snapshot.state == TaskState::Succeeded
559 })
560 })
561 })
562 .map(|(id, _)| id.clone())
563 .collect::<Vec<_>>();
564 for id in ready {
565 let record = self.record(&id)?;
566 let mut spec = AgentSpec::new(
567 record.snapshot.role.clone(),
568 format!("task {}: {}", id.as_str(), record.snapshot.title),
569 );
570 spec.model.clone_from(&record.snapshot.model);
571 spec.thinking.clone_from(&record.snapshot.thinking);
572 spec.worktree = Some(record.snapshot.worktree.clone());
573 spec.unrestricted = record.snapshot.unrestricted;
574 spec.max_runtime_seconds = Some(record.snapshot.budget.max_runtime_seconds);
575 spec.max_events = Some(record.snapshot.budget.max_events);
576 let agent = agents.create_agent(spec)?;
577 let record = self.record_mut(&id)?;
578 record.snapshot.assigned_agent = Some(agent);
579 record.snapshot.state = TaskState::Ready;
580 record.snapshot.attempt = record.snapshot.attempt.saturating_add(1);
581 record.snapshot.updated_at_ms = now_ms();
582 record.next_retry_at_ms = None;
583 }
584 Ok(())
585 }
586
587 fn apply_agent_event<B: TaskAgentBackend>(
588 &mut self,
589 agents: &mut B,
590 event: &AgentEvent,
591 ) -> DevelopmentResult<()> {
592 let Some(task_id) = self.task_for_agent(&event.agent_id) else {
593 return Ok(());
594 };
595 let state = self.record(&task_id)?.snapshot.state;
596 match event.kind.as_str() {
597 "ready" if state == TaskState::Ready => {
598 let prompt = self.record(&task_id)?.snapshot.prompt.clone();
599 agents.prompt_agent(&event.agent_id, prompt)?;
600 let record = self.record_mut(&task_id)?;
601 record.snapshot.state = TaskState::Running;
602 record.snapshot.started_at_ms.get_or_insert_with(now_ms);
603 record.snapshot.updated_at_ms = now_ms();
604 }
605 "agent_settled" if state == TaskState::Running => {
606 let record = self.record_mut(&task_id)?;
607 record.snapshot.state = TaskState::Verifying;
608 record.snapshot.updated_at_ms = now_ms();
609 push_evidence(
610 record,
611 TaskEvidence {
612 kind: "agent.settled".into(),
613 actor: event.agent_id.as_str().into(),
614 source: "pi-agent-session".into(),
615 passed: Some(true),
616 observed_at_ms: event.timestamp_ms,
617 details: event.payload.clone(),
618 },
619 );
620 }
621 kind => {
622 let record = self.record_mut(&task_id)?;
623 if kind == "message_end" {
624 record.snapshot.observed_tokens = record
625 .snapshot
626 .observed_tokens
627 .saturating_add(event_tokens(&event.payload));
628 }
629 push_evidence(
630 record,
631 TaskEvidence {
632 kind: format!("agent.{kind}"),
633 actor: event.agent_id.as_str().into(),
634 source: "pi-agent-session".into(),
635 passed: None,
636 observed_at_ms: event.timestamp_ms,
637 details: event.payload.clone(),
638 },
639 );
640 }
641 }
642 Ok(())
643 }
644
645 fn apply_agent_failures(&mut self, snapshots: &[AgentSnapshot]) {
646 for snapshot in snapshots {
647 if !matches!(
648 snapshot.status,
649 AgentStatus::Failed | AgentStatus::Cancelled
650 ) {
651 continue;
652 }
653 let Some(id) = self.task_for_agent(&snapshot.id) else {
654 continue;
655 };
656 let record = self.tasks.get_mut(&id).expect("task exists");
657 if !record.snapshot.state.terminal() {
658 record.snapshot.state = TaskState::Failed;
659 record.snapshot.last_error = snapshot
660 .last_error
661 .clone()
662 .or_else(|| Some("assigned agent stopped before verification".into()));
663 record.snapshot.completed_at_ms = Some(now_ms());
664 record.snapshot.updated_at_ms = now_ms();
665 }
666 }
667 }
668
669 fn verify_pending<B: TaskAgentBackend>(&mut self, agents: &mut B) -> DevelopmentResult<()> {
670 let pending = self
671 .tasks
672 .iter()
673 .filter(|(_, record)| record.snapshot.state == TaskState::Verifying)
674 .map(|(id, _)| id.clone())
675 .collect::<Vec<_>>();
676 for id in pending {
677 let (requirement, worktree, evidence) = {
678 let record = self.record(&id)?;
679 (
680 record.snapshot.verification.clone(),
681 record.snapshot.worktree.clone(),
682 record.snapshot.evidence.clone(),
683 )
684 };
685 match verify(&requirement, &worktree, &evidence)? {
686 VerificationOutcome::Passed(new_evidence) => {
687 let agent = self.record(&id)?.snapshot.assigned_agent.clone();
688 if let Some(agent) = agent {
689 agents.complete_agent(&agent)?;
690 }
691 let record = self.record_mut(&id)?;
692 for evidence in new_evidence {
693 push_evidence(record, evidence);
694 }
695 record.snapshot.state = TaskState::Succeeded;
696 record.snapshot.completed_at_ms = Some(now_ms());
697 record.snapshot.updated_at_ms = now_ms();
698 record.snapshot.last_error = None;
699 }
700 VerificationOutcome::Waiting => {
701 let record = self.record_mut(&id)?;
702 record.snapshot.state = TaskState::Waiting;
703 record.snapshot.updated_at_ms = now_ms();
704 }
705 VerificationOutcome::Failed(error, new_evidence) => {
706 let (agent, retry, attempt) = {
707 let record = self.record(&id)?;
708 (
709 record.snapshot.assigned_agent.clone(),
710 record.snapshot.retry.clone(),
711 record.snapshot.attempt,
712 )
713 };
714 if let Some(agent) = agent {
715 agents.cancel_agent(&agent)?;
716 }
717 let record = self.record_mut(&id)?;
718 for evidence in new_evidence {
719 push_evidence(record, evidence);
720 }
721 record.snapshot.assigned_agent = None;
722 record.snapshot.last_error = Some(error);
723 record.snapshot.updated_at_ms = now_ms();
724 if attempt <= retry.max_retries {
725 record.snapshot.state = TaskState::Queued;
726 record.next_retry_at_ms = Some(
727 now_ms().saturating_add(u128::from(retry.backoff_seconds) * 1_000),
728 );
729 } else {
730 record.snapshot.state = TaskState::Failed;
731 record.snapshot.completed_at_ms = Some(now_ms());
732 }
733 }
734 }
735 }
736 Ok(())
737 }
738
739 fn enforce_task_budgets<B: TaskAgentBackend>(
740 &mut self,
741 agents: &mut B,
742 ) -> DevelopmentResult<()> {
743 let now = now_ms();
744 let exceeded = self
745 .tasks
746 .iter()
747 .filter(|(_, record)| {
748 matches!(
749 record.snapshot.state,
750 TaskState::Running | TaskState::Waiting
751 )
752 })
753 .filter(|(_, record)| {
754 record.snapshot.started_at_ms.is_some_and(|started| {
755 now.saturating_sub(started)
756 > u128::from(record.snapshot.budget.max_runtime_seconds) * 1_000
757 }) || record
758 .snapshot
759 .budget
760 .max_tokens
761 .is_some_and(|limit| record.snapshot.observed_tokens >= limit)
762 })
763 .map(|(id, _)| id.clone())
764 .collect::<Vec<_>>();
765 for id in exceeded {
766 if let Some(agent) = self.record(&id)?.snapshot.assigned_agent.clone() {
767 agents.cancel_agent(&agent)?;
768 }
769 let record = self.record_mut(&id)?;
770 record.snapshot.state = TaskState::Failed;
771 record.snapshot.last_error = Some("task runtime or token budget exceeded".into());
772 record.snapshot.completed_at_ms = Some(now);
773 record.snapshot.updated_at_ms = now;
774 }
775 Ok(())
776 }
777
778 fn propagate_blocked(&mut self) {
779 loop {
780 let failed = self
781 .tasks
782 .iter()
783 .filter(|(_, record)| {
784 matches!(
785 record.snapshot.state,
786 TaskState::Failed | TaskState::Cancelled | TaskState::Blocked
787 )
788 })
789 .map(|(id, _)| id.clone())
790 .collect::<BTreeSet<_>>();
791 let blocked = self
792 .tasks
793 .iter()
794 .filter(|(_, record)| {
795 record.snapshot.state == TaskState::Queued && !record.snapshot.blocked_override
796 })
797 .filter(|(_, record)| {
798 record
799 .snapshot
800 .dependencies
801 .iter()
802 .any(|dependency| failed.contains(dependency))
803 })
804 .map(|(id, _)| id.clone())
805 .collect::<Vec<_>>();
806 if blocked.is_empty() {
807 break;
808 }
809 for id in blocked {
810 let record = self.tasks.get_mut(&id).expect("task exists");
811 record.snapshot.state = TaskState::Blocked;
812 record.snapshot.last_error = Some("a prerequisite task did not succeed".into());
813 record.snapshot.updated_at_ms = now_ms();
814 }
815 }
816 }
817
818 fn validate_spec(&self, spec: &TaskSpec) -> DevelopmentResult<()> {
819 for (label, value) in [
820 ("task title", spec.title.as_str()),
821 ("task goal", spec.goal.as_str()),
822 ("task prompt", spec.prompt.as_str()),
823 ("task role", spec.role.as_str()),
824 ] {
825 validate_text(label, value)?;
826 }
827 if spec.dependencies.len() > MAX_DEPENDENCIES {
828 return Err(DevelopmentError::InvalidInput(
829 "task has too many dependencies".into(),
830 ));
831 }
832 let mut unique = BTreeSet::new();
833 for dependency in &spec.dependencies {
834 if !unique.insert(dependency) {
835 return Err(DevelopmentError::InvalidInput(format!(
836 "duplicate task dependency {}",
837 dependency.as_str()
838 )));
839 }
840 if !self.tasks.contains_key(dependency) {
841 return Err(DevelopmentError::NotFound(format!(
842 "task dependency {}",
843 dependency.as_str()
844 )));
845 }
846 }
847 if spec.budget.max_runtime_seconds == 0 || spec.budget.max_events == 0 {
848 return Err(DevelopmentError::InvalidInput(
849 "task budgets must be positive".into(),
850 ));
851 }
852 validate_verification(&spec.verification, 0)?;
853 if let Some(worktree) = &spec.worktree {
854 let worktree = std::fs::canonicalize(worktree)?;
855 if worktree == Path::new("/")
856 || (!worktree.starts_with(&self.root) && !worktree.join(".git").exists())
857 {
858 return Err(DevelopmentError::PathOutsideWorkspace(worktree));
859 }
860 }
861 Ok(())
862 }
863
864 fn task_for_agent(&self, agent: &AgentId) -> Option<TaskId> {
865 self.tasks
866 .iter()
867 .find(|(_, record)| record.snapshot.assigned_agent.as_ref() == Some(agent))
868 .map(|(id, _)| id.clone())
869 }
870
871 fn record(&self, id: &TaskId) -> DevelopmentResult<&TaskRecord> {
872 self.tasks
873 .get(id)
874 .ok_or_else(|| DevelopmentError::NotFound(format!("task {}", id.as_str())))
875 }
876
877 fn record_mut(&mut self, id: &TaskId) -> DevelopmentResult<&mut TaskRecord> {
878 self.tasks
879 .get_mut(id)
880 .ok_or_else(|| DevelopmentError::NotFound(format!("task {}", id.as_str())))
881 }
882}
883
884enum VerificationOutcome {
885 Passed(Vec<TaskEvidence>),
886 Waiting,
887 Failed(String, Vec<TaskEvidence>),
888}
889
890fn verify(
891 requirement: &VerificationRequirement,
892 worktree: &Path,
893 evidence: &[TaskEvidence],
894) -> DevelopmentResult<VerificationOutcome> {
895 match requirement {
896 VerificationRequirement::Inferred => Err(DevelopmentError::Conflict(
897 "inferred task verification was not resolved at creation".into(),
898 )),
899 VerificationRequirement::Settled => Ok(VerificationOutcome::Passed(Vec::new())),
900 VerificationRequirement::Command {
901 command,
902 expected_exit,
903 timeout_seconds,
904 } => verify_command(worktree, command, *expected_exit, *timeout_seconds),
905 VerificationRequirement::All { requirements } => {
906 let mut collected = Vec::new();
907 for requirement in requirements {
908 match verify(requirement, worktree, evidence)? {
909 VerificationOutcome::Passed(mut current) => collected.append(&mut current),
910 VerificationOutcome::Waiting => return Ok(VerificationOutcome::Waiting),
911 VerificationOutcome::Failed(error, mut current) => {
912 collected.append(&mut current);
913 return Ok(VerificationOutcome::Failed(error, collected));
914 }
915 }
916 }
917 Ok(VerificationOutcome::Passed(collected))
918 }
919 requirement => {
920 let kind = verification_kind(requirement);
921 match evidence.iter().rev().find(|item| item.kind == kind) {
922 Some(item) if evidence_satisfies(requirement, item) => {
923 Ok(VerificationOutcome::Passed(Vec::new()))
924 }
925 Some(item) if item.passed == Some(false) => Ok(VerificationOutcome::Failed(
926 format!("{kind} verification failed"),
927 Vec::new(),
928 )),
929 _ => Ok(VerificationOutcome::Waiting),
930 }
931 }
932 }
933}
934
935fn evidence_satisfies(requirement: &VerificationRequirement, evidence: &TaskEvidence) -> bool {
936 if evidence.passed != Some(true) {
937 return false;
938 }
939 match requirement {
940 VerificationRequirement::LspDiagnostics { max_errors } => evidence
941 .details
942 .get("errors")
943 .and_then(Value::as_u64)
944 .is_some_and(|errors| errors <= *max_errors),
945 VerificationRequirement::SemanticRegression {
946 maximum_regressions,
947 ..
948 } => evidence
949 .details
950 .get("regressions")
951 .and_then(Value::as_u64)
952 .is_some_and(|regressions| regressions <= *maximum_regressions),
953 VerificationRequirement::GitChange {
954 require_changes,
955 require_clean,
956 } => {
957 evidence.details.get("hasChanges").and_then(Value::as_bool) == Some(*require_changes)
958 && (!*require_clean
959 || evidence.details.get("clean").and_then(Value::as_bool) == Some(true))
960 }
961 _ => true,
962 }
963}
964
965fn verify_command(
966 worktree: &Path,
967 command: &str,
968 expected_exit: i32,
969 timeout_seconds: u64,
970) -> DevelopmentResult<VerificationOutcome> {
971 let mut process = if cfg!(windows) {
972 let mut command_process = Command::new("cmd.exe");
973 command_process.args(["/d", "/s", "/c", command]);
974 command_process
975 } else {
976 let mut command_process = Command::new("sh");
977 command_process.args(["-lc", command]);
978 command_process
979 };
980 let mut child = process
981 .current_dir(worktree)
982 .stdin(Stdio::null())
983 .stdout(Stdio::null())
984 .stderr(Stdio::null())
985 .spawn()?;
986 let deadline = Instant::now() + Duration::from_secs(timeout_seconds);
987 loop {
988 if let Some(status) = child.try_wait()? {
989 let actual = status.code().unwrap_or(-1);
990 let passed = actual == expected_exit;
991 let observed = TaskEvidence {
992 kind: "command".into(),
993 actor: "glass-task-verifier".into(),
994 source: "trusted-command".into(),
995 passed: Some(passed),
996 observed_at_ms: now_ms(),
997 details: serde_json::json!({
998 "command": command,
999 "expectedExit": expected_exit,
1000 "actualExit": actual,
1001 }),
1002 };
1003 return Ok(if passed {
1004 VerificationOutcome::Passed(vec![observed])
1005 } else {
1006 VerificationOutcome::Failed(
1007 format!("verification command exited {actual}, expected {expected_exit}"),
1008 vec![observed],
1009 )
1010 });
1011 }
1012 if Instant::now() >= deadline {
1013 child.kill()?;
1014 child.wait()?;
1015 return Ok(VerificationOutcome::Failed(
1016 "verification command timed out".into(),
1017 vec![TaskEvidence {
1018 kind: "command".into(),
1019 actor: "glass-task-verifier".into(),
1020 source: "trusted-command".into(),
1021 passed: Some(false),
1022 observed_at_ms: now_ms(),
1023 details: serde_json::json!({"command":command,"timedOut":true}),
1024 }],
1025 ));
1026 }
1027 thread::sleep(Duration::from_millis(10));
1028 }
1029}
1030
1031fn verification_kind(requirement: &VerificationRequirement) -> &'static str {
1032 match requirement {
1033 VerificationRequirement::LspDiagnostics { .. } => "lspDiagnostics",
1034 VerificationRequirement::BrowserWorkflow { .. } => "browserWorkflow",
1035 VerificationRequirement::SemanticRegression { .. } => "semanticRegression",
1036 VerificationRequirement::DebuggerAssertion { .. } => "debuggerAssertion",
1037 VerificationRequirement::GitChange { .. } => "gitChange",
1038 VerificationRequirement::TrustedCustom { .. } => "trustedCustom",
1039 _ => "unknown",
1040 }
1041}
1042
1043fn infer_verification(worktree: &Path) -> VerificationRequirement {
1044 let command = if worktree.join("Cargo.lock").is_file() {
1045 Some("cargo test --workspace --all-targets --locked")
1046 } else if worktree.join("Cargo.toml").is_file() {
1047 Some("cargo test --workspace --all-targets")
1048 } else if worktree.join("package.json").is_file() {
1049 Some("npm test")
1050 } else if worktree.join("pyproject.toml").is_file() || worktree.join("pytest.ini").is_file() {
1051 Some("python -m pytest")
1052 } else if worktree.join("go.mod").is_file() {
1053 Some("go test ./...")
1054 } else {
1055 None
1056 };
1057 command.map_or(
1058 VerificationRequirement::GitChange {
1059 require_changes: true,
1060 require_clean: false,
1061 },
1062 |command| VerificationRequirement::Command {
1063 command: command.into(),
1064 expected_exit: 0,
1065 timeout_seconds: 600,
1066 },
1067 )
1068}
1069
1070fn event_tokens(value: &Value) -> u64 {
1071 value
1072 .pointer("/message/usage/totalTokens")
1073 .or_else(|| value.pointer("/usage/totalTokens"))
1074 .and_then(Value::as_u64)
1075 .or_else(|| {
1076 let usage = value
1077 .pointer("/message/usage")
1078 .or_else(|| value.get("usage"))?;
1079 Some(
1080 ["input", "output", "cacheRead", "cacheWrite"]
1081 .iter()
1082 .filter_map(|name| usage.get(name).and_then(Value::as_u64))
1083 .fold(0_u64, u64::saturating_add),
1084 )
1085 })
1086 .unwrap_or(0)
1087}
1088
1089fn validate_verification(
1090 requirement: &VerificationRequirement,
1091 depth: usize,
1092) -> DevelopmentResult<()> {
1093 if depth > 8 {
1094 return Err(DevelopmentError::InvalidInput(
1095 "task verification nesting exceeds 8".into(),
1096 ));
1097 }
1098 match requirement {
1099 VerificationRequirement::Command {
1100 command,
1101 timeout_seconds,
1102 ..
1103 } => {
1104 if command.is_empty()
1105 || command.len() > MAX_VERIFIER_COMMAND_BYTES
1106 || command.contains('\0')
1107 || *timeout_seconds == 0
1108 || *timeout_seconds > 600
1109 {
1110 return Err(DevelopmentError::InvalidInput(
1111 "invalid task verification command".into(),
1112 ));
1113 }
1114 }
1115 VerificationRequirement::All { requirements } => {
1116 if requirements.is_empty() || requirements.len() > 16 {
1117 return Err(DevelopmentError::InvalidInput(
1118 "task verification group must contain 1..=16 requirements".into(),
1119 ));
1120 }
1121 for requirement in requirements {
1122 validate_verification(requirement, depth + 1)?;
1123 }
1124 }
1125 _ => {}
1126 }
1127 Ok(())
1128}
1129
1130fn push_evidence(record: &mut TaskRecord, evidence: TaskEvidence) {
1131 if record.snapshot.evidence.len() == MAX_TASK_EVIDENCE {
1132 record.snapshot.evidence.remove(0);
1133 }
1134 record.snapshot.evidence.push(evidence);
1135 record.snapshot.updated_at_ms = now_ms();
1136}
1137
1138fn validate_text(label: &str, value: &str) -> DevelopmentResult<()> {
1139 if value.is_empty() || value.len() > MAX_TASK_TEXT_BYTES || value.contains('\0') {
1140 return Err(DevelopmentError::InvalidInput(format!(
1141 "{label} must contain 1..={MAX_TASK_TEXT_BYTES} bytes without NUL"
1142 )));
1143 }
1144 Ok(())
1145}
1146
1147fn now_ms() -> u128 {
1148 SystemTime::now()
1149 .duration_since(UNIX_EPOCH)
1150 .unwrap_or_default()
1151 .as_millis()
1152}
1153
1154#[cfg(test)]
1155mod tests {
1156 use super::*;
1157 use std::sync::atomic::{AtomicU64, Ordering};
1158
1159 static NEXT_TEST_ROOT: AtomicU64 = AtomicU64::new(1);
1160
1161 #[derive(Default)]
1162 struct FakeAgents {
1163 next: u64,
1164 next_event: u64,
1165 snapshots: BTreeMap<AgentId, AgentSnapshot>,
1166 events: Vec<AgentEvent>,
1167 prompts: Vec<(AgentId, String)>,
1168 }
1169
1170 impl FakeAgents {
1171 fn emit(&mut self, id: &AgentId, kind: &str, payload: Value) {
1172 self.next_event += 1;
1173 self.events.push(AgentEvent {
1174 sequence: self.next_event,
1175 agent_id: id.clone(),
1176 timestamp_ms: now_ms(),
1177 kind: kind.into(),
1178 payload,
1179 });
1180 }
1181
1182 fn snapshot(id: AgentId, spec: AgentSpec) -> AgentSnapshot {
1183 AgentSnapshot {
1184 id,
1185 role: spec.role,
1186 task: spec.task,
1187 status: AgentStatus::Idle,
1188 dependencies: Vec::new(),
1189 model: spec.model,
1190 thinking: spec.thinking,
1191 worktree: spec.worktree.unwrap(),
1192 unrestricted: spec.unrestricted,
1193 created_at_ms: now_ms(),
1194 started_at_ms: Some(now_ms()),
1195 updated_at_ms: now_ms(),
1196 event_count: 0,
1197 dropped_event_count: 0,
1198 last_error: None,
1199 last_response_id: None,
1200 evidence: Vec::new(),
1201 }
1202 }
1203 }
1204
1205 impl TaskAgentBackend for FakeAgents {
1206 fn refresh_agents(&mut self) -> DevelopmentResult<()> {
1207 Ok(())
1208 }
1209
1210 fn create_agent(&mut self, spec: AgentSpec) -> DevelopmentResult<AgentId> {
1211 self.next += 1;
1212 let id = AgentId::parse(format!("agent-{:04}", self.next))?;
1213 self.snapshots
1214 .insert(id.clone(), Self::snapshot(id.clone(), spec));
1215 self.emit(&id, "ready", Value::Null);
1216 Ok(id)
1217 }
1218
1219 fn agent_snapshots(&mut self) -> DevelopmentResult<Vec<AgentSnapshot>> {
1220 Ok(self.snapshots.values().cloned().collect())
1221 }
1222
1223 fn agent_events(&mut self, since: u64) -> DevelopmentResult<Vec<AgentEvent>> {
1224 Ok(self
1225 .events
1226 .iter()
1227 .filter(|event| event.sequence > since)
1228 .cloned()
1229 .collect())
1230 }
1231
1232 fn prompt_agent(&mut self, id: &AgentId, prompt: String) -> DevelopmentResult<()> {
1233 self.prompts.push((id.clone(), prompt));
1234 self.emit(id, "requestStarted", Value::String("request-1".into()));
1235 self.emit(id, "agent_settled", Value::Null);
1236 Ok(())
1237 }
1238
1239 fn cancel_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
1240 self.snapshots.get_mut(id).unwrap().status = AgentStatus::Cancelled;
1241 Ok(())
1242 }
1243
1244 fn complete_agent(&mut self, id: &AgentId) -> DevelopmentResult<()> {
1245 self.snapshots.get_mut(id).unwrap().status = AgentStatus::Completed;
1246 Ok(())
1247 }
1248 }
1249
1250 fn scheduler() -> (PathBuf, TaskScheduler, FakeAgents) {
1251 let root = std::env::temp_dir().join(format!(
1252 "glass-task-scheduler-test-{}-{}",
1253 std::process::id(),
1254 NEXT_TEST_ROOT.fetch_add(1, Ordering::Relaxed)
1255 ));
1256 std::fs::create_dir_all(&root).unwrap();
1257 let scheduler = TaskScheduler::new(&root).unwrap();
1258 (root, scheduler, FakeAgents::default())
1259 }
1260
1261 fn settle(scheduler: &mut TaskScheduler, agents: &mut FakeAgents) {
1262 scheduler.refresh(agents).unwrap();
1263 scheduler.refresh(agents).unwrap();
1264 }
1265
1266 fn settle_only(title: impl Into<String>, prompt: impl Into<String>) -> TaskSpec {
1267 let mut spec = TaskSpec::new(title, prompt);
1268 spec.verification = VerificationRequirement::Settled;
1269 spec
1270 }
1271
1272 #[test]
1273 fn tasks_dispatch_prompts_verify_and_wake_dag_dependents() {
1274 let (root, mut scheduler, mut agents) = scheduler();
1275 let first = scheduler
1276 .create(&mut agents, settle_only("investigate", "inspect failure"))
1277 .unwrap();
1278 settle(&mut scheduler, &mut agents);
1279 assert_eq!(
1280 scheduler.snapshot(&mut agents, &first).unwrap().state,
1281 TaskState::Succeeded
1282 );
1283 assert_eq!(agents.prompts[0].1, "inspect failure");
1284
1285 let mut dependent = settle_only("repair", "apply repair");
1286 dependent.dependencies.push(first);
1287 let dependent = scheduler.create(&mut agents, dependent).unwrap();
1288 settle(&mut scheduler, &mut agents);
1289 assert_eq!(
1290 scheduler.snapshot(&mut agents, &dependent).unwrap().state,
1291 TaskState::Succeeded
1292 );
1293 assert_eq!(agents.prompts[1].1, "apply repair");
1294 std::fs::remove_dir_all(root).unwrap();
1295 }
1296
1297 #[test]
1298 fn eight_ready_tasks_dispatch_before_integration_dependency_wakes() {
1299 let (root, mut scheduler, mut agents) = scheduler();
1300 let mut leaves = Vec::new();
1301 for index in 0..8 {
1302 leaves.push(
1303 scheduler
1304 .create(
1305 &mut agents,
1306 settle_only(format!("worker-{index}"), format!("inspect shard {index}")),
1307 )
1308 .unwrap(),
1309 );
1310 }
1311 assert_eq!(scheduler.tasks.len(), 8);
1312
1313 let mut integration = settle_only("integration", "verify all shards");
1314 integration.dependencies = leaves.clone();
1315 let integration = scheduler.create(&mut agents, integration).unwrap();
1316 assert_eq!(scheduler.tasks.len(), 9);
1317 settle(&mut scheduler, &mut agents);
1318 settle(&mut scheduler, &mut agents);
1319
1320 for leaf in leaves {
1321 assert_eq!(
1322 scheduler.snapshot(&mut agents, &leaf).unwrap().state,
1323 TaskState::Succeeded
1324 );
1325 }
1326 assert_eq!(
1327 scheduler.snapshot(&mut agents, &integration).unwrap().state,
1328 TaskState::Succeeded
1329 );
1330 assert_eq!(agents.prompts.len(), 9);
1331 std::fs::remove_dir_all(root).unwrap();
1332 }
1333
1334 #[test]
1335 fn verification_failure_retries_then_blocks_descendants() {
1336 let (root, mut scheduler, mut agents) = scheduler();
1337 let mut failing = TaskSpec::new("verify", "produce candidate");
1338 failing.verification = VerificationRequirement::Command {
1339 command: "exit 7".into(),
1340 expected_exit: 0,
1341 timeout_seconds: 2,
1342 };
1343 failing.retry = RetryPolicy {
1344 max_retries: 1,
1345 backoff_seconds: 0,
1346 };
1347 let failing = scheduler.create(&mut agents, failing).unwrap();
1348 let mut child = TaskSpec::new("integration", "integrate candidate");
1349 child.dependencies.push(failing.clone());
1350 let child = scheduler.create(&mut agents, child).unwrap();
1351 for _ in 0..6 {
1352 scheduler.refresh(&mut agents).unwrap();
1353 }
1354 let failed = scheduler.snapshot(&mut agents, &failing).unwrap();
1355 assert_eq!(failed.state, TaskState::Failed);
1356 assert_eq!(failed.attempt, 2);
1357 assert_eq!(
1358 scheduler.snapshot(&mut agents, &child).unwrap().state,
1359 TaskState::Blocked
1360 );
1361 assert_eq!(agents.prompts.len(), 2);
1362 std::fs::remove_dir_all(root).unwrap();
1363 }
1364
1365 #[test]
1366 fn external_verification_evidence_is_proof_not_agent_claim() {
1367 let (root, mut scheduler, mut agents) = scheduler();
1368 let mut spec = TaskSpec::new("diagnostics", "repair diagnostics");
1369 spec.verification = VerificationRequirement::LspDiagnostics { max_errors: 0 };
1370 let id = scheduler.create(&mut agents, spec).unwrap();
1371 settle(&mut scheduler, &mut agents);
1372 assert_eq!(
1373 scheduler.snapshot(&mut agents, &id).unwrap().state,
1374 TaskState::Waiting
1375 );
1376 scheduler
1377 .submit_evidence(
1378 &id,
1379 "lspDiagnostics",
1380 "resident-lsp",
1381 "task-test",
1382 true,
1383 serde_json::json!({"errors":0,"source":"resident-lsp"}),
1384 )
1385 .unwrap();
1386 scheduler.refresh(&mut agents).unwrap();
1387 assert_eq!(
1388 scheduler.snapshot(&mut agents, &id).unwrap().state,
1389 TaskState::Succeeded
1390 );
1391 std::fs::remove_dir_all(root).unwrap();
1392 }
1393
1394 #[test]
1395 fn implementation_tasks_infer_project_verification_and_settle_only_is_explicit() {
1396 let (root, mut scheduler, mut agents) = scheduler();
1397 std::fs::write(
1398 root.join("Cargo.toml"),
1399 "[package]\nname='verification-fixture'\nversion='0.1.0'\n",
1400 )
1401 .unwrap();
1402 let inferred = scheduler
1403 .create(
1404 &mut agents,
1405 TaskSpec::new("implement repair", "change the implementation"),
1406 )
1407 .unwrap();
1408 assert!(matches!(
1409 scheduler.tasks[&inferred].snapshot.verification,
1410 VerificationRequirement::Command { ref command, .. }
1411 if command == "cargo test --workspace --all-targets"
1412 ));
1413
1414 let research = scheduler
1415 .create(
1416 &mut agents,
1417 settle_only("research alternatives", "compare approaches"),
1418 )
1419 .unwrap();
1420 assert_eq!(
1421 scheduler.tasks[&research].snapshot.verification,
1422 VerificationRequirement::Settled
1423 );
1424 std::fs::remove_dir_all(root).unwrap();
1425 }
1426}