1use hatchery_harness_protocol::{
4 HarnessActorV1, HarnessContinuationRef, HarnessContinuationStateV1,
5 HarnessContinuationV1, HarnessDeliveryRef, HarnessDeliveryStateV1, HarnessDeliveryV1,
6 HarnessEntityReadScopeV1, HarnessExpectedExecutionSpecRevisionV1,
7 HarnessOperationId, HarnessOperationKindV1,
8 HarnessOperationStateV1, HarnessOperationV1, HarnessResultRef, HarnessRevision,
9 HarnessRunId, HarnessRunLifecycleV1, HarnessRunV1,
10 HarnessTaskExecutionSpecV1,
11 HarnessTaskExecutionSpecV2, HarnessTaskId, HarnessTaskLaunchIssuanceV1,
12 HarnessTaskStateV1, HarnessTaskV1, HarnessTransferAuthorityRefV1,
13 HarnessValidationError, SessionGrantId, SessionGrantStateV1, SessionGrantV1,
14 HARNESS_CHILD_DEPTH_MAX, HARNESS_CONTINUATIONS_MAX, HARNESS_DELIVERIES_MAX,
15 HARNESS_RESULTS_MAX, HARNESS_SCHEDULER_SCAN_MAX,
16};
17use serde::{Deserialize, Serialize};
18use std::cell::RefCell;
19use std::collections::{BTreeMap, BTreeSet};
20use thiserror::Error;
21
22pub const HARNESS_ENGINE_CHECKPOINT_VERSION_V1: u16 = 1;
23pub const HARNESS_VISIBILITY_SCAN_MAX: usize = 8_192;
24
25#[derive(Clone, Copy, Debug, Eq, PartialEq)]
26pub enum HarnessApplyOutcome {
27 Applied,
28 Replayed,
29}
30
31#[derive(Clone, Debug, Serialize)]
32#[serde(tag = "mutation", rename_all = "kebab-case")]
33pub enum HarnessMutationV1 {
34 CreateTask {
35 operation: HarnessOperationV1,
36 task: HarnessTaskV1,
37 },
38 ReplaceTask {
39 operation: HarnessOperationV1,
40 expected_revision: HarnessRevision,
41 task: HarnessTaskV1,
42 },
43 PutExecutionSpec {
44 operation: HarnessOperationV1,
45 expected_task_revision: HarnessRevision,
46 expected_spec_revision: HarnessExpectedExecutionSpecRevisionV1,
47 spec: HarnessTaskExecutionSpecV1,
48 },
49 PutIssuedExecutionSpec {
50 operation: HarnessOperationV1,
51 expected_task_revision: HarnessRevision,
52 expected_spec_revision: HarnessExpectedExecutionSpecRevisionV1,
53 expected_issuance_revision: HarnessExpectedExecutionSpecRevisionV1,
54 issuance: HarnessTaskLaunchIssuanceV1,
55 spec: HarnessTaskExecutionSpecV2,
56 },
57 CreateRun {
58 operation: HarnessOperationV1,
59 expected_task_revision: HarnessRevision,
60 task: HarnessTaskV1,
61 run: HarnessRunV1,
62 },
63 CreateIssuedRun {
64 operation: HarnessOperationV1,
65 expected_task_revision: HarnessRevision,
66 expected_execution_spec_revision: HarnessRevision,
67 expected_issuance_revision: HarnessRevision,
68 task: HarnessTaskV1,
69 run: HarnessRunV1,
70 delivery: Option<HarnessDeliveryV1>,
71 continuation: Option<HarnessContinuationV1>,
72 },
73 ReplaceRun {
74 operation: HarnessOperationV1,
75 expected_revision: HarnessRevision,
76 run: HarnessRunV1,
77 },
78 CreateGrant {
79 operation: HarnessOperationV1,
80 grant: SessionGrantV1,
81 },
82 ReplaceGrant {
83 operation: HarnessOperationV1,
84 expected_revision: HarnessRevision,
85 grant: SessionGrantV1,
86 },
87}
88
89impl HarnessMutationV1 {
90 pub fn operation(&self) -> &HarnessOperationV1 {
91 match self {
92 Self::CreateTask { operation, .. }
93 | Self::ReplaceTask { operation, .. }
94 | Self::PutExecutionSpec { operation, .. }
95 | Self::PutIssuedExecutionSpec { operation, .. }
96 | Self::CreateRun { operation, .. }
97 | Self::CreateIssuedRun { operation, .. }
98 | Self::ReplaceRun { operation, .. }
99 | Self::CreateGrant { operation, .. }
100 | Self::ReplaceGrant { operation, .. } => operation,
101 }
102 }
103
104 pub fn operation_mut(&mut self) -> &mut HarnessOperationV1 {
105 match self {
106 Self::CreateTask { operation, .. }
107 | Self::ReplaceTask { operation, .. }
108 | Self::PutExecutionSpec { operation, .. }
109 | Self::PutIssuedExecutionSpec { operation, .. }
110 | Self::CreateRun { operation, .. }
111 | Self::CreateIssuedRun { operation, .. }
112 | Self::ReplaceRun { operation, .. }
113 | Self::CreateGrant { operation, .. }
114 | Self::ReplaceGrant { operation, .. } => operation,
115 }
116 }
117
118 pub fn validate_payload(&self) -> Result<(), HarnessEngineError> {
121 self.operation().validate()?;
122 match self {
123 Self::CreateTask { task, .. } => task.validate()?,
124 Self::ReplaceTask { expected_revision, task, .. } => {
125 expected_revision.validate()?;
126 task.validate()?;
127 }
128 Self::PutExecutionSpec {
129 expected_task_revision,
130 expected_spec_revision,
131 spec,
132 ..
133 } => {
134 expected_task_revision.validate()?;
135 expected_spec_revision.validate()?;
136 spec.validate()?;
137 }
138 Self::PutIssuedExecutionSpec {
139 expected_task_revision,
140 expected_spec_revision,
141 expected_issuance_revision,
142 issuance,
143 spec,
144 ..
145 } => {
146 expected_task_revision.validate()?;
147 expected_spec_revision.validate()?;
148 expected_issuance_revision.validate()?;
149 issuance.validate()?;
150 spec.validate()?;
151 }
152 Self::CreateRun {
153 expected_task_revision,
154 task,
155 run,
156 ..
157 } => {
158 expected_task_revision.validate()?;
159 task.validate()?;
160 run.validate()?;
161 }
162 Self::CreateIssuedRun {
163 expected_task_revision,
164 expected_execution_spec_revision,
165 expected_issuance_revision,
166 task,
167 run,
168 delivery,
169 continuation,
170 ..
171 } => {
172 expected_task_revision.validate()?;
173 expected_execution_spec_revision.validate()?;
174 expected_issuance_revision.validate()?;
175 task.validate()?;
176 run.validate()?;
177 if let Some(delivery) = delivery { delivery.validate()?; }
178 if let Some(continuation) = continuation { continuation.validate()?; }
179 }
180 Self::ReplaceRun { expected_revision, run, .. } => {
181 expected_revision.validate()?;
182 run.validate()?;
183 }
184 Self::CreateGrant { grant, .. } => grant.validate()?,
185 Self::ReplaceGrant { expected_revision, grant, .. } => {
186 expected_revision.validate()?;
187 grant.validate()?;
188 }
189 }
190 Ok(())
191 }
192}
193
194#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
195pub struct HarnessEngineCheckpointV1 {
196 pub version: u16,
197 pub tasks: Vec<HarnessTaskV1>,
198 pub runs: Vec<HarnessRunV1>,
199 pub grants: Vec<SessionGrantV1>,
200 pub operations: Vec<HarnessOperationV1>,
201 #[serde(default)]
202 pub execution_specs: Vec<HarnessTaskExecutionSpecV1>,
203 #[serde(default)]
204 pub issuances: Vec<HarnessTaskLaunchIssuanceV1>,
205 #[serde(default)]
206 pub execution_specs_v2: Vec<HarnessTaskExecutionSpecV2>,
207 #[serde(default)]
208 pub deliveries: Vec<HarnessDeliveryV1>,
209 #[serde(default)]
210 pub continuations: Vec<HarnessContinuationV1>,
211}
212
213#[derive(Deserialize)]
224#[serde(deny_unknown_fields)]
225struct HarnessEngineCheckpointWireV1 {
226 version: u16,
227 tasks: Vec<HarnessTaskV1>,
228 runs: Vec<HarnessRunV1>,
229 grants: Vec<SessionGrantV1>,
230 operations: Vec<HarnessOperationV1>,
231 #[serde(default)]
232 execution_specs: Vec<HarnessTaskExecutionSpecV1>,
233 #[serde(default)]
234 issuances: Vec<HarnessTaskLaunchIssuanceV1>,
235 #[serde(default)]
236 execution_specs_v2: Vec<HarnessTaskExecutionSpecV2>,
237 #[serde(default)]
238 deliveries: Vec<HarnessDeliveryV1>,
239 #[serde(default)]
240 continuations: Vec<HarnessContinuationV1>,
241 #[serde(default)]
242 mail_messages: Option<serde::de::IgnoredAny>,
243 #[serde(default)]
244 mail_acks: Option<serde::de::IgnoredAny>,
245}
246
247impl<'de> Deserialize<'de> for HarnessEngineCheckpointV1 {
248 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
249 where
250 D: serde::Deserializer<'de>,
251 {
252 let wire = HarnessEngineCheckpointWireV1::deserialize(deserializer)?;
253 let _retired_mail_messages = wire.mail_messages;
254 let _retired_mail_acks = wire.mail_acks;
255 Ok(Self {
256 version: wire.version,
257 tasks: wire.tasks,
258 runs: wire.runs,
259 grants: wire.grants,
260 operations: wire.operations,
261 execution_specs: wire.execution_specs,
262 issuances: wire.issuances,
263 execution_specs_v2: wire.execution_specs_v2,
264 deliveries: wire.deliveries,
265 continuations: wire.continuations,
266 })
267 }
268}
269
270#[derive(Clone, Copy, Debug, Eq, PartialEq)]
279enum HarnessVisibilityGraphFaultV1 {
280 ResourceExhausted,
281 InvalidGraph,
282}
283
284impl From<HarnessVisibilityGraphFaultV1> for HarnessEngineError {
285 fn from(fault: HarnessVisibilityGraphFaultV1) -> Self {
286 match fault {
287 HarnessVisibilityGraphFaultV1::ResourceExhausted => {
288 HarnessEngineError::ReadVisibilityResourceExhausted
289 }
290 HarnessVisibilityGraphFaultV1::InvalidGraph => {
291 HarnessEngineError::ReadVisibilityInvalidGraph
292 }
293 }
294 }
295}
296
297#[derive(Clone, Debug, Default)]
298pub struct HarnessEngine {
299 tasks: BTreeMap<HarnessTaskId, HarnessTaskV1>,
300 runs: BTreeMap<HarnessRunId, HarnessRunV1>,
301 grants: BTreeMap<SessionGrantId, SessionGrantV1>,
302 operations: BTreeMap<HarnessOperationId, HarnessOperationV1>,
303 execution_specs: BTreeMap<HarnessTaskId, HarnessTaskExecutionSpecV1>,
304 issuances: BTreeMap<HarnessTaskId, HarnessTaskLaunchIssuanceV1>,
305 execution_specs_v2: BTreeMap<HarnessTaskId, HarnessTaskExecutionSpecV2>,
306 deliveries: BTreeMap<HarnessDeliveryRef, HarnessDeliveryV1>,
307 continuations: BTreeMap<HarnessContinuationRef, HarnessContinuationV1>,
308 mutation_generation: u64,
318 visibility_graph_cache: RefCell<Option<(u64, Result<(), HarnessVisibilityGraphFaultV1>)>>,
330}
331
332impl PartialEq for HarnessEngine {
341 fn eq(&self, other: &Self) -> bool {
342 self.tasks == other.tasks
343 && self.runs == other.runs
344 && self.grants == other.grants
345 && self.operations == other.operations
346 && self.execution_specs == other.execution_specs
347 && self.issuances == other.issuances
348 && self.execution_specs_v2 == other.execution_specs_v2
349 && self.deliveries == other.deliveries
350 && self.continuations == other.continuations
351 }
352}
353
354impl Eq for HarnessEngine {}
355
356#[derive(Clone, Debug, Default, Eq, PartialEq)]
357pub struct HarnessReadVisibilityV1 {
367 task_ids: BTreeSet<HarnessTaskId>,
368 run_ids: BTreeSet<HarnessRunId>,
369 operation_ids: BTreeSet<HarnessOperationId>,
370}
371
372impl HarnessReadVisibilityV1 {
373 pub fn task_visible(&self, task_id: &HarnessTaskId) -> bool {
374 self.task_ids.contains(task_id)
375 }
376
377 pub fn run_visible(&self, run_id: &HarnessRunId) -> bool {
378 self.run_ids.contains(run_id)
379 }
380
381 pub fn operation_visible(&self, operation_id: &HarnessOperationId) -> bool {
382 self.operation_ids.contains(operation_id)
383 }
384
385 pub fn task_ids(&self) -> impl Iterator<Item = &HarnessTaskId> {
386 self.task_ids.iter()
387 }
388
389 pub fn run_ids(&self) -> impl Iterator<Item = &HarnessRunId> {
390 self.run_ids.iter()
391 }
392
393 pub fn operation_ids(&self) -> impl Iterator<Item = &HarnessOperationId> {
394 self.operation_ids.iter()
395 }
396
397 pub fn is_empty(&self) -> bool {
398 self.task_ids.is_empty() && self.run_ids.is_empty() && self.operation_ids.is_empty()
399 }
400}
401
402#[derive(Clone, Debug)]
403pub struct PreparedHarnessMutation {
404 next: HarnessEngine,
405 outcome: HarnessApplyOutcome,
406 operation: HarnessOperationV1,
407}
408
409impl PreparedHarnessMutation {
410 pub fn outcome(&self) -> HarnessApplyOutcome {
411 self.outcome
412 }
413
414 pub fn operation(&self) -> &HarnessOperationV1 {
415 &self.operation
416 }
417
418 pub fn checkpoint(&self) -> HarnessEngineCheckpointV1 {
419 self.next.checkpoint()
420 }
421
422 pub fn checkpoint_deliveries(&self) -> Vec<HarnessDeliveryV1> {
423 self.next.deliveries.values().cloned().collect()
424 }
425
426 pub fn checkpoint_continuations(&self) -> Vec<HarnessContinuationV1> {
427 self.next.continuations.values().cloned().collect()
428 }
429}
430
431impl HarnessEngine {
432 pub fn new() -> Self {
433 Self::default()
434 }
435
436 pub fn task(&self, task_id: &HarnessTaskId) -> Option<&HarnessTaskV1> {
437 self.tasks.get(task_id)
438 }
439
440 pub fn run(&self, run_id: &HarnessRunId) -> Option<&HarnessRunV1> {
441 self.runs.get(run_id)
442 }
443
444 pub fn grant(&self, grant_id: &SessionGrantId) -> Option<&SessionGrantV1> {
445 self.grants.get(grant_id)
446 }
447
448 pub fn operation(
449 &self,
450 operation_id: &HarnessOperationId,
451 ) -> Option<&HarnessOperationV1> {
452 self.operations.get(operation_id)
453 }
454
455 pub fn execution_spec(
456 &self,
457 task_id: &HarnessTaskId,
458 ) -> Option<&HarnessTaskExecutionSpecV1> {
459 self.execution_specs.get(task_id)
460 }
461
462 pub fn execution_specs(
463 &self,
464 ) -> impl Iterator<Item = &HarnessTaskExecutionSpecV1> {
465 self.execution_specs.values()
466 }
467
468 pub fn task_launch_issuance(
469 &self,
470 task_id: &HarnessTaskId,
471 ) -> Option<&HarnessTaskLaunchIssuanceV1> {
472 self.issuances.get(task_id)
473 }
474
475 pub fn task_execution_spec_v2(
476 &self,
477 task_id: &HarnessTaskId,
478 ) -> Option<&HarnessTaskExecutionSpecV2> {
479 self.execution_specs_v2.get(task_id)
480 }
481
482 pub fn task_launch_issuances(
483 &self,
484 ) -> impl Iterator<Item = &HarnessTaskLaunchIssuanceV1> {
485 self.issuances.values()
486 }
487
488 pub fn task_execution_specs_v2(
489 &self,
490 ) -> impl Iterator<Item = &HarnessTaskExecutionSpecV2> {
491 self.execution_specs_v2.values()
492 }
493
494 pub fn delivery(&self, delivery_ref: &HarnessDeliveryRef) -> Option<&HarnessDeliveryV1> {
495 self.deliveries.get(delivery_ref)
496 }
497
498 pub fn continuation(
499 &self,
500 continuation_ref: &HarnessContinuationRef,
501 ) -> Option<&HarnessContinuationV1> {
502 self.continuations.get(continuation_ref)
503 }
504
505 pub fn continuation_for_run(&self, run_id: &HarnessRunId) -> Option<&HarnessContinuationV1> {
506 self.continuations.values().find(|record| &record.target_run_id == run_id)
507 }
508
509 pub fn delivery_for_run(&self, run_id: &HarnessRunId) -> Option<&HarnessDeliveryV1> {
510 self.deliveries.values().find(|delivery| &delivery.run_id == run_id)
511 }
512
513 pub fn tasks(&self) -> impl Iterator<Item = &HarnessTaskV1> {
514 self.tasks.values()
515 }
516
517 pub fn runs(&self) -> impl Iterator<Item = &HarnessRunV1> {
518 self.runs.values()
519 }
520
521 pub fn grants(&self) -> impl Iterator<Item = &SessionGrantV1> {
522 self.grants.values()
523 }
524
525 pub fn operations(&self) -> impl Iterator<Item = &HarnessOperationV1> {
526 self.operations.values()
527 }
528
529 pub fn deliveries(&self) -> impl Iterator<Item = &HarnessDeliveryV1> {
530 self.deliveries.values()
531 }
532
533 pub fn scheduler_pending_dispatch(
534 &self,
535 ) -> Result<Option<(&HarnessTaskV1, &HarnessRunV1, &HarnessOperationV1)>, HarnessEngineError> {
536 Ok(self.scheduler_pending_dispatches()?.into_iter().next())
537 }
538
539 pub fn scheduler_pending_dispatches(
540 &self,
541 ) -> Result<Vec<(&HarnessTaskV1, &HarnessRunV1, &HarnessOperationV1)>, HarnessEngineError> {
542 self.scheduler_pending_dispatches_inner(None)
543 }
544
545 pub fn scheduler_pending_dispatches_for(
546 &self,
547 operation_ids: &BTreeSet<HarnessOperationId>,
548 ) -> Result<Vec<(&HarnessTaskV1, &HarnessRunV1, &HarnessOperationV1)>, HarnessEngineError> {
549 self.scheduler_pending_dispatches_inner(Some(operation_ids))
550 }
551
552 fn scheduler_pending_dispatches_inner(
553 &self,
554 operation_ids: Option<&BTreeSet<HarnessOperationId>>,
555 ) -> Result<Vec<(&HarnessTaskV1, &HarnessRunV1, &HarnessOperationV1)>, HarnessEngineError> {
556 let mut pending = Vec::new();
561 for (scanned, run) in self.runs.values().enumerate() {
562 if scanned >= HARNESS_SCHEDULER_SCAN_MAX {
563 return Err(HarnessEngineError::SchedulerResourceExhausted);
564 }
565 if run.lifecycle != HarnessRunLifecycleV1::Requested
566 || operation_ids.map(|ids| !ids.contains(&run.operation_id)).unwrap_or(false)
567 {
568 continue;
569 }
570 let operation = self.operations.get(&run.operation_id)
571 .ok_or(HarnessEngineError::SchedulerInvalidGraph)?;
572 if operation.kind != HarnessOperationKindV1::CreateRun
573 || operation.state != HarnessOperationStateV1::Prepared
574 {
575 continue;
576 }
577 let task = self.tasks.get(&run.task_id)
578 .ok_or(HarnessEngineError::SchedulerInvalidGraph)?;
579 if task.state != HarnessTaskStateV1::Running
580 || task.run_ids.binary_search(&run.run_id).is_err()
581 || operation.run_id.as_ref() != Some(&run.run_id)
582 || operation.task_id.as_ref() != Some(&task.task_id)
583 {
584 return Err(HarnessEngineError::SchedulerInvalidGraph);
585 }
586 pending.push((task, run, operation));
587 }
588 Ok(pending)
589 }
590
591 pub fn scheduler_ready_task(&self) -> Result<Option<&HarnessTaskV1>, HarnessEngineError> {
598 for (scanned, task) in self.tasks.values().enumerate() {
599 if scanned >= HARNESS_SCHEDULER_SCAN_MAX {
600 return Err(HarnessEngineError::SchedulerResourceExhausted);
601 }
602 if task.state == HarnessTaskStateV1::Ready && self.scheduler_task_is_eligible(task)? {
603 return Ok(Some(task));
604 }
605 }
606 Ok(None)
607 }
608
609 pub fn scheduler_ready_task_by_id(
613 &self,
614 task_id: &HarnessTaskId,
615 ) -> Result<Option<&HarnessTaskV1>, HarnessEngineError> {
616 let task = self.tasks.get(task_id)
617 .ok_or_else(|| HarnessEngineError::NotFound(task_id.to_string()))?;
618 if task.state != HarnessTaskStateV1::Ready || !self.scheduler_task_is_eligible(task)? {
619 return Ok(None);
620 }
621 Ok(Some(task))
622 }
623
624 fn scheduler_task_is_eligible(
625 &self,
626 task: &HarnessTaskV1,
627 ) -> Result<bool, HarnessEngineError> {
628 let dependencies_done = task.dependencies.iter().all(|dependency_id| {
629 self.tasks.get(dependency_id)
630 .map(|dependency| dependency.state == HarnessTaskStateV1::Done)
631 .unwrap_or(false)
632 });
633 Ok(dependencies_done && !self.task_has_nonterminal_run(task)?)
634 }
635
636 pub fn task_nonterminal_run_id(
646 &self,
647 task: &HarnessTaskV1,
648 ) -> Result<Option<HarnessRunId>, HarnessEngineError> {
649 for run_id in &task.run_ids {
650 let run = self.runs.get(run_id).ok_or(HarnessEngineError::SchedulerInvalidGraph)?;
651 if run.task_id != task.task_id {
652 return Err(HarnessEngineError::SchedulerInvalidGraph);
653 }
654 if !matches!(
655 run.lifecycle,
656 HarnessRunLifecycleV1::Completed
657 | HarnessRunLifecycleV1::Failed
658 | HarnessRunLifecycleV1::Cancelled
659 ) {
660 return Ok(Some(run_id.clone()));
661 }
662 }
663 Ok(None)
664 }
665
666 pub fn task_has_nonterminal_run(
671 &self,
672 task: &HarnessTaskV1,
673 ) -> Result<bool, HarnessEngineError> {
674 Ok(self.task_nonterminal_run_id(task)?.is_some())
675 }
676
677 pub fn unmet_task_dependencies(&self, task: &HarnessTaskV1) -> Vec<HarnessTaskId> {
687 task.dependencies.iter()
688 .filter(|dependency_id| {
689 !self.tasks.get(*dependency_id)
690 .map(|dependency| dependency.state == HarnessTaskStateV1::Done)
691 .unwrap_or(false)
692 })
693 .cloned()
694 .collect()
695 }
696
697 pub fn restore_deliveries(
698 mut self,
699 deliveries: Vec<HarnessDeliveryV1>,
700 ) -> Result<Self, HarnessEngineError> {
701 if deliveries.len() > HARNESS_DELIVERIES_MAX {
702 return Err(HarnessEngineError::DeliveryCapacityExceeded);
703 }
704 for delivery in deliveries {
705 delivery.validate()?;
706 let delivery_ref = delivery.delivery_ref.clone();
707 if self.deliveries.insert(delivery_ref.clone(), delivery).is_some() {
708 return Err(HarnessEngineError::DuplicateCheckpointId(
709 delivery_ref.to_string(),
710 ));
711 }
712 }
713 self.validate_links()?;
714 Ok(self)
715 }
716
717 pub fn read_visibility(
718 &self,
719 grant_id: &SessionGrantId,
720 ) -> Result<HarnessReadVisibilityV1, HarnessEngineError> {
721 let Some(grant) = self.grants.get(grant_id) else {
722 return Ok(HarnessReadVisibilityV1::default());
723 };
724 if grant.state != SessionGrantStateV1::Active || grant.validate().is_err() {
725 return Ok(HarnessReadVisibilityV1::default());
726 }
727 if !self.runs.contains_key(&grant.actor_run_id) {
728 return Ok(HarnessReadVisibilityV1::default());
729 }
730 if grant.read_permissions == Default::default() {
731 return Ok(HarnessReadVisibilityV1::default());
732 }
733
734 self.visibility_graph_validation()?;
735
736 let task_attribution_runs = self.visible_runs_for_scope(
737 grant.read_permissions.tasks,
738 &grant.actor_run_id,
739 )?;
740 let run_ids = self.visible_runs_for_scope(
741 grant.read_permissions.runs,
742 &grant.actor_run_id,
743 )?;
744 let operation_actor_runs = self.visible_runs_for_scope(
745 grant.read_permissions.operations,
746 &grant.actor_run_id,
747 )?;
748
749 let task_ids = self.tasks_attributed_to_runs(&task_attribution_runs);
750 let operation_ids = self.visible_operations(
751 &operation_actor_runs,
752 &task_ids,
753 &run_ids,
754 )?;
755
756 Ok(HarnessReadVisibilityV1 {
757 task_ids,
758 run_ids,
759 operation_ids,
760 })
761 }
762
763 pub fn checkpoint(&self) -> HarnessEngineCheckpointV1 {
764 HarnessEngineCheckpointV1 {
765 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
766 tasks: self.tasks.values().cloned().collect(),
767 runs: self.runs.values().cloned().collect(),
768 grants: self.grants.values().cloned().collect(),
769 operations: self.operations.values().cloned().collect(),
770 execution_specs: self.execution_specs.values().cloned().collect(),
771 issuances: self.issuances.values().cloned().collect(),
772 execution_specs_v2: self.execution_specs_v2.values().cloned().collect(),
773 deliveries: self.deliveries.values().cloned().collect(),
774 continuations: self.continuations.values().cloned().collect(),
775 }
776 }
777
778 pub fn restore(
779 checkpoint: HarnessEngineCheckpointV1,
780 ) -> Result<Self, HarnessEngineError> {
781 if checkpoint.version != HARNESS_ENGINE_CHECKPOINT_VERSION_V1 {
782 return Err(HarnessEngineError::UnsupportedCheckpoint(checkpoint.version));
783 }
784 let mut engine = Self::new();
785 for task in checkpoint.tasks {
786 task.validate()?;
787 let task_id = task.task_id.clone();
788 if engine.tasks.insert(task_id.clone(), task).is_some() {
789 return Err(HarnessEngineError::DuplicateCheckpointId(task_id.to_string()));
790 }
791 }
792 for run in checkpoint.runs {
793 run.validate()?;
794 let run_id = run.run_id.clone();
795 if engine.runs.insert(run_id.clone(), run).is_some() {
796 return Err(HarnessEngineError::DuplicateCheckpointId(run_id.to_string()));
797 }
798 }
799 for grant in checkpoint.grants {
800 grant.validate()?;
801 let grant_id = grant.grant_id.clone();
802 if engine.grants.insert(grant_id.clone(), grant).is_some() {
803 return Err(HarnessEngineError::DuplicateCheckpointId(grant_id.to_string()));
804 }
805 }
806 for operation in checkpoint.operations {
807 operation.validate()?;
808 let operation_id = operation.operation_id.clone();
809 if engine.operations.insert(operation_id.clone(), operation).is_some() {
810 return Err(HarnessEngineError::DuplicateCheckpointId(operation_id.to_string()));
811 }
812 }
813 for spec in checkpoint.execution_specs {
814 validate_execution_spec_identity(&spec)?;
815 let task_id = spec.task_id.clone();
816 if engine.execution_specs.insert(task_id.clone(), spec).is_some() {
817 return Err(HarnessEngineError::DuplicateCheckpointId(
818 task_id.to_string(),
819 ));
820 }
821 }
822 for issuance in checkpoint.issuances {
823 issuance.validate()?;
824 let task_id = issuance.task_id.clone();
825 if engine.execution_specs.contains_key(&task_id) {
826 return Err(HarnessEngineError::DualExecutionSpec(task_id.to_string()));
827 }
828 if engine.issuances.insert(task_id.clone(), issuance).is_some() {
829 return Err(HarnessEngineError::DuplicateCheckpointId(
830 task_id.to_string(),
831 ));
832 }
833 }
834 for spec in checkpoint.execution_specs_v2 {
835 spec.validate()?;
836 let task_id = spec.task_id.clone();
837 if engine.execution_specs.contains_key(&task_id) {
838 return Err(HarnessEngineError::DualExecutionSpec(task_id.to_string()));
839 }
840 if engine.execution_specs_v2.insert(task_id.clone(), spec).is_some() {
841 return Err(HarnessEngineError::DuplicateCheckpointId(
842 task_id.to_string(),
843 ));
844 }
845 }
846 if checkpoint.deliveries.len() > HARNESS_DELIVERIES_MAX {
847 return Err(HarnessEngineError::DeliveryCapacityExceeded);
848 }
849 for delivery in checkpoint.deliveries {
850 delivery.validate()?;
851 let delivery_ref = delivery.delivery_ref.clone();
852 if engine.deliveries.insert(delivery_ref.clone(), delivery).is_some() {
853 return Err(HarnessEngineError::DuplicateCheckpointId(
854 delivery_ref.to_string(),
855 ));
856 }
857 }
858 if checkpoint.continuations.len() > HARNESS_CONTINUATIONS_MAX {
859 return Err(HarnessEngineError::ContinuationCapacityExceeded);
860 }
861 for continuation in checkpoint.continuations {
862 continuation.validate()?;
863 let continuation_ref = continuation.continuation_ref.clone();
864 if engine.continuations.insert(continuation_ref.clone(), continuation).is_some() {
865 return Err(HarnessEngineError::DuplicateCheckpointId(
866 continuation_ref.to_string(),
867 ));
868 }
869 }
870 engine.validate_links()?;
871 Ok(engine)
872 }
873
874 pub fn prepare(
875 &self,
876 mutation: HarnessMutationV1,
877 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
878 mutation.validate_payload()?;
879 let requested_operation = mutation.operation();
880 if let Some(existing) = self.operations.get(&requested_operation.operation_id) {
881 let issued_projection_matches = match &mutation {
882 HarnessMutationV1::CreateIssuedRun {
883 expected_execution_spec_revision,
884 expected_issuance_revision,
885 task,
886 run,
887 delivery,
888 continuation,
889 ..
890 } => self.same_issued_run_projection(
891 *expected_execution_spec_revision,
892 *expected_issuance_revision,
893 task,
894 run,
895 delivery.as_ref(),
896 continuation.as_ref(),
897 ),
898 _ => true,
899 };
900 if !same_operation_request(existing, requested_operation)
901 || !issued_projection_matches
902 {
903 return Err(HarnessEngineError::OperationIdConflict {
904 operation_id: requested_operation.operation_id.clone(),
905 });
906 }
907 return Ok(PreparedHarnessMutation {
908 next: self.clone(),
909 outcome: HarnessApplyOutcome::Replayed,
910 operation: existing.clone(),
911 });
912 }
913 self.validate_actor(&requested_operation.actor)?;
914 require_first_revision(requested_operation.revision, "operation")?;
915
916 let mut next = self.clone();
917 match mutation {
918 HarnessMutationV1::CreateTask { operation, task } => {
919 require_kind(operation.kind, HarnessOperationKindV1::CreateTask)?;
920 require_operation_state(operation.state, HarnessOperationStateV1::Succeeded)?;
921 task.validate()?;
922 require_first_revision(task.revision, "task")?;
923 require_same_id(operation.task_id.as_ref(), &task.task_id, "task")?;
924 if task.creator != operation.actor {
925 return Err(HarnessEngineError::MismatchedIdentity("task creator"));
926 }
927 if next.tasks.contains_key(&task.task_id) {
928 return Err(HarnessEngineError::AlreadyExists(task.task_id.to_string()));
929 }
930 next.validate_task_links(&task)?;
931 next.tasks.insert(task.task_id.clone(), task);
932 next.operations.insert(operation.operation_id.clone(), operation.clone());
933 next.finish(operation, HarnessApplyOutcome::Applied)
934 }
935 HarnessMutationV1::ReplaceTask { operation, expected_revision, task } => {
936 require_kind(operation.kind, HarnessOperationKindV1::MutateTask)?;
937 require_operation_state(operation.state, HarnessOperationStateV1::Succeeded)?;
938 require_operation_expected(&operation, expected_revision)?;
939 task.validate()?;
940 require_same_id(operation.task_id.as_ref(), &task.task_id, "task")?;
941 let current = next.tasks.get(&task.task_id)
942 .ok_or_else(|| HarnessEngineError::NotFound(task.task_id.to_string()))?;
943 validate_replacement(
944 "task",
945 current.revision,
946 expected_revision,
947 task.revision,
948 current.created_at_unix_ms,
949 task.created_at_unix_ms,
950 )?;
951 if current.creator != task.creator {
952 return Err(HarnessEngineError::MismatchedIdentity("immutable task creator"));
953 }
954 next.validate_task_links(&task)?;
955 next.tasks.insert(task.task_id.clone(), task);
956 next.operations.insert(operation.operation_id.clone(), operation.clone());
957 next.finish(operation, HarnessApplyOutcome::Applied)
958 }
959 HarnessMutationV1::PutExecutionSpec {
960 operation,
961 expected_task_revision,
962 expected_spec_revision,
963 spec,
964 } => {
965 require_kind(
966 operation.kind,
967 HarnessOperationKindV1::MutateExecutionSpec,
968 )?;
969 require_operation_state(operation.state, HarnessOperationStateV1::Succeeded)?;
970 require_operation_expected(&operation, expected_task_revision)?;
971 validate_execution_spec_identity(&spec)?;
972 require_same_id(operation.task_id.as_ref(), &spec.task_id, "task")?;
973 let task = next.tasks.get(&spec.task_id)
974 .ok_or_else(|| HarnessEngineError::NotFound(spec.task_id.to_string()))?;
975 if task.revision != expected_task_revision {
976 return Err(HarnessEngineError::ExpectedRevisionMismatch {
977 entity: "task",
978 expected: expected_task_revision.get(),
979 actual: Some(task.revision.get()),
980 });
981 }
982 match (
983 expected_spec_revision,
984 next.execution_specs.get(&spec.task_id),
985 ) {
986 (HarnessExpectedExecutionSpecRevisionV1::Absent, None) => {
987 require_first_revision(spec.revision, "execution spec")?;
988 }
989 (HarnessExpectedExecutionSpecRevisionV1::Exact(expected), Some(current)) => {
990 validate_replacement(
991 "execution spec",
992 current.revision,
993 expected,
994 spec.revision,
995 current.created_at_unix_ms,
996 spec.created_at_unix_ms,
997 )?;
998 if current.execution_spec_id != spec.execution_spec_id
999 || current.task_id != spec.task_id
1000 {
1001 return Err(HarnessEngineError::MismatchedIdentity(
1002 "immutable execution spec identity",
1003 ));
1004 }
1005 }
1006 (HarnessExpectedExecutionSpecRevisionV1::Absent, Some(current)) => {
1007 return Err(HarnessEngineError::ExpectedRevisionMismatch {
1008 entity: "execution spec",
1009 expected: 0,
1010 actual: Some(current.revision.get()),
1011 });
1012 }
1013 (HarnessExpectedExecutionSpecRevisionV1::Exact(expected), current) => {
1014 return Err(HarnessEngineError::ExpectedRevisionMismatch {
1015 entity: "execution spec",
1016 expected: expected.get(),
1017 actual: current.map(|spec| spec.revision.get()),
1018 });
1019 }
1020 }
1021 next.execution_specs.insert(spec.task_id.clone(), spec);
1022 next.operations.insert(operation.operation_id.clone(), operation.clone());
1023 next.finish(operation, HarnessApplyOutcome::Applied)
1024 }
1025 HarnessMutationV1::PutIssuedExecutionSpec {
1026 operation,
1027 expected_task_revision,
1028 expected_spec_revision,
1029 expected_issuance_revision,
1030 issuance,
1031 spec,
1032 } => {
1033 require_kind(
1034 operation.kind,
1035 HarnessOperationKindV1::MutateExecutionSpec,
1036 )?;
1037 require_operation_state(operation.state, HarnessOperationStateV1::Succeeded)?;
1038 require_operation_expected(&operation, expected_task_revision)?;
1039 issuance.validate()?;
1040 spec.validate()?;
1041 require_same_id(operation.task_id.as_ref(), &issuance.task_id, "task")?;
1042 if spec.task_id != issuance.task_id {
1043 return Err(HarnessEngineError::MismatchedIdentity(
1044 "issued execution spec task",
1045 ));
1046 }
1047 let task = next.tasks.get(&issuance.task_id)
1048 .ok_or_else(|| HarnessEngineError::NotFound(issuance.task_id.to_string()))?;
1049 if task.revision != expected_task_revision
1050 || issuance.task_revision != expected_task_revision
1051 {
1052 return Err(HarnessEngineError::ExpectedRevisionMismatch {
1053 entity: "task",
1054 expected: expected_task_revision.get(),
1055 actual: Some(task.revision.get()),
1056 });
1057 }
1058 if next.execution_specs.contains_key(&issuance.task_id) {
1059 return Err(HarnessEngineError::DualExecutionSpec(
1060 issuance.task_id.to_string(),
1061 ));
1062 }
1063 validate_issued_execution_spec_link(&issuance, &spec)?;
1064 match (
1065 expected_spec_revision,
1066 expected_issuance_revision,
1067 next.execution_specs_v2.get(&issuance.task_id),
1068 next.issuances.get(&issuance.task_id),
1069 ) {
1070 (
1071 HarnessExpectedExecutionSpecRevisionV1::Absent,
1072 HarnessExpectedExecutionSpecRevisionV1::Absent,
1073 None,
1074 None,
1075 ) => {
1076 require_first_revision(spec.revision, "issued execution spec")?;
1077 require_first_revision(issuance.revision, "task launch issuance")?;
1078 }
1079 (
1080 HarnessExpectedExecutionSpecRevisionV1::Exact(expected_spec),
1081 HarnessExpectedExecutionSpecRevisionV1::Exact(expected_issuance),
1082 Some(current_spec),
1083 Some(current_issuance),
1084 ) => {
1085 validate_replacement(
1086 "issued execution spec",
1087 current_spec.revision,
1088 expected_spec,
1089 spec.revision,
1090 current_spec.created_at_unix_ms,
1091 spec.created_at_unix_ms,
1092 )?;
1093 validate_replacement(
1094 "task launch issuance",
1095 current_issuance.revision,
1096 expected_issuance,
1097 issuance.revision,
1098 current_issuance.created_at_unix_ms,
1099 issuance.created_at_unix_ms,
1100 )?;
1101 if current_spec.execution_spec_id != spec.execution_spec_id
1102 || current_spec.task_id != spec.task_id
1103 || current_issuance.issuance_id != issuance.issuance_id
1104 || current_issuance.task_id != issuance.task_id
1105 {
1106 return Err(HarnessEngineError::MismatchedIdentity(
1107 "immutable issued execution identity",
1108 ));
1109 }
1110 }
1111 (_, _, current_spec, current_issuance) => {
1112 return Err(HarnessEngineError::IssuedExecutionCasMismatch {
1113 spec_revision: current_spec.map(|record| record.revision.get()),
1114 issuance_revision: current_issuance
1115 .map(|record| record.revision.get()),
1116 });
1117 }
1118 }
1119 next.execution_specs_v2.insert(spec.task_id.clone(), spec);
1120 next.issuances.insert(issuance.task_id.clone(), issuance);
1121 next.operations.insert(operation.operation_id.clone(), operation.clone());
1122 next.finish(operation, HarnessApplyOutcome::Applied)
1123 }
1124 HarnessMutationV1::CreateRun {
1125 operation,
1126 expected_task_revision,
1127 task,
1128 run,
1129 } => {
1130 require_kind(operation.kind, HarnessOperationKindV1::CreateRun)?;
1131 require_operation_state(operation.state, HarnessOperationStateV1::Prepared)?;
1132 if run.lifecycle != hatchery_harness_protocol::HarnessRunLifecycleV1::Requested
1133 || run.binding.is_some()
1134 {
1135 return Err(HarnessEngineError::InvalidInitialRunState);
1136 }
1137 require_operation_expected(&operation, expected_task_revision)?;
1138 task.validate()?;
1139 run.validate()?;
1140 require_first_revision(run.revision, "run")?;
1141 require_same_id(operation.run_id.as_ref(), &run.run_id, "run")?;
1142 require_same_id(operation.task_id.as_ref(), &run.task_id, "task")?;
1143 if operation.operation_id != run.operation_id {
1144 return Err(HarnessEngineError::MismatchedIdentity("run operation"));
1145 }
1146 if next.runs.contains_key(&run.run_id) {
1147 return Err(HarnessEngineError::AlreadyExists(run.run_id.to_string()));
1148 }
1149 let current_task = next.tasks.get(&run.task_id)
1150 .ok_or_else(|| HarnessEngineError::NotFound(run.task_id.to_string()))?;
1151 validate_replacement(
1152 "task",
1153 current_task.revision,
1154 expected_task_revision,
1155 task.revision,
1156 current_task.created_at_unix_ms,
1157 task.created_at_unix_ms,
1158 )?;
1159 if task.task_id != run.task_id || task.run_ids.binary_search(&run.run_id).is_err() {
1160 return Err(HarnessEngineError::MismatchedIdentity("task run link"));
1161 }
1162 next.operations.insert(operation.operation_id.clone(), operation.clone());
1163 next.validate_run_links(&run)?;
1164 next.tasks.insert(task.task_id.clone(), task);
1165 next.runs.insert(run.run_id.clone(), run);
1166 next.finish(operation, HarnessApplyOutcome::Applied)
1167 }
1168 HarnessMutationV1::CreateIssuedRun {
1169 operation,
1170 expected_task_revision,
1171 expected_execution_spec_revision,
1172 expected_issuance_revision,
1173 task,
1174 run,
1175 delivery,
1176 continuation,
1177 } => {
1178 require_kind(operation.kind, HarnessOperationKindV1::CreateRun)?;
1179 require_operation_state(operation.state, HarnessOperationStateV1::Prepared)?;
1180 require_operation_expected(&operation, expected_task_revision)?;
1181 if run.lifecycle != HarnessRunLifecycleV1::Requested || run.binding.is_some() {
1182 return Err(HarnessEngineError::InvalidInitialRunState);
1183 }
1184 require_first_revision(run.revision, "run")?;
1185 require_same_id(operation.run_id.as_ref(), &run.run_id, "run")?;
1186 require_same_id(operation.task_id.as_ref(), &run.task_id, "task")?;
1187 if operation.operation_id != run.operation_id {
1188 return Err(HarnessEngineError::MismatchedIdentity("run operation"));
1189 }
1190 if next.runs.contains_key(&run.run_id) {
1191 return Err(HarnessEngineError::AlreadyExists(run.run_id.to_string()));
1192 }
1193 let current_task = next.tasks.get(&run.task_id)
1194 .ok_or_else(|| HarnessEngineError::NotFound(run.task_id.to_string()))?;
1195 let current_spec = next.execution_specs_v2.get(&run.task_id)
1196 .ok_or(HarnessEngineError::IssuedExecutionAuthorityMissing)?;
1197 let current_issuance = next.issuances.get(&run.task_id)
1198 .ok_or(HarnessEngineError::IssuedExecutionAuthorityMissing)?;
1199 if current_spec.revision != expected_execution_spec_revision
1200 || current_issuance.revision != expected_issuance_revision
1201 || current_issuance.task_revision != expected_task_revision
1202 {
1203 return Err(HarnessEngineError::IssuedExecutionCasMismatch {
1204 spec_revision: Some(current_spec.revision.get()),
1205 issuance_revision: Some(current_issuance.revision.get()),
1206 });
1207 }
1208 validate_issued_execution_spec_link(current_issuance, current_spec)?;
1209 if operation.actor != current_task.creator {
1210 return Err(HarnessEngineError::MismatchedIdentity(
1211 "issued run operator actor",
1212 ));
1213 }
1214 validate_issued_run_task_projection(
1215 current_task,
1216 expected_task_revision,
1217 &task,
1218 &run,
1219 )?;
1220 validate_operator_issuance_run_intent(current_issuance, &run)?;
1221 let authority = HarnessTransferAuthorityRefV1::OperatorIssuance {
1222 issuance: current_issuance.reference(),
1223 };
1224 if delivery.is_some() != current_issuance.delivery.is_some()
1225 || continuation.is_some() != current_issuance.context_source.is_some()
1226 || delivery.as_ref().is_some_and(|record| record.authority != authority)
1227 || continuation.as_ref().is_some_and(|record| record.authority != authority)
1228 {
1229 return Err(HarnessEngineError::IssuedExecutionTransferMismatch);
1230 }
1231 if delivery.is_some() && next.deliveries.len() >= HARNESS_DELIVERIES_MAX {
1232 return Err(HarnessEngineError::DeliveryCapacityExceeded);
1233 }
1234 if continuation.is_some()
1235 && next.continuations.len() >= HARNESS_CONTINUATIONS_MAX
1236 {
1237 return Err(HarnessEngineError::ContinuationCapacityExceeded);
1238 }
1239 next.operations.insert(operation.operation_id.clone(), operation.clone());
1240 next.tasks.insert(task.task_id.clone(), task);
1241 next.runs.insert(run.run_id.clone(), run);
1242 if let Some(delivery) = delivery {
1243 if delivery.state != HarnessDeliveryStateV1::Prepared {
1244 return Err(HarnessEngineError::InvalidInitialDeliveryState);
1245 }
1246 require_first_revision(delivery.revision, "delivery")?;
1247 if next.deliveries.contains_key(&delivery.delivery_ref)
1248 || next.deliveries.values()
1249 .any(|record| record.run_id == delivery.run_id)
1250 {
1251 return Err(HarnessEngineError::DeliveryIdConflict {
1252 delivery_ref: delivery.delivery_ref,
1253 });
1254 }
1255 next.validate_delivery_authority(&delivery)?;
1256 next.deliveries.insert(delivery.delivery_ref.clone(), delivery);
1257 }
1258 if let Some(continuation) = continuation {
1259 if continuation.state != HarnessContinuationStateV1::Prepared {
1260 return Err(HarnessEngineError::InvalidInitialContinuationState);
1261 }
1262 require_first_revision(continuation.revision, "continuation")?;
1263 if next.continuations.contains_key(&continuation.continuation_ref)
1264 || next.continuations.values()
1265 .any(|record| record.target_run_id == continuation.target_run_id)
1266 || next.continuations.values()
1267 .any(|record| record.receipt_ref == continuation.receipt_ref)
1268 {
1269 return Err(HarnessEngineError::ContinuationIdConflict {
1270 continuation_ref: continuation.continuation_ref,
1271 });
1272 }
1273 next.validate_continuation_authority(&continuation)?;
1274 next.continuations.insert(
1275 continuation.continuation_ref.clone(),
1276 continuation,
1277 );
1278 }
1279 next.finish(operation, HarnessApplyOutcome::Applied)
1280 }
1281 HarnessMutationV1::ReplaceRun { operation, expected_revision, run } => {
1282 require_kind(operation.kind, HarnessOperationKindV1::MutateRun)?;
1283 require_operation_expected(&operation, expected_revision)?;
1284 run.validate()?;
1285 require_same_id(operation.run_id.as_ref(), &run.run_id, "run")?;
1286 let current = next.runs.get(&run.run_id)
1287 .ok_or_else(|| HarnessEngineError::NotFound(run.run_id.to_string()))?;
1288 validate_replacement(
1289 "run",
1290 current.revision,
1291 expected_revision,
1292 run.revision,
1293 current.created_at_unix_ms,
1294 run.created_at_unix_ms,
1295 )?;
1296 if current.task_id != run.task_id || current.operation_id != run.operation_id {
1297 return Err(HarnessEngineError::MismatchedIdentity("immutable run link"));
1298 }
1299 validate_run_immutable(current, &run)?;
1300 validate_run_lifecycle_transition(current, &run)?;
1301 validate_generic_run_operation(current, &run, &operation)?;
1302 next.validate_run_links(&run)?;
1303 next.runs.insert(run.run_id.clone(), run);
1304 next.operations.insert(operation.operation_id.clone(), operation.clone());
1305 next.finish(operation, HarnessApplyOutcome::Applied)
1306 }
1307 HarnessMutationV1::CreateGrant { operation, grant } => {
1308 require_kind(operation.kind, HarnessOperationKindV1::CreateGrant)?;
1309 require_operation_state(operation.state, HarnessOperationStateV1::Succeeded)?;
1310 grant.validate()?;
1311 if grant.state != hatchery_harness_protocol::SessionGrantStateV1::Active {
1312 return Err(HarnessEngineError::GrantMustStartActive);
1313 }
1314 require_first_revision(grant.revision, "grant")?;
1315 require_same_id(operation.grant_id.as_ref(), &grant.grant_id, "grant")?;
1316 if next.grants.contains_key(&grant.grant_id) {
1317 return Err(HarnessEngineError::AlreadyExists(grant.grant_id.to_string()));
1318 }
1319 if !next.runs.contains_key(&grant.actor_run_id) {
1320 return Err(HarnessEngineError::NotFound(grant.actor_run_id.to_string()));
1321 }
1322 next.grants.insert(grant.grant_id.clone(), grant);
1323 next.operations.insert(operation.operation_id.clone(), operation.clone());
1324 next.finish(operation, HarnessApplyOutcome::Applied)
1325 }
1326 HarnessMutationV1::ReplaceGrant { operation, expected_revision, grant } => {
1327 if !matches!(operation.kind, HarnessOperationKindV1::MutateGrant | HarnessOperationKindV1::RevokeGrant) {
1328 return Err(HarnessEngineError::WrongOperationKind);
1329 }
1330 require_operation_state(operation.state, HarnessOperationStateV1::Succeeded)?;
1331 require_operation_expected(&operation, expected_revision)?;
1332 grant.validate()?;
1333 require_same_id(operation.grant_id.as_ref(), &grant.grant_id, "grant")?;
1334 let current = next.grants.get(&grant.grant_id)
1335 .ok_or_else(|| HarnessEngineError::NotFound(grant.grant_id.to_string()))?;
1336 validate_replacement(
1337 "grant",
1338 current.revision,
1339 expected_revision,
1340 grant.revision,
1341 current.created_at_unix_ms,
1342 grant.created_at_unix_ms,
1343 )?;
1344 if current.actor_run_id != grant.actor_run_id {
1345 return Err(HarnessEngineError::MismatchedIdentity("grant actor run"));
1346 }
1347 validate_grant_transition(current, &grant, operation.kind)?;
1348 next.grants.insert(grant.grant_id.clone(), grant);
1349 next.operations.insert(operation.operation_id.clone(), operation.clone());
1350 next.finish(operation, HarnessApplyOutcome::Applied)
1351 }
1352 }
1353 }
1354
1355 pub fn prepare_delivery(
1356 &self,
1357 delivery: HarnessDeliveryV1,
1358 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1359 delivery.validate()?;
1360 if delivery.state != HarnessDeliveryStateV1::Prepared {
1361 return Err(HarnessEngineError::InvalidInitialDeliveryState);
1362 }
1363 require_first_revision(delivery.revision, "delivery")?;
1364 if let Some(current) = self.deliveries.get(&delivery.delivery_ref) {
1365 if current == &delivery {
1366 let operation = self.delivery_operation(current)?.clone();
1367 return Ok(PreparedHarnessMutation {
1368 next: self.clone(),
1369 outcome: HarnessApplyOutcome::Replayed,
1370 operation,
1371 });
1372 }
1373 return Err(HarnessEngineError::DeliveryIdConflict {
1374 delivery_ref: delivery.delivery_ref,
1375 });
1376 }
1377 if self.deliveries.len() >= HARNESS_DELIVERIES_MAX {
1378 return Err(HarnessEngineError::DeliveryCapacityExceeded);
1379 }
1380 self.validate_delivery_authority(&delivery)?;
1381 let operation = self.delivery_operation(&delivery)?.clone();
1382 let mut next = self.clone();
1383 next.deliveries.insert(delivery.delivery_ref.clone(), delivery);
1384 next.finish(operation, HarnessApplyOutcome::Applied)
1385 }
1386
1387 pub fn prepare_continuation(
1388 &self,
1389 continuation: HarnessContinuationV1,
1390 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1391 continuation.validate()?;
1392 if continuation.state != HarnessContinuationStateV1::Prepared {
1393 return Err(HarnessEngineError::InvalidInitialContinuationState);
1394 }
1395 require_first_revision(continuation.revision, "continuation")?;
1396 if let Some(current) = self.continuations.get(&continuation.continuation_ref) {
1397 if current == &continuation {
1398 return self.finish_continuation_replay(current);
1399 }
1400 return Err(HarnessEngineError::ContinuationIdConflict {
1401 continuation_ref: continuation.continuation_ref,
1402 });
1403 }
1404 if self.continuations.len() >= HARNESS_CONTINUATIONS_MAX {
1405 return Err(HarnessEngineError::ContinuationCapacityExceeded);
1406 }
1407 let operation = self.continuation_operation(&continuation)?.clone();
1408 let mut next = self.clone();
1409 next.validate_continuation_authority(&continuation)?;
1410 next.continuations.insert(continuation.continuation_ref.clone(), continuation);
1411 next.finish(operation, HarnessApplyOutcome::Applied)
1412 }
1413
1414 pub fn prepare_scheduled_run_authorities(
1415 &self,
1416 operation_id: &HarnessOperationId,
1417 delivery: Option<HarnessDeliveryV1>,
1418 continuation: Option<HarnessContinuationV1>,
1419 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1420 let operation = self.operations.get(operation_id)
1421 .ok_or_else(|| HarnessEngineError::NotFound(operation_id.to_string()))?
1422 .clone();
1423 let run = operation.run_id.as_ref()
1424 .and_then(|run_id| self.runs.get(run_id))
1425 .ok_or(HarnessEngineError::SchedulerInvalidGraph)?;
1426 if operation.state != HarnessOperationStateV1::Prepared
1427 || run.lifecycle != HarnessRunLifecycleV1::Requested
1428 {
1429 return Err(HarnessEngineError::DeliveryAuthorityWindowClosed);
1430 }
1431 let mut next = self.clone();
1432 let mut applied = false;
1433 if let Some(delivery) = delivery {
1434 delivery.validate()?;
1435 if delivery.operation_id != *operation_id
1436 || delivery.state != HarnessDeliveryStateV1::Prepared
1437 || delivery.revision.get() != 1
1438 {
1439 return Err(HarnessEngineError::InvalidInitialDeliveryState);
1440 }
1441 match self.deliveries.get(&delivery.delivery_ref) {
1442 Some(current) if current == &delivery => {}
1443 Some(_) => return Err(HarnessEngineError::DeliveryIdConflict {
1444 delivery_ref: delivery.delivery_ref,
1445 }),
1446 None => {
1447 if next.deliveries.len() >= HARNESS_DELIVERIES_MAX {
1448 return Err(HarnessEngineError::DeliveryCapacityExceeded);
1449 }
1450 next.validate_delivery_authority(&delivery)?;
1451 next.deliveries.insert(delivery.delivery_ref.clone(), delivery);
1452 applied = true;
1453 }
1454 }
1455 }
1456 if let Some(continuation) = continuation {
1457 continuation.validate()?;
1458 if continuation.operation_id != *operation_id
1459 || continuation.state != HarnessContinuationStateV1::Prepared
1460 || continuation.revision.get() != 1
1461 {
1462 return Err(HarnessEngineError::InvalidInitialContinuationState);
1463 }
1464 match self.continuations.get(&continuation.continuation_ref) {
1465 Some(current) if current == &continuation => {}
1466 Some(_) => return Err(HarnessEngineError::ContinuationIdConflict {
1467 continuation_ref: continuation.continuation_ref,
1468 }),
1469 None => {
1470 if next.continuations.len() >= HARNESS_CONTINUATIONS_MAX {
1471 return Err(HarnessEngineError::ContinuationCapacityExceeded);
1472 }
1473 next.validate_continuation_authority(&continuation)?;
1474 next.continuations.insert(
1475 continuation.continuation_ref.clone(),
1476 continuation,
1477 );
1478 applied = true;
1479 }
1480 }
1481 }
1482 next.finish(
1483 operation,
1484 if applied {
1485 HarnessApplyOutcome::Applied
1486 } else {
1487 HarnessApplyOutcome::Replayed
1488 },
1489 )
1490 }
1491
1492 pub fn prepare_continuation_export(
1493 &self,
1494 expected_revision: HarnessRevision,
1495 continuation: HarnessContinuationV1,
1496 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1497 continuation.validate()?;
1498 let current = self.continuations.get(&continuation.continuation_ref)
1499 .ok_or_else(|| HarnessEngineError::NotFound(continuation.continuation_ref.to_string()))?;
1500 if current == &continuation {
1501 return self.finish_continuation_replay(current);
1502 }
1503 validate_continuation_replacement(current, expected_revision, &continuation)?;
1504 if current.state != HarnessContinuationStateV1::Exporting
1505 || continuation.state != HarnessContinuationStateV1::Exported
1506 {
1507 return Err(HarnessEngineError::InvalidContinuationTransition);
1508 }
1509 let operation = self.continuation_operation(&continuation)?.clone();
1510 let current_run = self.runs.get(&continuation.target_run_id)
1511 .ok_or_else(|| HarnessEngineError::NotFound(
1512 continuation.target_run_id.to_string(),
1513 ))?;
1514 if current_run.continuation_receipt.is_some() {
1515 return Err(HarnessEngineError::InvalidContinuationRunBind);
1516 }
1517 let mut linked_run = current_run.clone();
1518 linked_run.revision = HarnessRevision::new(
1519 current_run.revision.get().checked_add(1)
1520 .ok_or(HarnessEngineError::InvalidNextRevision { entity: "run" })?,
1521 )?;
1522 linked_run.continuation_receipt = Some(continuation.receipt_ref.clone());
1523 linked_run.updated_at_unix_ms = continuation.updated_at_unix_ms;
1524 linked_run.validate()?;
1525 let mut next = self.clone();
1526 next.continuations.insert(
1527 continuation.continuation_ref.clone(),
1528 continuation.clone(),
1529 );
1530 next.runs.insert(linked_run.run_id.clone(), linked_run);
1531 next.validate_continuation_authority(&continuation)?;
1532 next.finish(operation, HarnessApplyOutcome::Applied)
1533 }
1534
1535 pub fn prepare_continuation_export_begin(
1536 &self,
1537 expected_revision: HarnessRevision,
1538 continuation: HarnessContinuationV1,
1539 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1540 continuation.validate()?;
1541 let current = self.continuations.get(&continuation.continuation_ref)
1542 .ok_or_else(|| HarnessEngineError::NotFound(continuation.continuation_ref.to_string()))?;
1543 if current == &continuation {
1544 return self.finish_continuation_replay(current);
1545 }
1546 validate_continuation_replacement(current, expected_revision, &continuation)?;
1547 if current.state != HarnessContinuationStateV1::Prepared
1548 || continuation.state != HarnessContinuationStateV1::Exporting
1549 {
1550 return Err(HarnessEngineError::InvalidContinuationTransition);
1551 }
1552 self.validate_continuation_authority(&continuation)?;
1553 let operation = self.continuation_operation(&continuation)?.clone();
1554 let mut next = self.clone();
1555 next.continuations.insert(continuation.continuation_ref.clone(), continuation);
1556 next.finish(operation, HarnessApplyOutcome::Applied)
1557 }
1558
1559 pub fn prepare_continuation_export_outcome_unknown(
1560 &self,
1561 expected_revision: HarnessRevision,
1562 continuation: HarnessContinuationV1,
1563 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1564 continuation.validate()?;
1565 let current = self.continuations.get(&continuation.continuation_ref)
1566 .ok_or_else(|| HarnessEngineError::NotFound(continuation.continuation_ref.to_string()))?;
1567 if current == &continuation {
1568 return self.finish_continuation_replay(current);
1569 }
1570 validate_continuation_replacement(current, expected_revision, &continuation)?;
1571 if current.state != HarnessContinuationStateV1::Exporting
1572 || continuation.state != HarnessContinuationStateV1::OutcomeUnknown
1573 {
1574 return Err(HarnessEngineError::InvalidContinuationTransition);
1575 }
1576 let operation = self.continuation_operation(&continuation)?.clone();
1577 let mut next = self.clone();
1578 next.continuations.insert(continuation.continuation_ref.clone(), continuation);
1579 next.finish(operation, HarnessApplyOutcome::Applied)
1580 }
1581
1582 pub fn prepare_continuation_expiry(
1583 &self,
1584 expected_revision: HarnessRevision,
1585 continuation: HarnessContinuationV1,
1586 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1587 continuation.validate()?;
1588 let current = self.continuations.get(&continuation.continuation_ref)
1589 .ok_or_else(|| HarnessEngineError::NotFound(continuation.continuation_ref.to_string()))?;
1590 if current == &continuation {
1591 return self.finish_continuation_replay(current);
1592 }
1593 validate_continuation_replacement(current, expected_revision, &continuation)?;
1594 if !matches!(
1595 current.state,
1596 HarnessContinuationStateV1::Prepared
1597 | HarnessContinuationStateV1::Exporting
1598 | HarnessContinuationStateV1::Exported
1599 | HarnessContinuationStateV1::OutcomeUnknown
1600 )
1601 || continuation.state != HarnessContinuationStateV1::Expired
1602 {
1603 return Err(HarnessEngineError::InvalidContinuationTransition);
1604 }
1605 let operation = self.continuation_operation(&continuation)?.clone();
1606 let mut next = self.clone();
1607 next.continuations.insert(continuation.continuation_ref.clone(), continuation);
1608 next.finish(operation, HarnessApplyOutcome::Applied)
1609 }
1610
1611 pub fn prepare_accepted_spawn_continuation_bind(
1612 &self,
1613 expected_run_revision: HarnessRevision,
1614 run: HarnessRunV1,
1615 expected_operation_revision: HarnessRevision,
1616 operation: HarnessOperationV1,
1617 expected_continuation_revision: HarnessRevision,
1618 continuation: HarnessContinuationV1,
1619 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1620 continuation.validate()?;
1621 run.validate()?;
1622 operation.validate()?;
1623 let current_continuation = self.continuations.get(&continuation.continuation_ref)
1624 .ok_or_else(|| HarnessEngineError::NotFound(continuation.continuation_ref.to_string()))?;
1625 let current_run = self.runs.get(&run.run_id)
1626 .ok_or_else(|| HarnessEngineError::NotFound(run.run_id.to_string()))?;
1627 let current_operation = self.operations.get(&operation.operation_id)
1628 .ok_or_else(|| HarnessEngineError::NotFound(operation.operation_id.to_string()))?;
1629 if current_continuation == &continuation
1630 && current_run == &run
1631 && current_operation == &operation
1632 {
1633 return Ok(PreparedHarnessMutation {
1634 next: self.clone(),
1635 outcome: HarnessApplyOutcome::Replayed,
1636 operation,
1637 });
1638 }
1639 validate_continuation_replacement(
1640 current_continuation,
1641 expected_continuation_revision,
1642 &continuation,
1643 )?;
1644 if current_continuation.state != HarnessContinuationStateV1::Exported
1645 || continuation.state != HarnessContinuationStateV1::Bound
1646 {
1647 return Err(HarnessEngineError::InvalidContinuationTransition);
1648 }
1649 validate_replacement(
1650 "run",
1651 current_run.revision,
1652 expected_run_revision,
1653 run.revision,
1654 current_run.created_at_unix_ms,
1655 run.created_at_unix_ms,
1656 )?;
1657 validate_run_immutable(current_run, &run)?;
1658 validate_operation_transition(current_operation, expected_operation_revision, &operation)?;
1659 validate_run_lifecycle_transition(current_run, &run)?;
1660 validate_coupled_run_operation(current_run, &run, &operation)?;
1661 if continuation.target_run_id != run.run_id
1662 || continuation.operation_id != operation.operation_id
1663 || current_run.continuation_receipt.as_ref() != Some(&continuation.receipt_ref)
1664 || run.continuation_receipt.as_ref() != Some(&continuation.receipt_ref)
1665 || continuation.target_binding.as_ref() != run.binding.as_ref()
1666 || current_run.delivery_receipt != run.delivery_receipt
1667 || current_run.result_disposition != run.result_disposition
1668 || current_run.failure != run.failure
1669 {
1670 return Err(HarnessEngineError::InvalidContinuationRunBind);
1671 }
1672 let mut next = self.clone();
1673 next.continuations.insert(continuation.continuation_ref.clone(), continuation);
1674 next.runs.insert(run.run_id.clone(), run);
1675 next.operations.insert(operation.operation_id.clone(), operation.clone());
1676 next.finish(operation, HarnessApplyOutcome::Applied)
1677 }
1678
1679 pub fn prepare_accepted_spawn_delivery_and_continuation_commit(
1680 &self,
1681 expected_run_revision: HarnessRevision,
1682 run: HarnessRunV1,
1683 expected_operation_revision: HarnessRevision,
1684 operation: HarnessOperationV1,
1685 expected_delivery_revision: HarnessRevision,
1686 delivery: HarnessDeliveryV1,
1687 expected_continuation_revision: HarnessRevision,
1688 continuation: HarnessContinuationV1,
1689 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1690 run.validate()?;
1691 operation.validate()?;
1692 delivery.validate()?;
1693 continuation.validate()?;
1694 let current_run = self.runs.get(&run.run_id)
1695 .ok_or_else(|| HarnessEngineError::NotFound(run.run_id.to_string()))?;
1696 let current_operation = self.operations.get(&operation.operation_id)
1697 .ok_or_else(|| HarnessEngineError::NotFound(operation.operation_id.to_string()))?;
1698 let current_delivery = self.deliveries.get(&delivery.delivery_ref)
1699 .ok_or_else(|| HarnessEngineError::NotFound(delivery.delivery_ref.to_string()))?;
1700 let current_continuation = self.continuations.get(&continuation.continuation_ref)
1701 .ok_or_else(|| HarnessEngineError::NotFound(continuation.continuation_ref.to_string()))?;
1702 if current_run == &run
1703 && current_operation == &operation
1704 && current_delivery == &delivery
1705 && current_continuation == &continuation
1706 {
1707 return Ok(PreparedHarnessMutation {
1708 next: self.clone(),
1709 outcome: HarnessApplyOutcome::Replayed,
1710 operation,
1711 });
1712 }
1713 validate_delivery_replacement(current_delivery, expected_delivery_revision, &delivery)?;
1714 validate_continuation_replacement(
1715 current_continuation,
1716 expected_continuation_revision,
1717 &continuation,
1718 )?;
1719 if current_delivery.state != HarnessDeliveryStateV1::Staged
1720 || delivery.state != HarnessDeliveryStateV1::Committed
1721 || current_continuation.state != HarnessContinuationStateV1::Exported
1722 || continuation.state != HarnessContinuationStateV1::Bound
1723 {
1724 return Err(HarnessEngineError::InvalidCombinedSpawnCommit);
1725 }
1726 let delivery_receipt = delivery.receipt.as_ref()
1727 .ok_or(HarnessEngineError::InvalidCombinedSpawnCommit)?;
1728 validate_replacement(
1729 "run",
1730 current_run.revision,
1731 expected_run_revision,
1732 run.revision,
1733 current_run.created_at_unix_ms,
1734 run.created_at_unix_ms,
1735 )?;
1736 validate_run_immutable(current_run, &run)?;
1737 validate_operation_transition(current_operation, expected_operation_revision, &operation)?;
1738 validate_run_lifecycle_transition(current_run, &run)?;
1739 validate_coupled_run_operation(current_run, &run, &operation)?;
1740 if delivery.run_id != run.run_id
1741 || continuation.target_run_id != run.run_id
1742 || delivery.operation_id != operation.operation_id
1743 || continuation.operation_id != operation.operation_id
1744 || current_run.delivery_receipt.is_some()
1745 || current_run.continuation_receipt.as_ref() != Some(&continuation.receipt_ref)
1746 || run.delivery_receipt.as_ref() != Some(&delivery_receipt.receipt_ref)
1747 || run.continuation_receipt.as_ref() != Some(&continuation.receipt_ref)
1748 || run.binding.as_ref() != Some(&delivery_receipt.binding)
1749 || continuation.target_binding.as_ref() != run.binding.as_ref()
1750 || current_run.result_disposition != run.result_disposition
1751 || current_run.failure != run.failure
1752 {
1753 return Err(HarnessEngineError::InvalidCombinedSpawnCommit);
1754 }
1755 let mut next = self.clone();
1756 next.deliveries.insert(delivery.delivery_ref.clone(), delivery);
1757 next.continuations.insert(continuation.continuation_ref.clone(), continuation);
1758 next.runs.insert(run.run_id.clone(), run);
1759 next.operations.insert(operation.operation_id.clone(), operation.clone());
1760 next.finish(operation, HarnessApplyOutcome::Applied)
1761 }
1762
1763 pub fn prepare_delivery_stage(
1764 &self,
1765 expected_revision: HarnessRevision,
1766 delivery: HarnessDeliveryV1,
1767 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1768 delivery.validate()?;
1769 let current = self.deliveries.get(&delivery.delivery_ref)
1770 .ok_or_else(|| HarnessEngineError::NotFound(delivery.delivery_ref.to_string()))?;
1771 if current == &delivery {
1772 let operation = self.delivery_operation(current)?.clone();
1773 return Ok(PreparedHarnessMutation {
1774 next: self.clone(),
1775 outcome: HarnessApplyOutcome::Replayed,
1776 operation,
1777 });
1778 }
1779 match (current.state, delivery.state) {
1780 (HarnessDeliveryStateV1::Prepared, HarnessDeliveryStateV1::Staged) => {
1781 validate_delivery_replacement(current, expected_revision, &delivery)?;
1782 }
1783 (HarnessDeliveryStateV1::Staged, HarnessDeliveryStateV1::Staged) => {
1784 validate_delivery_restage(current, expected_revision, &delivery)?;
1785 }
1786 _ => return Err(HarnessEngineError::InvalidDeliveryTransition),
1787 }
1788 self.validate_delivery_authority(&delivery)?;
1789 let operation = self.delivery_operation(&delivery)?.clone();
1790 let mut next = self.clone();
1791 next.deliveries.insert(delivery.delivery_ref.clone(), delivery);
1792 next.finish(operation, HarnessApplyOutcome::Applied)
1793 }
1794
1795 pub fn prepare_accepted_spawn_delivery_commit(
1796 &self,
1797 expected_run_revision: HarnessRevision,
1798 run: HarnessRunV1,
1799 expected_operation_revision: HarnessRevision,
1800 operation: HarnessOperationV1,
1801 expected_delivery_revision: HarnessRevision,
1802 delivery: HarnessDeliveryV1,
1803 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1804 delivery.validate()?;
1805 run.validate()?;
1806 operation.validate()?;
1807 let current_delivery = self.deliveries.get(&delivery.delivery_ref)
1808 .ok_or_else(|| HarnessEngineError::NotFound(delivery.delivery_ref.to_string()))?;
1809 let current_run = self.runs.get(&run.run_id)
1810 .ok_or_else(|| HarnessEngineError::NotFound(run.run_id.to_string()))?;
1811 let current_operation = self.operations.get(&operation.operation_id)
1812 .ok_or_else(|| HarnessEngineError::NotFound(operation.operation_id.to_string()))?;
1813 if current_delivery == &delivery
1814 && current_run == &run
1815 && current_operation == &operation
1816 {
1817 return Ok(PreparedHarnessMutation {
1818 next: self.clone(),
1819 outcome: HarnessApplyOutcome::Replayed,
1820 operation,
1821 });
1822 }
1823 validate_delivery_replacement(current_delivery, expected_delivery_revision, &delivery)?;
1824 if current_delivery.state != HarnessDeliveryStateV1::Staged
1825 || delivery.state != HarnessDeliveryStateV1::Committed
1826 {
1827 return Err(HarnessEngineError::InvalidDeliveryTransition);
1828 }
1829 let receipt = delivery.receipt.as_ref()
1830 .ok_or(HarnessEngineError::InvalidDeliveryTransition)?;
1831 validate_replacement(
1832 "run",
1833 current_run.revision,
1834 expected_run_revision,
1835 run.revision,
1836 current_run.created_at_unix_ms,
1837 run.created_at_unix_ms,
1838 )?;
1839 validate_run_immutable(current_run, &run)?;
1840 validate_operation_transition(
1841 current_operation,
1842 expected_operation_revision,
1843 &operation,
1844 )?;
1845 validate_run_lifecycle_transition(current_run, &run)?;
1846 validate_coupled_run_operation(current_run, &run, &operation)?;
1847 if run.operation_id != operation.operation_id
1848 || delivery.operation_id != operation.operation_id
1849 || delivery.run_id != run.run_id
1850 || current_run.delivery_receipt.is_some()
1851 || run.delivery_receipt.as_ref() != Some(&receipt.receipt_ref)
1852 || current_run.continuation_receipt != run.continuation_receipt
1853 || current_run.result_disposition != run.result_disposition
1854 || current_run.failure != run.failure
1855 || run.binding.as_ref() != Some(&receipt.binding)
1856 {
1857 return Err(HarnessEngineError::InvalidDeliveryRunCommit);
1858 }
1859 let mut next = self.clone();
1860 next.deliveries.insert(delivery.delivery_ref.clone(), delivery);
1861 next.runs.insert(run.run_id.clone(), run);
1862 next.operations.insert(operation.operation_id.clone(), operation.clone());
1863 next.finish(operation, HarnessApplyOutcome::Applied)
1864 }
1865
1866 pub fn prepare_operation_transition(
1867 &self,
1868 expected_revision: HarnessRevision,
1869 operation: HarnessOperationV1,
1870 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1871 operation.validate()?;
1872 if operation.run_id.is_some() {
1873 return Err(HarnessEngineError::RunOperationRequiresAtomicTransition);
1874 }
1875 let current = self.operations.get(&operation.operation_id)
1876 .ok_or_else(|| HarnessEngineError::NotFound(operation.operation_id.to_string()))?;
1877 validate_operation_transition(current, expected_revision, &operation)?;
1878 let mut next = self.clone();
1879 next.operations.insert(operation.operation_id.clone(), operation.clone());
1880 next.finish(operation, HarnessApplyOutcome::Applied)
1881 }
1882
1883 pub fn prepare_run_operation_transition(
1884 &self,
1885 expected_run_revision: HarnessRevision,
1886 run: HarnessRunV1,
1887 expected_operation_revision: HarnessRevision,
1888 operation: HarnessOperationV1,
1889 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1890 run.validate()?;
1891 operation.validate()?;
1892 let current_run = self.runs.get(&run.run_id)
1893 .ok_or_else(|| HarnessEngineError::NotFound(run.run_id.to_string()))?;
1894 validate_replacement(
1895 "run",
1896 current_run.revision,
1897 expected_run_revision,
1898 run.revision,
1899 current_run.created_at_unix_ms,
1900 run.created_at_unix_ms,
1901 )?;
1902 if current_run.task_id != run.task_id || current_run.operation_id != run.operation_id {
1903 return Err(HarnessEngineError::MismatchedIdentity("immutable run link"));
1904 }
1905 validate_run_immutable(current_run, &run)?;
1906 let current_operation = self.operations.get(&operation.operation_id)
1907 .ok_or_else(|| HarnessEngineError::NotFound(operation.operation_id.to_string()))?;
1908 validate_operation_transition(current_operation, expected_operation_revision, &operation)?;
1909 if run.operation_id != operation.operation_id {
1910 return Err(HarnessEngineError::MismatchedIdentity("run operation"));
1911 }
1912 validate_run_lifecycle_transition(current_run, &run)?;
1913 validate_coupled_run_operation(current_run, &run, &operation)?;
1914 let mut next = self.clone();
1915 next.runs.insert(run.run_id.clone(), run);
1916 next.operations.insert(operation.operation_id.clone(), operation.clone());
1917 next.finish(operation, HarnessApplyOutcome::Applied)
1918 }
1919
1920 pub fn prepare_run_event_commit(
1924 &self,
1925 operation: HarnessOperationV1,
1926 expected_run_revision: HarnessRevision,
1927 run: HarnessRunV1,
1928 expected_task_revision: HarnessRevision,
1929 task: HarnessTaskV1,
1930 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
1931 operation.validate()?;
1932 run.validate()?;
1933 task.validate()?;
1934 if let Some(current_operation) = self.operations.get(&operation.operation_id) {
1935 let exact_replay = current_operation == &operation
1936 && self.runs.get(&run.run_id) == Some(&run)
1937 && self.tasks.get(&task.task_id) == Some(&task)
1938 && operation.expected_revision == Some(expected_run_revision)
1939 && run.revision.get() == expected_run_revision.get().checked_add(1)
1940 .ok_or(HarnessEngineError::InvalidNextRevision { entity: "run" })?
1941 && task.revision.get() == expected_task_revision.get().checked_add(1)
1942 .ok_or(HarnessEngineError::InvalidNextRevision { entity: "task" })?;
1943 if exact_replay {
1944 return Ok(PreparedHarnessMutation {
1945 next: self.clone(),
1946 outcome: HarnessApplyOutcome::Replayed,
1947 operation,
1948 });
1949 }
1950 return Err(HarnessEngineError::OperationIdConflict {
1951 operation_id: operation.operation_id,
1952 });
1953 }
1954 require_first_revision(operation.revision, "operation")?;
1955 require_kind(operation.kind, HarnessOperationKindV1::MutateRun)?;
1956 require_operation_state(operation.state, HarnessOperationStateV1::Succeeded)?;
1957 require_operation_expected(&operation, expected_run_revision)?;
1958 require_same_id(operation.run_id.as_ref(), &run.run_id, "run")?;
1959 if operation.actor != (HarnessActorV1::ParentRun {
1960 run_id: run.run_id.clone(),
1961 })
1962 || operation.grant_id.is_some()
1963 || operation.reconciles_operation_id.is_some()
1964 || run.task_id != task.task_id
1965 || task.run_ids.binary_search(&run.run_id).is_err()
1966 || task.updated_at_unix_ms != run.updated_at_unix_ms
1967 || operation.updated_at_unix_ms != run.updated_at_unix_ms
1968 || operation.finished_at_unix_ms != Some(run.updated_at_unix_ms)
1969 {
1970 return Err(HarnessEngineError::InvalidRunEventAuthority);
1971 }
1972 let current_run = self.runs.get(&run.run_id)
1973 .ok_or_else(|| HarnessEngineError::NotFound(run.run_id.to_string()))?;
1974 let current_task = self.tasks.get(&task.task_id)
1975 .ok_or_else(|| HarnessEngineError::NotFound(task.task_id.to_string()))?;
1976 validate_replacement(
1977 "run",
1978 current_run.revision,
1979 expected_run_revision,
1980 run.revision,
1981 current_run.created_at_unix_ms,
1982 run.created_at_unix_ms,
1983 )?;
1984 validate_replacement(
1985 "task",
1986 current_task.revision,
1987 expected_task_revision,
1988 task.revision,
1989 current_task.created_at_unix_ms,
1990 task.created_at_unix_ms,
1991 )?;
1992 validate_run_immutable(current_run, &run)?;
1993 validate_run_lifecycle_transition(current_run, &run)?;
1994 validate_generic_run_operation(current_run, &run, &operation)?;
1995 validate_run_event_task_projection(current_task, &task, &run)?;
1996 let mut next = self.clone();
1997 next.runs.insert(run.run_id.clone(), run);
1998 next.tasks.insert(task.task_id.clone(), task);
1999 next.operations.insert(operation.operation_id.clone(), operation.clone());
2000 next.finish(operation, HarnessApplyOutcome::Applied)
2001 }
2002
2003 pub fn prepare_run_incarnation_settlement(
2029 &self,
2030 operation: HarnessOperationV1,
2031 expected_run_revision: HarnessRevision,
2032 run: HarnessRunV1,
2033 expected_task_revision: HarnessRevision,
2034 task: HarnessTaskV1,
2035 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
2036 operation.validate()?;
2037 run.validate()?;
2038 task.validate()?;
2039 if let Some(current_operation) = self.operations.get(&operation.operation_id) {
2040 let exact_replay = current_operation == &operation
2041 && self.runs.get(&run.run_id) == Some(&run)
2042 && self.tasks.get(&task.task_id) == Some(&task)
2043 && operation.expected_revision == Some(expected_run_revision)
2044 && run.revision.get() == expected_run_revision.get().checked_add(1)
2045 .ok_or(HarnessEngineError::InvalidNextRevision { entity: "run" })?
2046 && task.revision.get() == expected_task_revision.get().checked_add(1)
2047 .ok_or(HarnessEngineError::InvalidNextRevision { entity: "task" })?;
2048 if exact_replay {
2049 return Ok(PreparedHarnessMutation {
2050 next: self.clone(),
2051 outcome: HarnessApplyOutcome::Replayed,
2052 operation,
2053 });
2054 }
2055 return Err(HarnessEngineError::OperationIdConflict {
2056 operation_id: operation.operation_id,
2057 });
2058 }
2059 require_first_revision(operation.revision, "operation")?;
2060 require_kind(operation.kind, HarnessOperationKindV1::MutateRun)?;
2061 require_operation_state(operation.state, HarnessOperationStateV1::Succeeded)?;
2071 require_operation_expected(&operation, expected_run_revision)?;
2072 require_same_id(operation.run_id.as_ref(), &run.run_id, "run")?;
2073 if operation.actor != (HarnessActorV1::ParentRun {
2074 run_id: run.run_id.clone(),
2075 })
2076 || operation.grant_id.is_some()
2077 || operation.reconciles_operation_id.is_some()
2078 || run.task_id != task.task_id
2079 || task.run_ids.binary_search(&run.run_id).is_err()
2080 || task.updated_at_unix_ms != run.updated_at_unix_ms
2081 || operation.updated_at_unix_ms != run.updated_at_unix_ms
2082 || operation.finished_at_unix_ms != Some(run.updated_at_unix_ms)
2083 {
2084 return Err(HarnessEngineError::InvalidIncarnationSettlementAuthority);
2085 }
2086 let current_run = self.runs.get(&run.run_id)
2087 .ok_or_else(|| HarnessEngineError::NotFound(run.run_id.to_string()))?;
2088 let current_task = self.tasks.get(&task.task_id)
2089 .ok_or_else(|| HarnessEngineError::NotFound(task.task_id.to_string()))?;
2090 validate_replacement(
2091 "run",
2092 current_run.revision,
2093 expected_run_revision,
2094 run.revision,
2095 current_run.created_at_unix_ms,
2096 run.created_at_unix_ms,
2097 )?;
2098 validate_replacement(
2099 "task",
2100 current_task.revision,
2101 expected_task_revision,
2102 task.revision,
2103 current_task.created_at_unix_ms,
2104 task.created_at_unix_ms,
2105 )?;
2106 validate_run_immutable(current_run, &run)?;
2107 validate_run_lifecycle_transition(current_run, &run)?;
2108 validate_incarnation_settlement_task_projection(current_task, &task, &run)?;
2109 let mut next = self.clone();
2110 next.runs.insert(run.run_id.clone(), run);
2111 next.tasks.insert(task.task_id.clone(), task);
2112 next.operations.insert(operation.operation_id.clone(), operation.clone());
2113 next.finish(operation, HarnessApplyOutcome::Applied)
2114 }
2115
2116 pub fn prepare_run_context_pack_record(
2123 &self,
2124 operation: HarnessOperationV1,
2125 expected_run_revision: HarnessRevision,
2126 run: HarnessRunV1,
2127 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
2128 operation.validate()?;
2129 run.validate()?;
2130 let current_run = self.runs.get(&run.run_id)
2131 .ok_or_else(|| HarnessEngineError::NotFound(run.run_id.to_string()))?;
2132 validate_context_pack_recording(current_run, &run)?;
2133 if current_run.context_pack.is_some() {
2134 return Ok(PreparedHarnessMutation {
2138 next: self.clone(),
2139 outcome: HarnessApplyOutcome::Replayed,
2140 operation,
2141 });
2142 }
2143 if self.operations.contains_key(&operation.operation_id) {
2144 return Err(HarnessEngineError::OperationIdConflict {
2145 operation_id: operation.operation_id,
2146 });
2147 }
2148 require_first_revision(operation.revision, "operation")?;
2149 require_kind(operation.kind, HarnessOperationKindV1::RecordRunContextPack)?;
2150 require_operation_state(operation.state, HarnessOperationStateV1::Succeeded)?;
2151 require_operation_expected(&operation, expected_run_revision)?;
2152 require_same_id(operation.run_id.as_ref(), &run.run_id, "run")?;
2153 if operation.actor != (HarnessActorV1::ParentRun { run_id: run.run_id.clone() })
2154 || operation.task_id.is_some()
2155 || operation.grant_id.is_some()
2156 || operation.reconciles_operation_id.is_some()
2157 || operation.updated_at_unix_ms != run.updated_at_unix_ms
2158 || operation.finished_at_unix_ms != Some(run.updated_at_unix_ms)
2159 {
2160 return Err(HarnessEngineError::InvalidContextPackRecordAuthority);
2161 }
2162 validate_replacement(
2163 "run",
2164 current_run.revision,
2165 expected_run_revision,
2166 run.revision,
2167 current_run.created_at_unix_ms,
2168 run.created_at_unix_ms,
2169 )?;
2170 let mut next = self.clone();
2171 next.runs.insert(run.run_id.clone(), run);
2172 next.operations.insert(operation.operation_id.clone(), operation.clone());
2173 next.finish(operation, HarnessApplyOutcome::Applied)
2174 }
2175
2176 pub fn prepare_run_git_facts_record(
2185 &self,
2186 operation: HarnessOperationV1,
2187 expected_run_revision: HarnessRevision,
2188 run: HarnessRunV1,
2189 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
2190 operation.validate()?;
2191 run.validate()?;
2192 let current_run = self.runs.get(&run.run_id)
2193 .ok_or_else(|| HarnessEngineError::NotFound(run.run_id.to_string()))?;
2194 validate_git_facts_recording(current_run, &run)?;
2195 if current_run.git_facts.is_some() {
2196 return Ok(PreparedHarnessMutation {
2200 next: self.clone(),
2201 outcome: HarnessApplyOutcome::Replayed,
2202 operation,
2203 });
2204 }
2205 if self.operations.contains_key(&operation.operation_id) {
2206 return Err(HarnessEngineError::OperationIdConflict {
2207 operation_id: operation.operation_id,
2208 });
2209 }
2210 require_first_revision(operation.revision, "operation")?;
2211 require_kind(operation.kind, HarnessOperationKindV1::RecordRunGitFacts)?;
2212 require_operation_state(operation.state, HarnessOperationStateV1::Succeeded)?;
2213 require_operation_expected(&operation, expected_run_revision)?;
2214 require_same_id(operation.run_id.as_ref(), &run.run_id, "run")?;
2215 if operation.actor != (HarnessActorV1::ParentRun { run_id: run.run_id.clone() })
2216 || operation.task_id.is_some()
2217 || operation.grant_id.is_some()
2218 || operation.reconciles_operation_id.is_some()
2219 || operation.updated_at_unix_ms != run.updated_at_unix_ms
2220 || operation.finished_at_unix_ms != Some(run.updated_at_unix_ms)
2221 {
2222 return Err(HarnessEngineError::InvalidGitFactsRecordAuthority);
2223 }
2224 validate_replacement(
2225 "run",
2226 current_run.revision,
2227 expected_run_revision,
2228 run.revision,
2229 current_run.created_at_unix_ms,
2230 run.created_at_unix_ms,
2231 )?;
2232 let mut next = self.clone();
2233 next.runs.insert(run.run_id.clone(), run);
2234 next.operations.insert(operation.operation_id.clone(), operation.clone());
2235 next.finish(operation, HarnessApplyOutcome::Applied)
2236 }
2237
2238 pub fn prepare_task_result_ref_record(
2246 &self,
2247 operation: HarnessOperationV1,
2248 expected_task_revision: HarnessRevision,
2249 task: HarnessTaskV1,
2250 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
2251 operation.validate()?;
2252 task.validate()?;
2253 let current_task = self.tasks.get(&task.task_id)
2254 .ok_or_else(|| HarnessEngineError::NotFound(task.task_id.to_string()))?;
2255 validate_task_result_ref_recording(current_task, &task)?;
2256 if current_task.result_refs == task.result_refs {
2257 return Ok(PreparedHarnessMutation {
2261 next: self.clone(),
2262 outcome: HarnessApplyOutcome::Replayed,
2263 operation,
2264 });
2265 }
2266 let inserted = task.result_refs.iter()
2267 .find(|candidate| current_task.result_refs.binary_search(candidate).is_err())
2268 .ok_or(HarnessEngineError::InvalidResultRefRecordProjection)?;
2269 if self.operations.contains_key(&operation.operation_id) {
2270 return Err(HarnessEngineError::OperationIdConflict {
2271 operation_id: operation.operation_id,
2272 });
2273 }
2274 require_first_revision(operation.revision, "operation")?;
2275 require_kind(operation.kind, HarnessOperationKindV1::RecordTaskResultRef)?;
2276 require_operation_state(operation.state, HarnessOperationStateV1::Succeeded)?;
2277 require_operation_expected(&operation, expected_task_revision)?;
2278 require_same_id(operation.task_id.as_ref(), &task.task_id, "task")?;
2279 let expected_actor = inserted.run_id()
2280 .map(|run_id| HarnessActorV1::ParentRun { run_id })
2281 .map_err(|_| HarnessEngineError::InvalidResultRefRecordAuthority)?;
2282 if operation.actor != expected_actor
2283 || operation.run_id.is_some()
2284 || operation.grant_id.is_some()
2285 || operation.reconciles_operation_id.is_some()
2286 || operation.updated_at_unix_ms != task.updated_at_unix_ms
2287 || operation.finished_at_unix_ms != Some(task.updated_at_unix_ms)
2288 {
2289 return Err(HarnessEngineError::InvalidResultRefRecordAuthority);
2290 }
2291 validate_replacement(
2292 "task",
2293 current_task.revision,
2294 expected_task_revision,
2295 task.revision,
2296 current_task.created_at_unix_ms,
2297 task.created_at_unix_ms,
2298 )?;
2299 let mut next = self.clone();
2300 next.tasks.insert(task.task_id.clone(), task);
2301 next.operations.insert(operation.operation_id.clone(), operation.clone());
2302 next.finish(operation, HarnessApplyOutcome::Applied)
2303 }
2304
2305 pub fn prepare_dispatch_outcome_commit(
2308 &self,
2309 expected_run_revision: HarnessRevision,
2310 run: HarnessRunV1,
2311 expected_operation_revision: HarnessRevision,
2312 operation: HarnessOperationV1,
2313 expected_task_revision: HarnessRevision,
2314 task: HarnessTaskV1,
2315 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
2316 run.validate()?;
2317 operation.validate()?;
2318 task.validate()?;
2319 let current_run = self.runs.get(&run.run_id)
2320 .ok_or_else(|| HarnessEngineError::NotFound(run.run_id.to_string()))?;
2321 let current_operation = self.operations.get(&operation.operation_id)
2322 .ok_or_else(|| HarnessEngineError::NotFound(operation.operation_id.to_string()))?;
2323 let current_task = self.tasks.get(&task.task_id)
2324 .ok_or_else(|| HarnessEngineError::NotFound(task.task_id.to_string()))?;
2325 if current_run == &run && current_operation == &operation && current_task == &task {
2326 let exact_replay = run.revision.get()
2327 == expected_run_revision.get().checked_add(1)
2328 .ok_or(HarnessEngineError::InvalidNextRevision { entity: "run" })?
2329 && operation.revision.get()
2330 == expected_operation_revision.get().checked_add(1)
2331 .ok_or(HarnessEngineError::InvalidNextRevision {
2332 entity: "operation",
2333 })?
2334 && task.revision.get()
2335 == expected_task_revision.get().checked_add(1)
2336 .ok_or(HarnessEngineError::InvalidNextRevision { entity: "task" })?;
2337 if exact_replay {
2338 return Ok(PreparedHarnessMutation {
2339 next: self.clone(),
2340 outcome: HarnessApplyOutcome::Replayed,
2341 operation,
2342 });
2343 }
2344 return Err(HarnessEngineError::InvalidDispatchOutcomeProjection);
2345 }
2346 validate_replacement(
2347 "run",
2348 current_run.revision,
2349 expected_run_revision,
2350 run.revision,
2351 current_run.created_at_unix_ms,
2352 run.created_at_unix_ms,
2353 )?;
2354 validate_operation_transition(
2355 current_operation,
2356 expected_operation_revision,
2357 &operation,
2358 )?;
2359 validate_replacement(
2360 "task",
2361 current_task.revision,
2362 expected_task_revision,
2363 task.revision,
2364 current_task.created_at_unix_ms,
2365 task.created_at_unix_ms,
2366 )?;
2367 validate_run_immutable(current_run, &run)?;
2368 validate_run_lifecycle_transition(current_run, &run)?;
2369 validate_coupled_run_operation(current_run, &run, &operation)?;
2370 if !matches!(
2371 (current_run.lifecycle, current_operation.state),
2372 (HarnessRunLifecycleV1::Requested, HarnessOperationStateV1::Prepared)
2373 | (HarnessRunLifecycleV1::Preparing, HarnessOperationStateV1::Prepared)
2374 | (HarnessRunLifecycleV1::Dispatching, HarnessOperationStateV1::Dispatching)
2375 )
2376 || current_operation.kind != HarnessOperationKindV1::CreateRun
2377 || current_operation.operation_id != current_run.operation_id
2378 || operation.operation_id != run.operation_id
2379 || operation.run_id.as_ref() != Some(&run.run_id)
2380 || operation.task_id.as_ref() != Some(&task.task_id)
2381 || run.task_id != task.task_id
2382 || task.run_ids.binary_search(&run.run_id).is_err()
2383 || current_run.binding != run.binding
2384 || current_run.delivery_receipt != run.delivery_receipt
2385 || current_run.continuation_receipt != run.continuation_receipt
2386 || task.updated_at_unix_ms != run.updated_at_unix_ms
2387 || operation.updated_at_unix_ms != run.updated_at_unix_ms
2388 {
2389 return Err(HarnessEngineError::InvalidDispatchOutcomeProjection);
2390 }
2391 validate_dispatch_outcome_task_projection(current_task, &task, &run, &operation)?;
2392 let mut next = self.clone();
2393 next.runs.insert(run.run_id.clone(), run);
2394 next.tasks.insert(task.task_id.clone(), task);
2395 next.operations.insert(operation.operation_id.clone(), operation.clone());
2396 next.finish(operation, HarnessApplyOutcome::Applied)
2397 }
2398
2399 pub fn accept(&mut self, prepared: PreparedHarnessMutation) {
2400 *self = prepared.next;
2401 }
2402
2403 fn finish(
2404 mut self,
2405 operation: HarnessOperationV1,
2406 outcome: HarnessApplyOutcome,
2407 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
2408 self.validate_links()?;
2409 if outcome == HarnessApplyOutcome::Applied {
2414 self.mutation_generation = self.mutation_generation.wrapping_add(1);
2415 }
2416 Ok(PreparedHarnessMutation { next: self, outcome, operation })
2417 }
2418
2419 fn validate_links(&self) -> Result<(), HarnessEngineError> {
2420 for task in self.tasks.values() {
2421 self.validate_task_links(task)?;
2422 }
2423 for run in self.runs.values() {
2424 self.validate_run_links(run)?;
2425 }
2426 for grant in self.grants.values() {
2427 if !self.runs.contains_key(&grant.actor_run_id) {
2428 return Err(HarnessEngineError::NotFound(grant.actor_run_id.to_string()));
2429 }
2430 }
2431 for operation in self.operations.values() {
2432 self.validate_actor(&operation.actor)?;
2433 self.validate_operation_links(operation)?;
2434 }
2435 for spec in self.execution_specs.values() {
2436 validate_execution_spec_identity(spec)?;
2437 if !self.tasks.contains_key(&spec.task_id) {
2438 return Err(HarnessEngineError::NotFound(spec.task_id.to_string()));
2439 }
2440 if self.execution_specs_v2.contains_key(&spec.task_id)
2441 || self.issuances.contains_key(&spec.task_id)
2442 {
2443 return Err(HarnessEngineError::DualExecutionSpec(spec.task_id.to_string()));
2444 }
2445 }
2446 let mut issuance_ids = BTreeSet::new();
2447 let mut v2_spec_ids = BTreeSet::new();
2448 for issuance in self.issuances.values() {
2449 issuance.validate()?;
2450 if !issuance_ids.insert(issuance.issuance_id.clone()) {
2451 return Err(HarnessEngineError::DuplicateCheckpointId(
2452 issuance.issuance_id.to_string(),
2453 ));
2454 }
2455 let task = self.tasks.get(&issuance.task_id)
2456 .ok_or_else(|| HarnessEngineError::NotFound(issuance.task_id.to_string()))?;
2457 if issuance.task_revision > task.revision {
2458 return Err(HarnessEngineError::InvalidIssuanceLink);
2459 }
2460 let spec = self.execution_specs_v2.get(&issuance.task_id)
2461 .ok_or(HarnessEngineError::OrphanIssuance)?;
2462 validate_issued_execution_spec_link(issuance, spec)?;
2463 }
2464 for spec in self.execution_specs_v2.values() {
2465 spec.validate()?;
2466 if !v2_spec_ids.insert(spec.execution_spec_id.clone()) {
2467 return Err(HarnessEngineError::DuplicateCheckpointId(
2468 spec.execution_spec_id.to_string(),
2469 ));
2470 }
2471 if self.execution_specs.contains_key(&spec.task_id) {
2472 return Err(HarnessEngineError::DualExecutionSpec(spec.task_id.to_string()));
2473 }
2474 let issuance = self.issuances.get(&spec.task_id)
2475 .ok_or(HarnessEngineError::OrphanIssuedExecutionSpec)?;
2476 validate_issued_execution_spec_link(issuance, spec)?;
2477 }
2478 for delivery in self.deliveries.values() {
2479 self.validate_delivery_links(delivery)?;
2480 self.validate_delivery_authority_reference(delivery)?;
2481 }
2482 for continuation in self.continuations.values() {
2483 self.validate_continuation_links(continuation)?;
2484 self.validate_continuation_authority_reference(continuation)?;
2485 }
2486 let mut delivery_runs = BTreeSet::new();
2487 for delivery in self.deliveries.values() {
2488 if !delivery_runs.insert(delivery.run_id.clone()) {
2489 return Err(HarnessEngineError::DuplicateDeliveryRun);
2490 }
2491 }
2492 let mut continuation_runs = BTreeSet::new();
2493 let mut continuation_receipts = BTreeSet::new();
2494 for continuation in self.continuations.values() {
2495 if !continuation_runs.insert(continuation.target_run_id.clone()) {
2496 return Err(HarnessEngineError::DuplicateContinuationRun);
2497 }
2498 if !continuation_receipts.insert(continuation.receipt_ref.clone()) {
2499 return Err(HarnessEngineError::DuplicateContinuationReceipt);
2500 }
2501 }
2502 for run in self.runs.values() {
2503 let task = self.tasks.get(&run.task_id)
2504 .ok_or_else(|| HarnessEngineError::NotFound(run.task_id.to_string()))?;
2505 if task.run_ids.binary_search(&run.run_id).is_err() {
2506 return Err(HarnessEngineError::MismatchedIdentity("run task backlink"));
2507 }
2508 }
2509 self.validate_task_cycles()?;
2510 self.validate_run_cycles()?;
2511 Ok(())
2512 }
2513
2514 fn delivery_operation(
2515 &self,
2516 delivery: &HarnessDeliveryV1,
2517 ) -> Result<&HarnessOperationV1, HarnessEngineError> {
2518 self.operations.get(&delivery.operation_id)
2519 .ok_or_else(|| HarnessEngineError::NotFound(delivery.operation_id.to_string()))
2520 }
2521
2522 fn continuation_operation(
2523 &self,
2524 continuation: &HarnessContinuationV1,
2525 ) -> Result<&HarnessOperationV1, HarnessEngineError> {
2526 self.operations.get(&continuation.operation_id)
2527 .ok_or_else(|| HarnessEngineError::NotFound(continuation.operation_id.to_string()))
2528 }
2529
2530 fn finish_continuation_replay(
2531 &self,
2532 continuation: &HarnessContinuationV1,
2533 ) -> Result<PreparedHarnessMutation, HarnessEngineError> {
2534 Ok(PreparedHarnessMutation {
2535 next: self.clone(),
2536 outcome: HarnessApplyOutcome::Replayed,
2537 operation: self.continuation_operation(continuation)?.clone(),
2538 })
2539 }
2540
2541 fn validate_continuation_authority(
2542 &self,
2543 continuation: &HarnessContinuationV1,
2544 ) -> Result<(), HarnessEngineError> {
2545 self.validate_continuation_links(continuation)?;
2546 let source = self.runs.get(&continuation.source_run_id)
2547 .ok_or_else(|| HarnessEngineError::NotFound(continuation.source_run_id.to_string()))?;
2548 let target = self.runs.get(&continuation.target_run_id)
2549 .ok_or_else(|| HarnessEngineError::NotFound(continuation.target_run_id.to_string()))?;
2550 let operation = self.continuation_operation(continuation)?;
2551 match &continuation.authority {
2552 HarnessTransferAuthorityRefV1::ParentGrant { grant_id, revision } => {
2553 let grant = self.grants.get(grant_id)
2554 .ok_or_else(|| HarnessEngineError::NotFound(grant_id.to_string()))?;
2555 if grant.revision != *revision
2556 || grant.state != SessionGrantStateV1::Active
2557 || !grant.context_permissions.export
2558 || !grant.context_permissions.restore
2559 || grant.actor_run_id != continuation.source_run_id
2560 || target.parent_run_id.as_ref() != Some(&grant.actor_run_id)
2561 || operation.actor != (HarnessActorV1::ParentRun {
2562 run_id: grant.actor_run_id.clone(),
2563 })
2564 || !grant.allows_target(
2565 &target.intent.node_id,
2566 &target.intent.workspace_id,
2567 &target.intent.provider_profile,
2568 target.intent.mode,
2569 )
2570 {
2571 return Err(HarnessEngineError::ContinuationGrantDenied);
2572 }
2573 }
2574 HarnessTransferAuthorityRefV1::OperatorIssuance { issuance } => {
2575 let current = self.operator_issuance_for_run(target, issuance)?;
2576 let Some(context_source) = ¤t.context_source else {
2577 return Err(HarnessEngineError::IssuedExecutionTransferMismatch);
2578 };
2579 let source_session_matches = matches!(
2589 &continuation.source_binding.session,
2590 hatchery_harness_protocol::HarnessSessionIdentityV1::Managed {
2591 record_id,
2592 active_session,
2593 } if record_id == &context_source.session_record_id
2594 && (context_source.availability
2595 == hatchery_harness_protocol::HarnessContextSourceAvailabilityV1::Durable
2596 || active_session.as_ref() == context_source.active_session.as_ref())
2597 );
2598 if context_source.source_run_id != continuation.source_run_id
2599 || context_source.node_id != continuation.node_id
2600 || context_source.node_incarnation != continuation.node_incarnation
2601 || context_source.workspace_id != continuation.workspace_id
2602 || &operation.actor != target_task_actor(self, target)?
2603 || target.parent_run_id.is_some()
2604 || !source_session_matches
2605 {
2606 return Err(HarnessEngineError::IssuedExecutionTransferMismatch);
2607 }
2608 }
2609 }
2610 let source_is_live_managed = matches!(
2611 &continuation.source_binding.session,
2612 hatchery_harness_protocol::HarnessSessionIdentityV1::Managed {
2613 active_session: Some(_),
2614 ..
2615 }
2616 );
2617 let source_is_durable_managed = source.context_pack.is_some()
2618 && matches!(
2619 &continuation.source_binding.session,
2620 hatchery_harness_protocol::HarnessSessionIdentityV1::Managed { .. }
2621 );
2622 let source_binding_matches = match source.binding.as_ref() {
2638 Some(binding) if source_is_durable_managed => {
2639 binding.node_id == continuation.source_binding.node_id
2640 && binding.workspace_id == continuation.source_binding.workspace_id
2641 && matches!(
2642 (&binding.session, &continuation.source_binding.session),
2643 (
2644 hatchery_harness_protocol::HarnessSessionIdentityV1::Managed {
2645 record_id: left,
2646 ..
2647 },
2648 hatchery_harness_protocol::HarnessSessionIdentityV1::Managed {
2649 record_id: right,
2650 ..
2651 },
2652 ) if left == right
2653 )
2654 }
2655 binding => binding == Some(&continuation.source_binding),
2656 };
2657 if !matches!(
2658 source.lifecycle,
2659 HarnessRunLifecycleV1::Running
2660 | HarnessRunLifecycleV1::Waiting
2661 | HarnessRunLifecycleV1::Completed
2662 )
2663 || !source_binding_matches
2664 || !(source_is_live_managed || source_is_durable_managed)
2665 {
2666 return Err(HarnessEngineError::InvalidContinuationLink);
2667 }
2668 if matches!(
2669 continuation.state,
2670 HarnessContinuationStateV1::Prepared
2671 | HarnessContinuationStateV1::Exporting
2672 | HarnessContinuationStateV1::Exported
2673 )
2674 && (target.lifecycle != HarnessRunLifecycleV1::Requested
2675 || operation.state != HarnessOperationStateV1::Prepared)
2676 {
2677 return Err(HarnessEngineError::ContinuationAuthorityWindowClosed);
2678 }
2679 Ok(())
2680 }
2681
2682 fn validate_continuation_authority_reference(
2683 &self,
2684 continuation: &HarnessContinuationV1,
2685 ) -> Result<(), HarnessEngineError> {
2686 let target = self.runs.get(&continuation.target_run_id)
2687 .ok_or_else(|| HarnessEngineError::NotFound(continuation.target_run_id.to_string()))?;
2688 match &continuation.authority {
2689 HarnessTransferAuthorityRefV1::ParentGrant { grant_id, revision } => {
2690 let grant = self.grants.get(grant_id)
2691 .ok_or_else(|| HarnessEngineError::NotFound(grant_id.to_string()))?;
2692 if revision.get() > grant.revision.get()
2693 || grant.actor_run_id != continuation.source_run_id
2694 {
2695 return Err(HarnessEngineError::InvalidContinuationLink);
2696 }
2697 }
2698 HarnessTransferAuthorityRefV1::OperatorIssuance { issuance } => {
2699 let current = self.operator_issuance_for_run(target, issuance)?;
2700 let Some(context_source) = ¤t.context_source else {
2701 return Err(HarnessEngineError::IssuedExecutionTransferMismatch);
2702 };
2703 if context_source.source_run_id != continuation.source_run_id
2704 || context_source.node_id != continuation.node_id
2705 || context_source.node_incarnation != continuation.node_incarnation
2706 || context_source.workspace_id != continuation.workspace_id
2707 {
2708 return Err(HarnessEngineError::IssuedExecutionTransferMismatch);
2709 }
2710 }
2711 }
2712 Ok(())
2713 }
2714
2715 fn validate_continuation_links(
2716 &self,
2717 continuation: &HarnessContinuationV1,
2718 ) -> Result<(), HarnessEngineError> {
2719 continuation.validate()?;
2720 let source = self.runs.get(&continuation.source_run_id)
2721 .ok_or_else(|| HarnessEngineError::NotFound(continuation.source_run_id.to_string()))?;
2722 let target = self.runs.get(&continuation.target_run_id)
2723 .ok_or_else(|| HarnessEngineError::NotFound(continuation.target_run_id.to_string()))?;
2724 let operation = self.continuation_operation(continuation)?;
2725 let selected_source = target.intent.continuation_source_run_id()?;
2726 if selected_source.as_ref() != Some(&continuation.source_run_id)
2727 || target.operation_id != continuation.operation_id
2728 || operation.kind != HarnessOperationKindV1::CreateRun
2729 || operation.run_id.as_ref() != Some(&continuation.target_run_id)
2730 || source.intent.node_id != continuation.node_id
2731 || source.intent.workspace_id != continuation.workspace_id
2732 || target.intent.node_id != continuation.node_id
2733 {
2734 return Err(HarnessEngineError::InvalidContinuationLink);
2735 }
2736 match continuation.state {
2737 HarnessContinuationStateV1::Exported | HarnessContinuationStateV1::Bound => {
2738 if target.continuation_receipt.as_ref() != Some(&continuation.receipt_ref)
2739 || continuation.state == HarnessContinuationStateV1::Bound
2740 && target.binding.as_ref() != continuation.target_binding.as_ref()
2741 {
2742 return Err(HarnessEngineError::InvalidContinuationLink);
2743 }
2744 }
2745 HarnessContinuationStateV1::Expired if continuation.context.is_some() => {
2746 if target.continuation_receipt.as_ref() != Some(&continuation.receipt_ref) {
2747 return Err(HarnessEngineError::InvalidContinuationLink);
2748 }
2749 }
2750 _ if target.continuation_receipt.is_some() => {
2751 return Err(HarnessEngineError::InvalidContinuationLink);
2752 }
2753 _ => {}
2754 }
2755 Ok(())
2756 }
2757
2758 fn validate_delivery_authority(
2759 &self,
2760 delivery: &HarnessDeliveryV1,
2761 ) -> Result<(), HarnessEngineError> {
2762 self.validate_delivery_links(delivery)?;
2763 let run = self.runs.get(&delivery.run_id)
2764 .ok_or_else(|| HarnessEngineError::NotFound(delivery.run_id.to_string()))?;
2765 let operation = self.delivery_operation(delivery)?;
2766 if run.lifecycle != hatchery_harness_protocol::HarnessRunLifecycleV1::Requested
2767 || operation.state != HarnessOperationStateV1::Prepared
2768 {
2769 return Err(HarnessEngineError::DeliveryAuthorityWindowClosed);
2770 }
2771 match &delivery.authority {
2772 HarnessTransferAuthorityRefV1::ParentGrant { grant_id, revision } => {
2773 let grant = self.grants.get(grant_id)
2774 .ok_or_else(|| HarnessEngineError::NotFound(grant_id.to_string()))?;
2775 if grant.revision != *revision
2776 || grant.state != SessionGrantStateV1::Active
2777 || !grant.allows_delivery_bundle(&delivery.bundle.selector)
2778 || !grant.allows_target(
2779 &run.intent.node_id,
2780 &run.intent.workspace_id,
2781 &run.intent.provider_profile,
2782 run.intent.mode,
2783 )
2784 {
2785 return Err(HarnessEngineError::DeliveryGrantDenied);
2786 }
2787 }
2788 HarnessTransferAuthorityRefV1::OperatorIssuance { issuance } => {
2789 let current = self.operator_issuance_for_run(run, issuance)?;
2790 if current.delivery.as_ref().map(|selection| &selection.bundle)
2791 != Some(&delivery.bundle)
2792 || &operation.actor != target_task_actor(self, run)?
2793 || run.parent_run_id.is_some()
2794 {
2795 return Err(HarnessEngineError::IssuedExecutionTransferMismatch);
2796 }
2797 }
2798 }
2799 Ok(())
2800 }
2801
2802 fn validate_delivery_authority_reference(
2803 &self,
2804 delivery: &HarnessDeliveryV1,
2805 ) -> Result<(), HarnessEngineError> {
2806 let run = self.runs.get(&delivery.run_id)
2807 .ok_or_else(|| HarnessEngineError::NotFound(delivery.run_id.to_string()))?;
2808 match &delivery.authority {
2809 HarnessTransferAuthorityRefV1::ParentGrant { grant_id, revision } => {
2810 let grant = self.grants.get(grant_id)
2811 .ok_or_else(|| HarnessEngineError::NotFound(grant_id.to_string()))?;
2812 if revision.get() > grant.revision.get() {
2813 return Err(HarnessEngineError::InvalidDeliveryLink);
2814 }
2815 }
2816 HarnessTransferAuthorityRefV1::OperatorIssuance { issuance } => {
2817 let current = self.operator_issuance_for_run(run, issuance)?;
2818 if current.delivery.as_ref().map(|selection| &selection.bundle)
2819 != Some(&delivery.bundle)
2820 {
2821 return Err(HarnessEngineError::IssuedExecutionTransferMismatch);
2822 }
2823 }
2824 }
2825 Ok(())
2826 }
2827
2828 fn validate_delivery_links(
2829 &self,
2830 delivery: &HarnessDeliveryV1,
2831 ) -> Result<(), HarnessEngineError> {
2832 delivery.validate()?;
2833 let task = self.tasks.get(&delivery.task_id)
2834 .ok_or_else(|| HarnessEngineError::NotFound(delivery.task_id.to_string()))?;
2835 let run = self.runs.get(&delivery.run_id)
2836 .ok_or_else(|| HarnessEngineError::NotFound(delivery.run_id.to_string()))?;
2837 let operation = self.delivery_operation(delivery)?;
2838 if run.task_id != delivery.task_id
2839 || run.operation_id != delivery.operation_id
2840 || task.run_ids.binary_search(&delivery.run_id).is_err()
2841 || operation.kind != HarnessOperationKindV1::CreateRun
2842 || operation.task_id.as_ref() != Some(&delivery.task_id)
2843 || operation.run_id.as_ref() != Some(&delivery.run_id)
2844 || run.intent.delivery_bundle.as_ref() != Some(&delivery.bundle.selector)
2845 {
2846 return Err(HarnessEngineError::InvalidDeliveryLink);
2847 }
2848 if let Some(stage) = &delivery.stage_receipt {
2849 if stage.node_id != run.intent.node_id
2850 || stage.workspace_id != run.intent.workspace_id
2851 {
2852 return Err(HarnessEngineError::InvalidDeliveryLink);
2853 }
2854 }
2855 if let Some(receipt) = &delivery.receipt {
2856 if run.delivery_receipt.as_ref() != Some(&receipt.receipt_ref)
2857 || run.binding.as_ref() != Some(&receipt.binding)
2858 {
2859 return Err(HarnessEngineError::InvalidDeliveryLink);
2860 }
2861 }
2862 Ok(())
2863 }
2864
2865 fn operator_issuance_for_run<'a>(
2866 &'a self,
2867 run: &HarnessRunV1,
2868 reference: &hatchery_harness_protocol::HarnessTaskLaunchIssuanceRefV1,
2869 ) -> Result<&'a HarnessTaskLaunchIssuanceV1, HarnessEngineError> {
2870 let issuance = self.issuances.get(&run.task_id)
2871 .ok_or(HarnessEngineError::IssuedExecutionAuthorityMissing)?;
2872 let spec = self.execution_specs_v2.get(&run.task_id)
2873 .ok_or(HarnessEngineError::IssuedExecutionAuthorityMissing)?;
2874 validate_issued_execution_spec_link(issuance, spec)?;
2875 if issuance.reference() != *reference || spec.launch_issuance != *reference {
2876 return Err(HarnessEngineError::IssuedExecutionTransferMismatch);
2877 }
2878 validate_operator_issuance_run_intent(issuance, run)?;
2879 Ok(issuance)
2880 }
2881
2882 fn same_issued_run_projection(
2883 &self,
2884 expected_execution_spec_revision: HarnessRevision,
2885 expected_issuance_revision: HarnessRevision,
2886 task: &HarnessTaskV1,
2887 run: &HarnessRunV1,
2888 delivery: Option<&HarnessDeliveryV1>,
2889 continuation: Option<&HarnessContinuationV1>,
2890 ) -> bool {
2891 self.tasks.get(&task.task_id) == Some(task)
2892 && self.runs.get(&run.run_id) == Some(run)
2893 && self.execution_specs_v2.get(&task.task_id)
2894 .is_some_and(|spec| spec.revision == expected_execution_spec_revision)
2895 && self.issuances.get(&task.task_id)
2896 .is_some_and(|issuance| issuance.revision == expected_issuance_revision)
2897 && match delivery {
2898 Some(delivery) => self.deliveries.get(&delivery.delivery_ref) == Some(delivery),
2899 None => self.deliveries.values().all(|record| record.run_id != run.run_id),
2900 }
2901 && match continuation {
2902 Some(continuation) => self.continuations
2903 .get(&continuation.continuation_ref) == Some(continuation),
2904 None => self.continuations.values()
2905 .all(|record| record.target_run_id != run.run_id),
2906 }
2907 }
2908
2909 fn visibility_graph_validation(&self) -> Result<(), HarnessEngineError> {
2935 let cached = *self.visibility_graph_cache.borrow();
2940 if let Some((generation, outcome)) = cached {
2941 if generation == self.mutation_generation {
2942 return outcome.map_err(HarnessEngineError::from);
2943 }
2944 }
2945 let outcome = self.recompute_visibility_graph_validation();
2946 *self.visibility_graph_cache.borrow_mut() = Some((self.mutation_generation, outcome));
2947 outcome.map_err(HarnessEngineError::from)
2948 }
2949
2950 fn recompute_visibility_graph_validation(
2951 &self,
2952 ) -> Result<(), HarnessVisibilityGraphFaultV1> {
2953 self.validate_visibility_run_graph()?;
2954 self.validate_visibility_operation_graph()?;
2955 Ok(())
2956 }
2957
2958 fn validate_visibility_run_graph(&self) -> Result<(), HarnessVisibilityGraphFaultV1> {
2959 for run_id in self.runs.keys() {
2960 let mut current = Some(run_id);
2961 let mut visited = BTreeSet::new();
2962 let mut depth = 0_u16;
2963 while let Some(current_id) = current {
2964 if !visited.insert(current_id.clone()) {
2965 return Err(HarnessVisibilityGraphFaultV1::InvalidGraph);
2966 }
2967 let run = self.runs.get(current_id)
2968 .ok_or(HarnessVisibilityGraphFaultV1::InvalidGraph)?;
2969 current = run.parent_run_id.as_ref();
2970 if current.is_some() {
2971 depth = depth.checked_add(1)
2972 .ok_or(HarnessVisibilityGraphFaultV1::InvalidGraph)?;
2973 if depth > HARNESS_CHILD_DEPTH_MAX {
2974 return Err(HarnessVisibilityGraphFaultV1::InvalidGraph);
2975 }
2976 }
2977 }
2978 }
2979 Ok(())
2980 }
2981
2982 fn validate_visibility_operation_graph(&self) -> Result<(), HarnessVisibilityGraphFaultV1> {
2983 let mut complete = BTreeSet::new();
2984 for operation_id in self.operations.keys() {
2985 if complete.contains(operation_id) {
2986 continue;
2987 }
2988 let mut path = Vec::new();
2989 let mut visiting = BTreeSet::new();
2990 let mut current = Some(operation_id);
2991 while let Some(current_id) = current {
2992 if complete.contains(current_id) {
2993 break;
2994 }
2995 if !visiting.insert(current_id.clone()) {
2996 return Err(HarnessVisibilityGraphFaultV1::InvalidGraph);
2997 }
2998 if path.len() >= HARNESS_VISIBILITY_SCAN_MAX {
2999 return Err(HarnessVisibilityGraphFaultV1::ResourceExhausted);
3000 }
3001 let operation = self.operations.get(current_id)
3002 .ok_or(HarnessVisibilityGraphFaultV1::InvalidGraph)?;
3003 if matches!(
3004 &operation.actor,
3005 HarnessActorV1::ParentRun { run_id } if !self.runs.contains_key(run_id)
3006 ) || operation.task_id.as_ref()
3007 .is_some_and(|task_id| !self.tasks.contains_key(task_id))
3008 || operation.run_id.as_ref()
3009 .is_some_and(|run_id| !self.runs.contains_key(run_id))
3010 || operation.grant_id.as_ref()
3011 .is_some_and(|grant_id| !self.grants.contains_key(grant_id))
3012 {
3013 return Err(HarnessVisibilityGraphFaultV1::InvalidGraph);
3014 }
3015 path.push(current_id.clone());
3016 current = operation.reconciles_operation_id.as_ref();
3017 if current.is_some_and(|target| !self.operations.contains_key(target)) {
3018 return Err(HarnessVisibilityGraphFaultV1::InvalidGraph);
3019 }
3020 }
3021 complete.extend(path);
3022 }
3023 Ok(())
3024 }
3025
3026 fn visible_operations(
3027 &self,
3028 actor_runs: &BTreeSet<HarnessRunId>,
3029 visible_tasks: &BTreeSet<HarnessTaskId>,
3030 visible_runs: &BTreeSet<HarnessRunId>,
3031 ) -> Result<BTreeSet<HarnessOperationId>, HarnessEngineError> {
3032 let mut visible = BTreeSet::new();
3033 let mut frontier = Vec::new();
3034 let mut waiting: BTreeMap<HarnessOperationId, Vec<HarnessOperationId>> =
3035 BTreeMap::new();
3036
3037 for operation in self.operations.values() {
3038 let actor_visible = matches!(
3039 &operation.actor,
3040 HarnessActorV1::ParentRun { run_id } if actor_runs.contains(run_id)
3041 );
3042 let targets_visible = operation.task_id.as_ref()
3043 .map_or(true, |task_id| visible_tasks.contains(task_id))
3044 && operation.run_id.as_ref()
3045 .map_or(true, |run_id| visible_runs.contains(run_id));
3046 if !actor_visible || !targets_visible || operation.grant_id.is_some() {
3047 continue;
3048 }
3049 if let Some(target_id) = &operation.reconciles_operation_id {
3050 waiting.entry(target_id.clone())
3051 .or_default()
3052 .push(operation.operation_id.clone());
3053 } else if visible.insert(operation.operation_id.clone()) {
3054 frontier.push(operation.operation_id.clone());
3055 }
3056 }
3057
3058 let mut cursor = 0_usize;
3059 while cursor < frontier.len() {
3060 if cursor >= HARNESS_VISIBILITY_SCAN_MAX {
3061 return Err(HarnessEngineError::ReadVisibilityResourceExhausted);
3062 }
3063 let visible_id = frontier[cursor].clone();
3064 cursor += 1;
3065 if let Some(dependents) = waiting.remove(&visible_id) {
3066 for dependent_id in dependents {
3067 if visible.insert(dependent_id.clone()) {
3068 frontier.push(dependent_id);
3069 }
3070 }
3071 }
3072 }
3073 Ok(visible)
3074 }
3075
3076 fn visible_runs_for_scope(
3077 &self,
3078 scope: HarnessEntityReadScopeV1,
3079 actor_run_id: &HarnessRunId,
3080 ) -> Result<BTreeSet<HarnessRunId>, HarnessEngineError> {
3081 match scope {
3082 HarnessEntityReadScopeV1::None => Ok(BTreeSet::new()),
3083 HarnessEntityReadScopeV1::SelfOnly => {
3084 Ok(BTreeSet::from([actor_run_id.clone()]))
3085 }
3086 HarnessEntityReadScopeV1::Descendants => {
3087 let mut visible = BTreeSet::from([actor_run_id.clone()]);
3088 let mut changed = true;
3089 let mut depth = 0_u16;
3090 while changed {
3091 changed = false;
3092 for run in self.runs.values() {
3093 if !visible.contains(&run.run_id)
3094 && run.parent_run_id.as_ref()
3095 .is_some_and(|parent| visible.contains(parent))
3096 {
3097 visible.insert(run.run_id.clone());
3098 changed = true;
3099 }
3100 }
3101 if changed {
3102 depth = depth.checked_add(1)
3103 .ok_or(HarnessEngineError::ReadVisibilityInvalidGraph)?;
3104 if depth > HARNESS_CHILD_DEPTH_MAX {
3105 return Err(HarnessEngineError::ReadVisibilityInvalidGraph);
3106 }
3107 }
3108 }
3109 Ok(visible)
3110 }
3111 }
3112 }
3113
3114 fn tasks_attributed_to_runs(
3115 &self,
3116 attribution_runs: &BTreeSet<HarnessRunId>,
3117 ) -> BTreeSet<HarnessTaskId> {
3118 if attribution_runs.is_empty() {
3119 return BTreeSet::new();
3120 }
3121 let mut visible_tasks: BTreeSet<_> = attribution_runs.iter().filter_map(|run_id| {
3122 self.runs.get(run_id).map(|run| run.task_id.clone())
3123 }).collect();
3124 visible_tasks.extend(self.tasks.values().filter_map(|task| {
3125 let tied_to_run = task.run_ids.iter()
3126 .any(|run_id| attribution_runs.contains(run_id));
3127 let created_by_run = matches!(
3128 &task.creator,
3129 HarnessActorV1::ParentRun { run_id } if attribution_runs.contains(run_id)
3130 );
3131 (tied_to_run || created_by_run).then(|| task.task_id.clone())
3132 }));
3133 visible_tasks
3134 }
3135
3136 pub fn task_subtree(
3151 &self,
3152 root: &HarnessTaskId,
3153 depth_bound: u16,
3154 ) -> BTreeSet<HarnessTaskId> {
3155 let mut visible = BTreeSet::from([root.clone()]);
3156 let mut changed = true;
3157 let mut depth = 0_u16;
3158 while changed {
3159 changed = false;
3160 for task in self.tasks.values() {
3161 if !visible.contains(&task.task_id)
3162 && task.parent_task_id.as_ref().is_some_and(|parent| visible.contains(parent))
3163 {
3164 visible.insert(task.task_id.clone());
3165 changed = true;
3166 }
3167 }
3168 if changed {
3169 depth = depth.saturating_add(1);
3170 if depth > depth_bound {
3171 break;
3172 }
3173 }
3174 }
3175 visible
3176 }
3177
3178 pub fn task_is_strict_descendant(&self, task: &HarnessTaskId, root: &HarnessTaskId) -> bool {
3182 task != root && self.task_subtree(root, HARNESS_CHILD_DEPTH_MAX).contains(task)
3183 }
3184
3185 pub fn task_child_count(&self, root: &HarnessTaskId) -> u64 {
3194 self.tasks.values()
3195 .filter(|task| task.parent_task_id.as_ref() == Some(root))
3196 .count() as u64
3197 }
3198
3199 pub fn task_subtree_depth(&self, root: &HarnessTaskId) -> u64 {
3213 let mut visible = BTreeSet::from([root.clone()]);
3214 let mut depth = 0_u64;
3215 let mut changed = true;
3216 while changed {
3217 changed = false;
3218 for task in self.tasks.values() {
3219 if !visible.contains(&task.task_id)
3220 && task.parent_task_id.as_ref().is_some_and(|parent| visible.contains(parent))
3221 {
3222 visible.insert(task.task_id.clone());
3223 changed = true;
3224 }
3225 }
3226 if changed {
3227 depth += 1;
3228 }
3229 }
3230 depth
3231 }
3232
3233 pub fn own_task_of_run(&self, run_id: &HarnessRunId) -> Option<HarnessTaskId> {
3238 self.run(run_id).map(|run| run.task_id.clone())
3239 }
3240
3241 pub fn operations_for_task(
3248 &self,
3249 task_id: &HarnessTaskId,
3250 limit: usize,
3251 ) -> Vec<&HarnessOperationV1> {
3252 let mut matching: Vec<&HarnessOperationV1> = self.operations.values()
3253 .filter(|operation| operation.task_id.as_ref() == Some(task_id))
3254 .collect();
3255 matching.sort_by(|left, right| {
3256 right.created_at_unix_ms.cmp(&left.created_at_unix_ms)
3257 .then_with(|| right.operation_id.cmp(&left.operation_id))
3258 });
3259 matching.truncate(limit);
3260 matching
3261 }
3262
3263 fn validate_task_links(&self, task: &HarnessTaskV1) -> Result<(), HarnessEngineError> {
3264 self.validate_actor(&task.creator)?;
3265 if let Some(parent) = &task.parent_task_id {
3266 if !self.tasks.contains_key(parent) {
3267 return Err(HarnessEngineError::NotFound(parent.to_string()));
3268 }
3269 }
3270 for dependency in &task.dependencies {
3271 if !self.tasks.contains_key(dependency) {
3272 return Err(HarnessEngineError::NotFound(dependency.to_string()));
3273 }
3274 }
3275 for run_id in &task.run_ids {
3276 let run = self.runs.get(run_id)
3277 .ok_or_else(|| HarnessEngineError::NotFound(run_id.to_string()))?;
3278 if run.task_id != task.task_id {
3279 return Err(HarnessEngineError::MismatchedIdentity("task run ownership"));
3280 }
3281 }
3282 Ok(())
3283 }
3284
3285 fn validate_run_links(&self, run: &HarnessRunV1) -> Result<(), HarnessEngineError> {
3286 if !self.tasks.contains_key(&run.task_id) {
3287 return Err(HarnessEngineError::NotFound(run.task_id.to_string()));
3288 }
3289 if let Some(parent) = &run.parent_run_id {
3290 if !self.runs.contains_key(parent) {
3291 return Err(HarnessEngineError::NotFound(parent.to_string()));
3292 }
3293 }
3294 let operation = self.operations.get(&run.operation_id)
3295 .ok_or_else(|| HarnessEngineError::NotFound(run.operation_id.to_string()))?;
3296 if operation.kind != HarnessOperationKindV1::CreateRun
3297 || operation.run_id.as_ref() != Some(&run.run_id)
3298 || operation.task_id.as_ref() != Some(&run.task_id)
3299 {
3300 return Err(HarnessEngineError::MismatchedIdentity(
3301 "run original CreateRun operation",
3302 ));
3303 }
3304 validate_run_operation_coherence(run, operation)?;
3305 Ok(())
3306 }
3307
3308 fn validate_task_cycles(&self) -> Result<(), HarnessEngineError> {
3309 let mut complete = BTreeSet::new();
3310 for task_id in self.tasks.keys() {
3311 visit_task(task_id, self, &mut BTreeSet::new(), &mut complete)?;
3312 }
3313 Ok(())
3314 }
3315
3316 fn validate_run_cycles(&self) -> Result<(), HarnessEngineError> {
3317 let mut complete = BTreeSet::new();
3318 for run_id in self.runs.keys() {
3319 visit_run(run_id, self, &mut BTreeSet::new(), &mut complete)?;
3320 }
3321 Ok(())
3322 }
3323
3324 fn validate_actor(&self, actor: &HarnessActorV1) -> Result<(), HarnessEngineError> {
3325 if let HarnessActorV1::ParentRun { run_id } = actor {
3326 if !self.runs.contains_key(run_id) {
3327 return Err(HarnessEngineError::NotFound(run_id.to_string()));
3328 }
3329 }
3330 Ok(())
3331 }
3332
3333 fn validate_operation_links(
3334 &self,
3335 operation: &HarnessOperationV1,
3336 ) -> Result<(), HarnessEngineError> {
3337 if let Some(task_id) = &operation.task_id {
3338 if !self.tasks.contains_key(task_id) {
3339 return Err(HarnessEngineError::NotFound(task_id.to_string()));
3340 }
3341 }
3342 if let Some(run_id) = &operation.run_id {
3343 if !self.runs.contains_key(run_id) {
3344 return Err(HarnessEngineError::NotFound(run_id.to_string()));
3345 }
3346 }
3347 if let Some(grant_id) = &operation.grant_id {
3348 if !self.grants.contains_key(grant_id) {
3349 return Err(HarnessEngineError::NotFound(grant_id.to_string()));
3350 }
3351 }
3352 if let Some(reconciles) = &operation.reconciles_operation_id {
3353 if !self.operations.contains_key(reconciles) {
3354 return Err(HarnessEngineError::NotFound(reconciles.to_string()));
3355 }
3356 }
3357 Ok(())
3358 }
3359}
3360
3361fn visit_task(
3362 task_id: &HarnessTaskId,
3363 engine: &HarnessEngine,
3364 visiting: &mut BTreeSet<HarnessTaskId>,
3365 complete: &mut BTreeSet<HarnessTaskId>,
3366) -> Result<(), HarnessEngineError> {
3367 if complete.contains(task_id) {
3368 return Ok(());
3369 }
3370 if !visiting.insert(task_id.clone()) {
3371 return Err(HarnessEngineError::Cycle("task graph"));
3372 }
3373 let task = engine.tasks.get(task_id)
3374 .ok_or_else(|| HarnessEngineError::NotFound(task_id.to_string()))?;
3375 if let Some(parent) = &task.parent_task_id {
3376 visit_task(parent, engine, visiting, complete)?;
3377 }
3378 for dependency in &task.dependencies {
3379 visit_task(dependency, engine, visiting, complete)?;
3380 }
3381 visiting.remove(task_id);
3382 complete.insert(task_id.clone());
3383 Ok(())
3384}
3385
3386fn visit_run(
3387 run_id: &HarnessRunId,
3388 engine: &HarnessEngine,
3389 visiting: &mut BTreeSet<HarnessRunId>,
3390 complete: &mut BTreeSet<HarnessRunId>,
3391) -> Result<(), HarnessEngineError> {
3392 if complete.contains(run_id) {
3393 return Ok(());
3394 }
3395 if !visiting.insert(run_id.clone()) {
3396 return Err(HarnessEngineError::Cycle("run parent graph"));
3397 }
3398 let run = engine.runs.get(run_id)
3399 .ok_or_else(|| HarnessEngineError::NotFound(run_id.to_string()))?;
3400 if let Some(parent) = &run.parent_run_id {
3401 visit_run(parent, engine, visiting, complete)?;
3402 }
3403 visiting.remove(run_id);
3404 complete.insert(run_id.clone());
3405 Ok(())
3406}
3407
3408fn same_operation_request(left: &HarnessOperationV1, right: &HarnessOperationV1) -> bool {
3409 left.operation_id == right.operation_id
3410 && left.actor == right.actor
3411 && left.kind == right.kind
3412 && left.task_id == right.task_id
3413 && left.run_id == right.run_id
3414 && left.grant_id == right.grant_id
3415 && left.reconciles_operation_id == right.reconciles_operation_id
3416 && left.expected_revision == right.expected_revision
3417 && left.request_digest == right.request_digest
3418 && left.idempotency_ref == right.idempotency_ref
3419 && left.created_at_unix_ms == right.created_at_unix_ms
3420}
3421
3422fn require_first_revision(
3423 revision: HarnessRevision,
3424 entity: &'static str,
3425) -> Result<(), HarnessEngineError> {
3426 if revision.get() != 1 {
3427 return Err(HarnessEngineError::InvalidNextRevision { entity });
3428 }
3429 Ok(())
3430}
3431
3432fn validate_execution_spec_identity(
3433 spec: &HarnessTaskExecutionSpecV1,
3434) -> Result<(), HarnessEngineError> {
3435 spec.validate()?;
3436 Ok(())
3437}
3438
3439fn validate_issued_execution_spec_link(
3440 issuance: &HarnessTaskLaunchIssuanceV1,
3441 spec: &HarnessTaskExecutionSpecV2,
3442) -> Result<(), HarnessEngineError> {
3443 issuance.validate()?;
3444 spec.validate()?;
3445 if issuance.task_id != spec.task_id
3446 || issuance.revision != spec.revision
3447 || issuance.reference() != spec.launch_issuance
3448 {
3449 return Err(HarnessEngineError::InvalidIssuanceLink);
3450 }
3451 Ok(())
3452}
3453
3454fn validate_issued_run_task_projection(
3455 current: &HarnessTaskV1,
3456 expected_revision: HarnessRevision,
3457 next: &HarnessTaskV1,
3458 run: &HarnessRunV1,
3459) -> Result<(), HarnessEngineError> {
3460 validate_replacement(
3461 "task",
3462 current.revision,
3463 expected_revision,
3464 next.revision,
3465 current.created_at_unix_ms,
3466 next.created_at_unix_ms,
3467 )?;
3468 let mut expected_run_ids = current.run_ids.clone();
3469 match expected_run_ids.binary_search(&run.run_id) {
3470 Ok(_) => return Err(HarnessEngineError::AlreadyExists(run.run_id.to_string())),
3471 Err(index) => expected_run_ids.insert(index, run.run_id.clone()),
3472 }
3473 if current.state != HarnessTaskStateV1::Ready
3474 || next.state != HarnessTaskStateV1::Running
3475 || current.task_id != next.task_id
3476 || next.task_id != run.task_id
3477 || current.title != next.title
3478 || current.body != next.body
3479 || current.creator != next.creator
3480 || current.parent_task_id != next.parent_task_id
3481 || current.dependencies != next.dependencies
3482 || expected_run_ids != next.run_ids
3483 || current.result_refs != next.result_refs
3484 || current.artifact_refs != next.artifact_refs
3485 {
3486 return Err(HarnessEngineError::InvalidIssuedRunProjection);
3487 }
3488 Ok(())
3489}
3490
3491fn validate_operator_issuance_run_intent(
3492 issuance: &HarnessTaskLaunchIssuanceV1,
3493 run: &HarnessRunV1,
3494) -> Result<(), HarnessEngineError> {
3495 let worktree_matches = match (&issuance.target.worktree, &run.intent.worktree) {
3496 (
3497 hatchery_harness_protocol::HarnessLaunchWorktreeSelectionV1::Existing,
3498 hatchery_harness_protocol::HarnessWorktreeIntentV1::Existing,
3499 ) => true,
3500 (
3501 hatchery_harness_protocol::HarnessLaunchWorktreeSelectionV1::Managed {
3502 profile_id,
3503 expected_profile_revision,
3504 },
3505 hatchery_harness_protocol::HarnessWorktreeIntentV1::ManagedProfile {
3506 profile_id: actual_profile_id,
3507 expected_profile_revision: actual_revision,
3508 },
3509 ) => profile_id == actual_profile_id && expected_profile_revision == actual_revision,
3510 _ => false,
3511 };
3512 let delivery_selector = issuance.delivery.as_ref()
3513 .map(|selection| &selection.bundle.selector);
3514 let continuation_selector = issuance.context_source.as_ref()
3515 .map(|source| source.source_run_id.as_str());
3516 if issuance.task_id != run.task_id
3517 || issuance.task_revision.get() == 0
3518 || issuance.target.node_id != run.intent.node_id
3519 || issuance.target.source_workspace_id != run.intent.workspace_id
3520 || issuance.target.provider_profile != run.intent.provider_profile
3521 || issuance.target.mode != run.intent.mode
3522 || !worktree_matches
3523 || run.intent.delivery_bundle.as_ref() != delivery_selector
3524 || run.intent.continuation.as_ref().map(|selector| selector.as_str())
3525 != continuation_selector
3526 {
3527 return Err(HarnessEngineError::IssuedExecutionTransferMismatch);
3528 }
3529 Ok(())
3530}
3531
3532fn target_task_actor<'a>(
3533 engine: &'a HarnessEngine,
3534 run: &HarnessRunV1,
3535) -> Result<&'a HarnessActorV1, HarnessEngineError> {
3536 engine.tasks.get(&run.task_id)
3537 .map(|task| &task.creator)
3538 .ok_or_else(|| HarnessEngineError::NotFound(run.task_id.to_string()))
3539}
3540
3541fn require_kind(
3542 actual: HarnessOperationKindV1,
3543 expected: HarnessOperationKindV1,
3544) -> Result<(), HarnessEngineError> {
3545 if actual != expected {
3546 return Err(HarnessEngineError::WrongOperationKind);
3547 }
3548 Ok(())
3549}
3550
3551fn require_operation_state(
3552 actual: HarnessOperationStateV1,
3553 expected: HarnessOperationStateV1,
3554) -> Result<(), HarnessEngineError> {
3555 if actual != expected {
3556 return Err(HarnessEngineError::WrongOperationState { expected, actual });
3557 }
3558 Ok(())
3559}
3560
3561fn require_operation_expected(
3562 operation: &HarnessOperationV1,
3563 expected_revision: HarnessRevision,
3564) -> Result<(), HarnessEngineError> {
3565 if operation.expected_revision != Some(expected_revision) {
3566 return Err(HarnessEngineError::ExpectedRevisionMismatch {
3567 entity: "operation request",
3568 expected: expected_revision.get(),
3569 actual: operation.expected_revision.map(HarnessRevision::get),
3570 });
3571 }
3572 Ok(())
3573}
3574
3575fn require_same_id<T: PartialEq>(
3576 operation_id: Option<&T>,
3577 entity_id: &T,
3578 entity: &'static str,
3579) -> Result<(), HarnessEngineError> {
3580 if operation_id != Some(entity_id) {
3581 return Err(HarnessEngineError::MismatchedIdentity(entity));
3582 }
3583 Ok(())
3584}
3585
3586fn validate_replacement(
3587 entity: &'static str,
3588 current_revision: HarnessRevision,
3589 expected_revision: HarnessRevision,
3590 next_revision: HarnessRevision,
3591 current_created_at: u64,
3592 next_created_at: u64,
3593) -> Result<(), HarnessEngineError> {
3594 if current_revision != expected_revision {
3595 return Err(HarnessEngineError::ExpectedRevisionMismatch {
3596 entity,
3597 expected: expected_revision.get(),
3598 actual: Some(current_revision.get()),
3599 });
3600 }
3601 if next_revision.get() != expected_revision.get().checked_add(1).unwrap_or(0) {
3602 return Err(HarnessEngineError::InvalidNextRevision { entity });
3603 }
3604 if current_created_at != next_created_at {
3605 return Err(HarnessEngineError::CreatedTimestampChanged { entity });
3606 }
3607 Ok(())
3608}
3609
3610fn validate_delivery_replacement(
3611 current: &HarnessDeliveryV1,
3612 expected_revision: HarnessRevision,
3613 next: &HarnessDeliveryV1,
3614) -> Result<(), HarnessEngineError> {
3615 validate_replacement(
3616 "delivery",
3617 current.revision,
3618 expected_revision,
3619 next.revision,
3620 current.created_at_unix_ms,
3621 next.created_at_unix_ms,
3622 )?;
3623 if current.delivery_ref != next.delivery_ref
3624 || current.authority != next.authority
3625 || current.task_id != next.task_id
3626 || current.run_id != next.run_id
3627 || current.operation_id != next.operation_id
3628 || current.bundle != next.bundle
3629 || current.stage_receipt.is_some() && current.stage_receipt != next.stage_receipt
3630 || current.receipt.is_some()
3631 {
3632 return Err(HarnessEngineError::InvalidDeliveryImmutableMutation);
3633 }
3634 Ok(())
3635}
3636
3637fn validate_continuation_replacement(
3638 current: &HarnessContinuationV1,
3639 expected_revision: HarnessRevision,
3640 next: &HarnessContinuationV1,
3641) -> Result<(), HarnessEngineError> {
3642 validate_replacement(
3643 "continuation",
3644 current.revision,
3645 expected_revision,
3646 next.revision,
3647 current.created_at_unix_ms,
3648 next.created_at_unix_ms,
3649 )?;
3650 if current.continuation_ref != next.continuation_ref
3651 || current.receipt_ref != next.receipt_ref
3652 || current.authority != next.authority
3653 || current.source_run_id != next.source_run_id
3654 || current.target_run_id != next.target_run_id
3655 || current.operation_id != next.operation_id
3656 || current.node_id != next.node_id
3657 || current.node_incarnation != next.node_incarnation
3658 || current.workspace_id != next.workspace_id
3659 || current.source_provider != next.source_provider
3660 || current.source_binding != next.source_binding
3661 || current.prepared_at_unix_ms != next.prepared_at_unix_ms
3662 || current.cleanup_state != next.cleanup_state
3663 || current.context.is_some() && current.context != next.context
3664 || current.target_binding.is_some() && current.target_binding != next.target_binding
3665 {
3666 return Err(HarnessEngineError::InvalidContinuationImmutableMutation);
3667 }
3668 Ok(())
3669}
3670
3671fn validate_delivery_restage(
3672 current: &HarnessDeliveryV1,
3673 expected_revision: HarnessRevision,
3674 next: &HarnessDeliveryV1,
3675) -> Result<(), HarnessEngineError> {
3676 validate_replacement(
3677 "delivery",
3678 current.revision,
3679 expected_revision,
3680 next.revision,
3681 current.created_at_unix_ms,
3682 next.created_at_unix_ms,
3683 )?;
3684 let (Some(current_stage), Some(next_stage)) =
3685 (¤t.stage_receipt, &next.stage_receipt)
3686 else {
3687 return Err(HarnessEngineError::InvalidDeliveryTransition);
3688 };
3689 if current.delivery_ref != next.delivery_ref
3690 || current.authority != next.authority
3691 || current.task_id != next.task_id
3692 || current.run_id != next.run_id
3693 || current.operation_id != next.operation_id
3694 || current.bundle != next.bundle
3695 || current.receipt.is_some()
3696 || next.receipt.is_some()
3697 || current_stage.node_id != next_stage.node_id
3698 || current_stage.workspace_id != next_stage.workspace_id
3699 || current_stage.bundle != next_stage.bundle
3700 || current_stage.node_incarnation == next_stage.node_incarnation
3701 || next_stage.staged_at_unix_ms <= current_stage.staged_at_unix_ms
3702 {
3703 return Err(HarnessEngineError::InvalidDeliveryRestage);
3704 }
3705 Ok(())
3706}
3707
3708fn validate_operation_transition(
3709 current: &HarnessOperationV1,
3710 expected_revision: HarnessRevision,
3711 next: &HarnessOperationV1,
3712) -> Result<(), HarnessEngineError> {
3713 validate_replacement(
3714 "operation",
3715 current.revision,
3716 expected_revision,
3717 next.revision,
3718 current.created_at_unix_ms,
3719 next.created_at_unix_ms,
3720 )?;
3721 if current.actor != next.actor
3722 || current.kind != next.kind
3723 || current.task_id != next.task_id
3724 || current.run_id != next.run_id
3725 || current.grant_id != next.grant_id
3726 || current.reconciles_operation_id != next.reconciles_operation_id
3727 || current.expected_revision != next.expected_revision
3728 || current.request_digest != next.request_digest
3729 || current.idempotency_ref != next.idempotency_ref
3730 {
3731 return Err(HarnessEngineError::MismatchedIdentity("immutable operation request"));
3732 }
3733 let allowed = matches!(
3734 (current.state, next.state),
3735 (HarnessOperationStateV1::Prepared, HarnessOperationStateV1::Dispatching)
3736 | (HarnessOperationStateV1::Prepared, HarnessOperationStateV1::Succeeded)
3737 | (HarnessOperationStateV1::Prepared, HarnessOperationStateV1::Failed)
3738 | (HarnessOperationStateV1::Prepared, HarnessOperationStateV1::OutcomeUnknown)
3739 | (HarnessOperationStateV1::Dispatching, HarnessOperationStateV1::OutcomeUnknown)
3740 | (HarnessOperationStateV1::Dispatching, HarnessOperationStateV1::Succeeded)
3741 | (HarnessOperationStateV1::Dispatching, HarnessOperationStateV1::Failed)
3742 );
3743 if !allowed {
3744 return Err(HarnessEngineError::InvalidOperationTransition {
3745 from: current.state,
3746 to: next.state,
3747 });
3748 }
3749 Ok(())
3750}
3751
3752fn validate_run_event_task_projection(
3753 current: &HarnessTaskV1,
3754 next: &HarnessTaskV1,
3755 run: &HarnessRunV1,
3756) -> Result<(), HarnessEngineError> {
3757 if current.task_id != next.task_id
3758 || current.title != next.title
3759 || current.body != next.body
3760 || current.creator != next.creator
3761 || current.parent_task_id != next.parent_task_id
3762 || current.dependencies != next.dependencies
3763 || current.run_ids != next.run_ids
3764 || current.result_refs != next.result_refs
3765 || current.artifact_refs != next.artifact_refs
3766 || !matches!(current.state, HarnessTaskStateV1::Running | HarnessTaskStateV1::Waiting)
3767 {
3768 return Err(HarnessEngineError::InvalidRunEventTaskProjection);
3769 }
3770 let expected_task_state = match run.lifecycle {
3771 HarnessRunLifecycleV1::Waiting => HarnessTaskStateV1::Waiting,
3772 HarnessRunLifecycleV1::Running => HarnessTaskStateV1::Running,
3773 HarnessRunLifecycleV1::Completed => HarnessTaskStateV1::Review,
3774 HarnessRunLifecycleV1::Failed => HarnessTaskStateV1::Failed,
3775 HarnessRunLifecycleV1::Cancelled => HarnessTaskStateV1::Cancelled,
3776 _ => return Err(HarnessEngineError::InvalidRunEventTaskProjection),
3777 };
3778 if next.state != expected_task_state {
3779 return Err(HarnessEngineError::InvalidRunEventTaskProjection);
3780 }
3781 Ok(())
3782}
3783
3784fn validate_incarnation_settlement_task_projection(
3796 current: &HarnessTaskV1,
3797 next: &HarnessTaskV1,
3798 run: &HarnessRunV1,
3799) -> Result<(), HarnessEngineError> {
3800 if current.task_id != next.task_id
3801 || current.title != next.title
3802 || current.body != next.body
3803 || current.creator != next.creator
3804 || current.parent_task_id != next.parent_task_id
3805 || current.dependencies != next.dependencies
3806 || current.run_ids != next.run_ids
3807 || current.result_refs != next.result_refs
3808 || current.artifact_refs != next.artifact_refs
3809 || !matches!(current.state, HarnessTaskStateV1::Running | HarnessTaskStateV1::Waiting)
3810 || run.lifecycle != HarnessRunLifecycleV1::Failed
3811 || next.state != HarnessTaskStateV1::Failed
3812 {
3813 return Err(HarnessEngineError::InvalidIncarnationSettlementProjection);
3814 }
3815 Ok(())
3816}
3817
3818fn validate_dispatch_outcome_task_projection(
3819 current: &HarnessTaskV1,
3820 next: &HarnessTaskV1,
3821 run: &HarnessRunV1,
3822 operation: &HarnessOperationV1,
3823) -> Result<(), HarnessEngineError> {
3824 if current.task_id != next.task_id
3825 || current.title != next.title
3826 || current.body != next.body
3827 || current.creator != next.creator
3828 || current.parent_task_id != next.parent_task_id
3829 || current.dependencies != next.dependencies
3830 || current.run_ids != next.run_ids
3831 || current.result_refs != next.result_refs
3832 || current.artifact_refs != next.artifact_refs
3833 || current.state != HarnessTaskStateV1::Running
3834 {
3835 return Err(HarnessEngineError::InvalidDispatchOutcomeProjection);
3836 }
3837 let coherent = match (run.lifecycle, operation.state, next.state) {
3838 (
3839 HarnessRunLifecycleV1::Failed,
3840 HarnessOperationStateV1::Failed,
3841 HarnessTaskStateV1::Failed,
3842 ) => run.result_disposition
3843 == Some(hatchery_harness_protocol::HarnessResultDispositionV1::Failed)
3844 && run.failure.is_some()
3845 && run.failure == operation.failure
3846 && operation.outcome_unknown_reason.is_none()
3847 && operation.finished_at_unix_ms == Some(run.updated_at_unix_ms),
3848 (
3849 HarnessRunLifecycleV1::OutcomeUnknown,
3850 HarnessOperationStateV1::OutcomeUnknown,
3851 HarnessTaskStateV1::Waiting,
3852 ) => run.result_disposition.is_none()
3853 && run.failure.is_none()
3854 && operation.failure.is_none()
3855 && operation.outcome_unknown_reason.is_some()
3856 && operation.finished_at_unix_ms.is_none(),
3857 _ => false,
3858 };
3859 if !coherent {
3860 return Err(HarnessEngineError::InvalidDispatchOutcomeProjection);
3861 }
3862 Ok(())
3863}
3864
3865fn validate_run_lifecycle_transition(
3866 current: &HarnessRunV1,
3867 next: &HarnessRunV1,
3868) -> Result<(), HarnessEngineError> {
3869 use hatchery_harness_protocol::HarnessRunLifecycleV1;
3870
3871 if matches!(
3872 current.lifecycle,
3873 HarnessRunLifecycleV1::Completed
3874 | HarnessRunLifecycleV1::Failed
3875 | HarnessRunLifecycleV1::Cancelled
3876 | HarnessRunLifecycleV1::OutcomeUnknown
3877 ) {
3878 if current.lifecycle == HarnessRunLifecycleV1::OutcomeUnknown {
3879 return Err(HarnessEngineError::OutcomeUnknownRunFrozen);
3880 }
3881 return Err(HarnessEngineError::TerminalRunMutation);
3882 }
3883 if current.lifecycle == next.lifecycle {
3884 return Ok(());
3885 }
3886 let allowed = matches!(
3887 (current.lifecycle, next.lifecycle),
3888 (HarnessRunLifecycleV1::Requested, HarnessRunLifecycleV1::Preparing)
3889 | (HarnessRunLifecycleV1::Requested, HarnessRunLifecycleV1::Dispatching)
3890 | (HarnessRunLifecycleV1::Requested, HarnessRunLifecycleV1::OutcomeUnknown)
3891 | (HarnessRunLifecycleV1::Requested, HarnessRunLifecycleV1::Failed)
3892 | (HarnessRunLifecycleV1::Requested, HarnessRunLifecycleV1::Cancelled)
3893 | (HarnessRunLifecycleV1::Preparing, HarnessRunLifecycleV1::Dispatching)
3894 | (HarnessRunLifecycleV1::Preparing, HarnessRunLifecycleV1::OutcomeUnknown)
3895 | (HarnessRunLifecycleV1::Preparing, HarnessRunLifecycleV1::Failed)
3896 | (HarnessRunLifecycleV1::Preparing, HarnessRunLifecycleV1::Cancelled)
3897 | (HarnessRunLifecycleV1::Dispatching, HarnessRunLifecycleV1::OutcomeUnknown)
3898 | (HarnessRunLifecycleV1::Dispatching, HarnessRunLifecycleV1::Running)
3899 | (HarnessRunLifecycleV1::Dispatching, HarnessRunLifecycleV1::Failed)
3900 | (HarnessRunLifecycleV1::Dispatching, HarnessRunLifecycleV1::Cancelled)
3901 | (HarnessRunLifecycleV1::Running, HarnessRunLifecycleV1::Waiting)
3902 | (HarnessRunLifecycleV1::Running, HarnessRunLifecycleV1::Completed)
3903 | (HarnessRunLifecycleV1::Running, HarnessRunLifecycleV1::Failed)
3904 | (HarnessRunLifecycleV1::Running, HarnessRunLifecycleV1::Cancelled)
3905 | (HarnessRunLifecycleV1::Waiting, HarnessRunLifecycleV1::Running)
3906 | (HarnessRunLifecycleV1::Waiting, HarnessRunLifecycleV1::Completed)
3907 | (HarnessRunLifecycleV1::Waiting, HarnessRunLifecycleV1::Failed)
3908 | (HarnessRunLifecycleV1::Waiting, HarnessRunLifecycleV1::Cancelled)
3909 );
3910 if !allowed {
3911 return Err(HarnessEngineError::InvalidRunLifecycleTransition {
3912 from: current.lifecycle,
3913 to: next.lifecycle,
3914 });
3915 }
3916 Ok(())
3917}
3918
3919fn validate_generic_run_operation(
3920 current: &HarnessRunV1,
3921 next: &HarnessRunV1,
3922 operation: &HarnessOperationV1,
3923) -> Result<(), HarnessEngineError> {
3924 use hatchery_harness_protocol::HarnessRunLifecycleV1;
3925
3926 let allowed = match operation.kind {
3927 HarnessOperationKindV1::MutateRun => {
3928 operation.state == HarnessOperationStateV1::Succeeded
3929 && matches!(
3930 current.lifecycle,
3931 HarnessRunLifecycleV1::Running | HarnessRunLifecycleV1::Waiting
3932 )
3933 && matches!(
3934 next.lifecycle,
3935 HarnessRunLifecycleV1::Running
3936 | HarnessRunLifecycleV1::Waiting
3937 | HarnessRunLifecycleV1::Completed
3938 | HarnessRunLifecycleV1::Failed
3939 | HarnessRunLifecycleV1::Cancelled
3940 )
3941 }
3942 _ => false,
3943 };
3944 if !allowed {
3945 return Err(HarnessEngineError::InvalidGenericRunOperation);
3946 }
3947 Ok(())
3948}
3949
3950fn validate_context_pack_recording(
3957 current: &HarnessRunV1,
3958 next: &HarnessRunV1,
3959) -> Result<(), HarnessEngineError> {
3960 if current.lifecycle == HarnessRunLifecycleV1::OutcomeUnknown {
3961 return Err(HarnessEngineError::OutcomeUnknownRunFrozen);
3962 }
3963 if current.run_id != next.run_id
3964 || current.parent_run_id != next.parent_run_id
3965 || current.task_id != next.task_id
3966 || current.operation_id != next.operation_id
3967 || current.intent != next.intent
3968 || current.delivery_receipt != next.delivery_receipt
3969 || current.continuation_receipt != next.continuation_receipt
3970 || current.git_facts != next.git_facts
3971 || current.binding != next.binding
3972 || current.lifecycle != next.lifecycle
3973 || current.result_disposition != next.result_disposition
3974 || current.failure != next.failure
3975 || current.created_at_unix_ms != next.created_at_unix_ms
3976 {
3977 return Err(HarnessEngineError::InvalidContextPackRecordProjection);
3978 }
3979 let Some(candidate) = &next.context_pack else {
3980 return Err(HarnessEngineError::InvalidContextPackRecordProjection);
3981 };
3982 match ¤t.context_pack {
3983 None => Ok(()),
3984 Some(existing) if existing == candidate => Ok(()),
3985 Some(_) => Err(HarnessEngineError::TerminalRunMutation),
3986 }
3987}
3988
3989fn validate_git_facts_recording(
3998 current: &HarnessRunV1,
3999 next: &HarnessRunV1,
4000) -> Result<(), HarnessEngineError> {
4001 if current.lifecycle == HarnessRunLifecycleV1::OutcomeUnknown {
4002 return Err(HarnessEngineError::OutcomeUnknownRunFrozen);
4003 }
4004 if current.run_id != next.run_id
4005 || current.parent_run_id != next.parent_run_id
4006 || current.task_id != next.task_id
4007 || current.operation_id != next.operation_id
4008 || current.intent != next.intent
4009 || current.delivery_receipt != next.delivery_receipt
4010 || current.continuation_receipt != next.continuation_receipt
4011 || current.context_pack != next.context_pack
4012 || current.binding != next.binding
4013 || current.lifecycle != next.lifecycle
4014 || current.result_disposition != next.result_disposition
4015 || current.failure != next.failure
4016 || current.created_at_unix_ms != next.created_at_unix_ms
4017 {
4018 return Err(HarnessEngineError::InvalidGitFactsRecordProjection);
4019 }
4020 let Some(candidate) = &next.git_facts else {
4021 return Err(HarnessEngineError::InvalidGitFactsRecordProjection);
4022 };
4023 match ¤t.git_facts {
4024 None => Ok(()),
4025 Some(existing) if existing == candidate => Ok(()),
4026 Some(_) => Err(HarnessEngineError::TerminalRunMutation),
4027 }
4028}
4029
4030fn validate_task_result_ref_recording(
4039 current: &HarnessTaskV1,
4040 next: &HarnessTaskV1,
4041) -> Result<(), HarnessEngineError> {
4042 if current.task_id != next.task_id
4043 || current.title != next.title
4044 || current.body != next.body
4045 || current.creator != next.creator
4046 || current.parent_task_id != next.parent_task_id
4047 || current.dependencies != next.dependencies
4048 || current.run_ids != next.run_ids
4049 || current.state != next.state
4050 || current.artifact_refs != next.artifact_refs
4051 || current.created_at_unix_ms != next.created_at_unix_ms
4052 {
4053 return Err(HarnessEngineError::InvalidResultRefRecordProjection);
4054 }
4055 if current.result_refs.len() >= HARNESS_RESULTS_MAX {
4056 return (next.result_refs == current.result_refs)
4057 .then_some(())
4058 .ok_or(HarnessEngineError::InvalidResultRefRecordProjection);
4059 }
4060 validate_single_sorted_insertion(¤t.result_refs, &next.result_refs)
4061 .map_err(|()| HarnessEngineError::InvalidResultRefRecordProjection)
4062}
4063
4064fn validate_single_sorted_insertion(
4071 current: &[HarnessResultRef],
4072 next: &[HarnessResultRef],
4073) -> Result<(), ()> {
4074 if current == next {
4075 return Ok(());
4076 }
4077 if next.len() != current.len() + 1 {
4078 return Err(());
4079 }
4080 let mut current_iter = current.iter().peekable();
4081 let mut inserted = false;
4082 for item in next {
4083 if current_iter.peek() == Some(&item) {
4084 current_iter.next();
4085 } else if inserted {
4086 return Err(());
4087 } else {
4088 inserted = true;
4089 }
4090 }
4091 if current_iter.next().is_some() {
4092 return Err(());
4093 }
4094 Ok(())
4095}
4096
4097fn validate_coupled_run_operation(
4098 current: &HarnessRunV1,
4099 next: &HarnessRunV1,
4100 operation: &HarnessOperationV1,
4101) -> Result<(), HarnessEngineError> {
4102 use hatchery_harness_protocol::HarnessRunLifecycleV1;
4103
4104 let allowed = operation.kind == HarnessOperationKindV1::CreateRun
4105 && matches!(
4106 (current.lifecycle, next.lifecycle, operation.state),
4107 (
4108 HarnessRunLifecycleV1::Requested,
4109 HarnessRunLifecycleV1::Dispatching,
4110 HarnessOperationStateV1::Dispatching,
4111 )
4112 | (
4113 HarnessRunLifecycleV1::Requested,
4114 HarnessRunLifecycleV1::Failed,
4115 HarnessOperationStateV1::Failed,
4116 )
4117 | (
4118 HarnessRunLifecycleV1::Requested,
4119 HarnessRunLifecycleV1::OutcomeUnknown,
4120 HarnessOperationStateV1::OutcomeUnknown,
4121 )
4122 | (
4123 HarnessRunLifecycleV1::Preparing,
4124 HarnessRunLifecycleV1::Failed,
4125 HarnessOperationStateV1::Failed,
4126 )
4127 | (
4128 HarnessRunLifecycleV1::Preparing,
4129 HarnessRunLifecycleV1::OutcomeUnknown,
4130 HarnessOperationStateV1::OutcomeUnknown,
4131 )
4132 | (
4133 HarnessRunLifecycleV1::Dispatching,
4134 HarnessRunLifecycleV1::OutcomeUnknown,
4135 HarnessOperationStateV1::OutcomeUnknown,
4136 )
4137 | (
4138 HarnessRunLifecycleV1::Dispatching,
4139 HarnessRunLifecycleV1::Running,
4140 HarnessOperationStateV1::Succeeded,
4141 )
4142 | (
4143 HarnessRunLifecycleV1::Dispatching,
4144 HarnessRunLifecycleV1::Failed,
4145 HarnessOperationStateV1::Failed,
4146 )
4147 );
4148 if !allowed {
4149 return Err(HarnessEngineError::InvalidCoupledRunOperation);
4150 }
4151 Ok(())
4152}
4153
4154fn validate_run_immutable(
4155 current: &HarnessRunV1,
4156 next: &HarnessRunV1,
4157) -> Result<(), HarnessEngineError> {
4158 if current.parent_run_id != next.parent_run_id
4159 || current.task_id != next.task_id
4160 || current.operation_id != next.operation_id
4161 || current.intent != next.intent
4162 {
4163 return Err(HarnessEngineError::MismatchedIdentity("immutable run request"));
4164 }
4165 Ok(())
4166}
4167
4168fn validate_run_operation_coherence(
4169 run: &HarnessRunV1,
4170 operation: &HarnessOperationV1,
4171) -> Result<(), HarnessEngineError> {
4172 use hatchery_harness_protocol::HarnessRunLifecycleV1;
4173
4174 let coherent = match run.lifecycle {
4175 HarnessRunLifecycleV1::Requested | HarnessRunLifecycleV1::Preparing => {
4176 operation.state == HarnessOperationStateV1::Prepared
4177 }
4178 HarnessRunLifecycleV1::Dispatching => {
4179 operation.state == HarnessOperationStateV1::Dispatching
4180 }
4181 HarnessRunLifecycleV1::OutcomeUnknown => {
4182 operation.state == HarnessOperationStateV1::OutcomeUnknown
4183 }
4184 HarnessRunLifecycleV1::Running
4185 | HarnessRunLifecycleV1::Waiting
4186 | HarnessRunLifecycleV1::Completed => {
4187 operation.state == HarnessOperationStateV1::Succeeded
4188 }
4189 HarnessRunLifecycleV1::Failed => matches!(
4190 operation.state,
4191 HarnessOperationStateV1::Failed | HarnessOperationStateV1::Succeeded
4192 ),
4193 HarnessRunLifecycleV1::Cancelled => matches!(
4194 operation.state,
4195 HarnessOperationStateV1::Failed | HarnessOperationStateV1::Succeeded
4196 ),
4197 };
4198 if !coherent {
4199 return Err(HarnessEngineError::RunOperationStateIncoherent {
4200 lifecycle: run.lifecycle,
4201 operation_state: operation.state,
4202 });
4203 }
4204 Ok(())
4205}
4206
4207fn validate_grant_transition(
4208 current: &SessionGrantV1,
4209 next: &SessionGrantV1,
4210 operation_kind: HarnessOperationKindV1,
4211) -> Result<(), HarnessEngineError> {
4212 use hatchery_harness_protocol::SessionGrantStateV1;
4213
4214 if current.state == SessionGrantStateV1::Revoked {
4215 return Err(HarnessEngineError::RevokedGrantMutation);
4216 }
4217 match operation_kind {
4218 HarnessOperationKindV1::RevokeGrant
4219 if next.state == SessionGrantStateV1::Revoked => Ok(()),
4220 HarnessOperationKindV1::MutateGrant if next.state == current.state => Ok(()),
4221 HarnessOperationKindV1::RevokeGrant => Err(HarnessEngineError::RevokeGrantMustRevoke),
4222 HarnessOperationKindV1::MutateGrant => Err(HarnessEngineError::GrantStateRequiresRevoke),
4223 _ => Err(HarnessEngineError::WrongOperationKind),
4224 }
4225}
4226
4227#[derive(Debug, Error)]
4228pub enum HarnessEngineError {
4229 #[error(transparent)]
4230 Validation(#[from] HarnessValidationError),
4231 #[error("unsupported harness checkpoint version {0}")]
4232 UnsupportedCheckpoint(u16),
4233 #[error("duplicate checkpoint identity {0}")]
4234 DuplicateCheckpointId(String),
4235 #[error("harness entity {0} already exists")]
4236 AlreadyExists(String),
4237 #[error("harness entity {0} was not found")]
4238 NotFound(String),
4239 #[error("operation id {operation_id} was reused with a different request digest")]
4240 OperationIdConflict { operation_id: HarnessOperationId },
4241 #[error("task {0} cannot carry both legacy and issuance-bound execution specifications")]
4242 DualExecutionSpec(String),
4243 #[error("task launch issuance has no matching V2 execution specification")]
4244 OrphanIssuance,
4245 #[error("V2 execution specification has no matching task launch issuance")]
4246 OrphanIssuedExecutionSpec,
4247 #[error("task launch issuance and V2 execution specification do not have exact task/id/revision/digest linkage")]
4248 InvalidIssuanceLink,
4249 #[error("issued execution CAS does not match current spec {spec_revision:?} and issuance {issuance_revision:?} revisions")]
4250 IssuedExecutionCasMismatch {
4251 spec_revision: Option<u64>,
4252 issuance_revision: Option<u64>,
4253 },
4254 #[error("wrong harness operation kind for mutation")]
4255 WrongOperationKind,
4256 #[error("wrong harness operation state: expected {expected:?}, actual {actual:?}")]
4257 WrongOperationState {
4258 expected: HarnessOperationStateV1,
4259 actual: HarnessOperationStateV1,
4260 },
4261 #[error("mismatched {0} identity")]
4262 MismatchedIdentity(&'static str),
4263 #[error("{entity} expected revision {expected}, actual {actual:?}")]
4264 ExpectedRevisionMismatch {
4265 entity: &'static str,
4266 expected: u64,
4267 actual: Option<u64>,
4268 },
4269 #[error("{entity} next revision is not the exact successor")]
4270 InvalidNextRevision { entity: &'static str },
4271 #[error("{entity} created timestamp changed")]
4272 CreatedTimestampChanged { entity: &'static str },
4273 #[error("cycle in {0}")]
4274 Cycle(&'static str),
4275 #[error("invalid operation state transition from {from:?} to {to:?}")]
4276 InvalidOperationTransition {
4277 from: HarnessOperationStateV1,
4278 to: HarnessOperationStateV1,
4279 },
4280 #[error("invalid run lifecycle transition from {from:?} to {to:?}")]
4281 InvalidRunLifecycleTransition {
4282 from: hatchery_harness_protocol::HarnessRunLifecycleV1,
4283 to: hatchery_harness_protocol::HarnessRunLifecycleV1,
4284 },
4285 #[error("terminal harness run metadata is immutable")]
4286 TerminalRunMutation,
4287 #[error("OutcomeUnknown run is frozen until H4 authoritative reconciliation")]
4288 OutcomeUnknownRunFrozen,
4289 #[error("generic run mutation does not have an exact succeeded BindRun/MutateRun authority")]
4290 InvalidGenericRunOperation,
4291 #[error("run lifecycle event operation does not have exact target and actor authority")]
4292 InvalidRunEventAuthority,
4293 #[error("run lifecycle event does not have the exact atomic task projection")]
4294 InvalidRunEventTaskProjection,
4295 #[error("host-incarnation settlement operation does not have exact single-run atomic authority")]
4296 InvalidIncarnationSettlementAuthority,
4297 #[error("host-incarnation settlement does not have the exact atomic run/task projection")]
4298 InvalidIncarnationSettlementProjection,
4299 #[error("ContextPack record operation does not have exact single-run, task-free authority")]
4300 InvalidContextPackRecordAuthority,
4301 #[error("ContextPack record must not change any run field besides context_pack, revision, or updated_at_unix_ms")]
4302 InvalidContextPackRecordProjection,
4303 #[error("git-facts record operation does not have exact single-run, task-free authority")]
4304 InvalidGitFactsRecordAuthority,
4305 #[error("git-facts record must not change any run field besides git_facts, revision, or updated_at_unix_ms")]
4306 InvalidGitFactsRecordProjection,
4307 #[error("result-ref record operation does not have exact single-task, run-free authority")]
4308 InvalidResultRefRecordAuthority,
4309 #[error("result-ref record must not change any task field besides result_refs, revision, or updated_at_unix_ms")]
4310 InvalidResultRefRecordProjection,
4311 #[error("CreateRun dispatch outcome does not have the exact atomic run/operation/task projection")]
4312 InvalidDispatchOutcomeProjection,
4313 #[error("coupled CreateRun operation state does not match run lifecycle")]
4314 InvalidCoupledRunOperation,
4315 #[error("run operation requires an atomic run and operation transition")]
4316 RunOperationRequiresAtomicTransition,
4317 #[error("RevokeGrant operation must produce a revoked grant")]
4318 RevokeGrantMustRevoke,
4319 #[error("grant state can change only through RevokeGrant")]
4320 GrantStateRequiresRevoke,
4321 #[error("revoked grant is terminal and cannot be mutated or reactivated")]
4322 RevokedGrantMutation,
4323 #[error("new session grant must start active")]
4324 GrantMustStartActive,
4325 #[error("new harness run must start Requested without a binding")]
4326 InvalidInitialRunState,
4327 #[error("issued run requires exact current issuance and execution spec authority")]
4328 IssuedExecutionAuthorityMissing,
4329 #[error("issued run task projection must atomically move Ready to Running and append one run")]
4330 InvalidIssuedRunProjection,
4331 #[error("issued run transfer authority or exact issuance selection does not match")]
4332 IssuedExecutionTransferMismatch,
4333 #[error("run lifecycle {lifecycle:?} is incoherent with original operation state {operation_state:?}")]
4334 RunOperationStateIncoherent {
4335 lifecycle: hatchery_harness_protocol::HarnessRunLifecycleV1,
4336 operation_state: HarnessOperationStateV1,
4337 },
4338 #[error("harness read visibility scan exceeded its fixed resource bound")]
4339 ReadVisibilityResourceExhausted,
4340 #[error("harness read visibility graph is invalid")]
4341 ReadVisibilityInvalidGraph,
4342 #[error("harness scheduler scan exceeded its fixed resource bound")]
4343 SchedulerResourceExhausted,
4344 #[error("harness scheduler durable task/run/operation graph is invalid")]
4345 SchedulerInvalidGraph,
4346 #[error("new delivery authority must start Prepared")]
4347 InvalidInitialDeliveryState,
4348 #[error("delivery reference {delivery_ref} was reused with a different authority")]
4349 DeliveryIdConflict { delivery_ref: HarnessDeliveryRef },
4350 #[error("delivery authority is not permitted by the exact active grant")]
4351 DeliveryGrantDenied,
4352 #[error("delivery authority has an invalid task/run/operation/grant link")]
4353 InvalidDeliveryLink,
4354 #[error("delivery state transition is invalid or terminal")]
4355 InvalidDeliveryTransition,
4356 #[error("delivery immutable authority or staged receipt changed")]
4357 InvalidDeliveryImmutableMutation,
4358 #[error("delivery commit must atomically attach its exact receipt to the bound run")]
4359 InvalidDeliveryRunCommit,
4360 #[error("new continuation authority must start Prepared")]
4361 InvalidInitialContinuationState,
4362 #[error("continuation reference {continuation_ref} was reused with different authority")]
4363 ContinuationIdConflict { continuation_ref: HarnessContinuationRef },
4364 #[error("continuation authority capacity exceeded")]
4365 ContinuationCapacityExceeded,
4366 #[error("continuation authority is not permitted by the exact active grant")]
4367 ContinuationGrantDenied,
4368 #[error("continuation authority has an invalid source/target/operation link")]
4369 InvalidContinuationLink,
4370 #[error("continuation authority may be prepared or exported only before dispatch")]
4371 ContinuationAuthorityWindowClosed,
4372 #[error("continuation state transition is invalid or terminal")]
4373 InvalidContinuationTransition,
4374 #[error("continuation immutable authority or exported receipt changed")]
4375 InvalidContinuationImmutableMutation,
4376 #[error("continuation bind must atomically attach exact receipt and target binding")]
4377 InvalidContinuationRunBind,
4378 #[error("delivery and continuation spawn must commit all authorities atomically")]
4379 InvalidCombinedSpawnCommit,
4380 #[error("a harness run cannot have multiple continuation authorities")]
4381 DuplicateContinuationRun,
4382 #[error("continuation receipt reference is not unique")]
4383 DuplicateContinuationReceipt,
4384 #[error("a harness run cannot have more than one durable delivery authority")]
4385 DuplicateDeliveryRun,
4386 #[error("delivery authority can be prepared or staged only before CreateRun dispatch")]
4387 DeliveryAuthorityWindowClosed,
4388 #[error("durable delivery authority exceeded its fixed record bound")]
4389 DeliveryCapacityExceeded,
4390 #[error("delivery restage must change only to a different current authoritative Node incarnation")]
4391 InvalidDeliveryRestage,
4392}
4393
4394#[cfg(test)]
4395mod tests {
4396 use super::*;
4397 use hatchery_harness_protocol::{
4398 HarnessActorV1, HarnessContextPermissionsV1, HarnessContextSourceSelectionV1,
4399 HarnessContextSourceAvailabilityV1,
4400 HarnessEntityReadScopeV1,
4401 HarnessContextPackLineageV1, HarnessContinuationCleanupStateV1,
4402 HarnessDeliveryBundleDigestV1, HarnessDeliveryBundleIdV1,
4403 HarnessDeliveryBundleRevisionV1, HarnessDeliveryBundleSelectionV1,
4404 HarnessDeliveryBundleV1, HarnessDeliveryComponentCountV1,
4405 HarnessDeliveryComponentKindV1,
4406 HarnessDeliveryManifestDigestV2,
4407 HarnessDeliveryRef, HarnessDeliveryStageReceiptV1,
4408 HarnessExecutionModeV1, HarnessExecutionSpecId, HarnessFailureCategoryV1,
4409 HarnessFailureV1, HarnessLaunchAuthorityRefV1, HarnessLaunchPlanRefV1,
4410 HarnessLaunchTargetSelectionV1, HarnessLaunchWorktreeSelectionV1,
4411 HarnessGrantTargetV1, HarnessIdempotencyRef,
4412 HarnessMonitoringVisibilityV1, HarnessOutcomeUnknownReasonV1,
4413 HarnessReadPermissionsV1,
4414 HarnessOperationKindV1, HarnessOperationStateV1, HarnessOperationTimeoutsV1,
4415 HarnessReceiptRef, HarnessRequestDigest, HarnessResolvedContextPackReceiptV1,
4416 HarnessResultDispositionV1,
4417 HarnessRunGitCommitSummaryV1, HarnessRunGitFactsOutcomeV1, HarnessRunGitFactsV1,
4418 HarnessRunGitStatusCodeV1, HarnessRunGitStatusEntryV1, HarnessRunGitSummaryV1,
4419 HarnessRunIntentV1, HarnessRunLifecycleV1, HarnessRuntimeIdentityV1,
4420 HarnessScheduledLaunchRefV2, HarnessTaskReviewPolicyV1,
4421 HarnessSelectorV1, HarnessSessionBindingV1, HarnessSessionIdentityV1,
4422 HarnessTaskLaunchIssuanceId, HarnessTaskPermissionsV1,
4423 HarnessTaskStateV1, HarnessTransferAuthorityRefV1, HarnessWorktreeIntentV1,
4424 SessionGrantStateV1,
4425 };
4426
4427 fn delivery_ref(hex: char) -> HarnessDeliveryRef {
4428 HarnessDeliveryRef::new(format!("hdelivery_{}", hex.to_string().repeat(24))).unwrap()
4429 }
4430
4431 fn delivery_bundle() -> HarnessDeliveryBundleV1 {
4432 HarnessDeliveryBundleV1 {
4433 selector: HarnessSelectorV1::new("review-kit").unwrap(),
4434 bundle_id: HarnessDeliveryBundleIdV1::new("bundle.review-kit").unwrap(),
4435 revision: HarnessDeliveryBundleRevisionV1::new("rev-7").unwrap(),
4436 digest: HarnessDeliveryBundleDigestV1::new(format!(
4437 "sha256:{}",
4438 "d".repeat(64),
4439 )).unwrap(),
4440 manifest_digest: HarnessDeliveryManifestDigestV2::new(format!(
4441 "sha256:{}",
4442 "e".repeat(64),
4443 )).unwrap(),
4444 }
4445 }
4446
4447 fn delivery(state: HarnessDeliveryStateV1, revision_value: u64) -> HarnessDeliveryV1 {
4448 let bundle = delivery_bundle();
4449 let stage_receipt = matches!(state, HarnessDeliveryStateV1::Staged | HarnessDeliveryStateV1::Committed)
4450 .then(|| HarnessDeliveryStageReceiptV1 {
4451 node_id: HarnessSelectorV1::new("node-a").unwrap(),
4452 node_incarnation: HarnessSelectorV1::new("incarnation-a").unwrap(),
4453 workspace_id: HarnessSelectorV1::new("workspace-a").unwrap(),
4454 bundle: bundle.clone(),
4455 staged_at_unix_ms: 20,
4456 });
4457 HarnessDeliveryV1 {
4458 delivery_ref: delivery_ref('d'),
4459 revision: revision(revision_value),
4460 authority: HarnessTransferAuthorityRefV1::ParentGrant {
4461 grant_id: grant_id(),
4462 revision: revision(1),
4463 },
4464 task_id: task_id('a'),
4465 run_id: run_id(),
4466 operation_id: operation_id('b'),
4467 bundle,
4468 state,
4469 stage_receipt,
4470 receipt: None,
4471 created_at_unix_ms: 15,
4472 updated_at_unix_ms: 15 + revision_value * 5,
4473 }
4474 }
4475
4476 fn task_id(hex: char) -> HarnessTaskId {
4477 HarnessTaskId::new(format!("htask_{}", hex.to_string().repeat(24))).unwrap()
4478 }
4479
4480 fn operation_id(hex: char) -> HarnessOperationId {
4481 HarnessOperationId::new(format!("hop_{}", hex.to_string().repeat(24))).unwrap()
4482 }
4483
4484 fn run_id() -> HarnessRunId {
4485 HarnessRunId::new(format!("hrun_{}", "a".repeat(24))).unwrap()
4486 }
4487
4488 fn numbered_run_id(value: usize) -> HarnessRunId {
4489 HarnessRunId::new(format!("hrun_{value:024x}")).unwrap()
4490 }
4491
4492 fn numbered_task_id(value: usize) -> HarnessTaskId {
4493 HarnessTaskId::new(format!("htask_{value:024x}")).unwrap()
4494 }
4495
4496 fn grant_id() -> SessionGrantId {
4497 SessionGrantId::new(format!("hgrant_{}", "a".repeat(24))).unwrap()
4498 }
4499
4500 fn revision(value: u64) -> HarnessRevision {
4501 HarnessRevision::new(value).unwrap()
4502 }
4503
4504 fn task(id: HarnessTaskId, revision_value: u64, title: &str) -> HarnessTaskV1 {
4505 HarnessTaskV1 {
4506 task_id: id,
4507 revision: revision(revision_value),
4508 title: title.to_owned(),
4509 body: String::new(),
4510 creator: HarnessActorV1::User {
4511 actor_id: HarnessSelectorV1::new("operator").unwrap(),
4512 },
4513 parent_task_id: None,
4514 dependencies: Vec::new(),
4515 state: HarnessTaskStateV1::Backlog,
4516 run_ids: Vec::new(),
4517 result_refs: Vec::new(),
4518 artifact_refs: Vec::new(),
4519 created_at_unix_ms: 10,
4520 updated_at_unix_ms: 10 + revision_value,
4521 }
4522 }
4523
4524 fn task_operation(
4525 id: HarnessOperationId,
4526 task_id: HarnessTaskId,
4527 digest_hex: char,
4528 expected_revision: Option<HarnessRevision>,
4529 ) -> HarnessOperationV1 {
4530 HarnessOperationV1 {
4531 operation_id: id,
4532 revision: revision(1),
4533 actor: HarnessActorV1::User {
4534 actor_id: HarnessSelectorV1::new("operator").unwrap(),
4535 },
4536 kind: if expected_revision.is_some() {
4537 HarnessOperationKindV1::MutateTask
4538 } else {
4539 HarnessOperationKindV1::CreateTask
4540 },
4541 state: HarnessOperationStateV1::Succeeded,
4542 task_id: Some(task_id),
4543 run_id: None,
4544 grant_id: None,
4545 reconciles_operation_id: None,
4546 expected_revision,
4547 request_digest: HarnessRequestDigest::new(digest_hex.to_string().repeat(64)).unwrap(),
4548 idempotency_ref: HarnessIdempotencyRef::new(format!(
4549 "hidem_{}",
4550 digest_hex.to_string().repeat(24)
4551 )).unwrap(),
4552 failure: None,
4553 outcome_unknown_reason: None,
4554 reconciliation_outcome: None,
4555 created_at_unix_ms: 10,
4556 updated_at_unix_ms: 12,
4557 dispatched_at_unix_ms: None,
4558 finished_at_unix_ms: Some(12),
4559 }
4560 }
4561
4562 fn create_task(engine: &mut HarnessEngine) -> (HarnessTaskId, HarnessOperationV1) {
4563 let id = task_id('a');
4564 let operation = task_operation(operation_id('a'), id.clone(), 'a', None);
4565 let prepared = engine.prepare(HarnessMutationV1::CreateTask {
4566 operation: operation.clone(),
4567 task: task(id.clone(), 1, "first"),
4568 }).unwrap();
4569 engine.accept(prepared);
4570 (id, operation)
4571 }
4572
4573 fn execution_spec(task_id: HarnessTaskId, revision_value: u64) -> HarnessTaskExecutionSpecV1 {
4574 HarnessTaskExecutionSpecV1 {
4575 execution_spec_id: HarnessExecutionSpecId::new(format!(
4576 "hespec_{}",
4577 "e".repeat(24),
4578 )).unwrap(),
4579 revision: revision(revision_value),
4580 task_id,
4581 scheduled_launch: HarnessScheduledLaunchRefV2 {
4582 plan: HarnessLaunchPlanRefV1 {
4583 plan_id: HarnessSelectorV1::new("ordinary").unwrap(),
4584 revision: revision(1),
4585 digest: HarnessRequestDigest::new("a".repeat(64)).unwrap(),
4586 },
4587 authority: HarnessLaunchAuthorityRefV1::OrdinaryOperator,
4588 },
4589 scheduled_launch_digest: HarnessRequestDigest::new("b".repeat(64)).unwrap(),
4590 review_policy: HarnessTaskReviewPolicyV1::OperatorReview,
4591 created_at_unix_ms: 20,
4592 updated_at_unix_ms: 20 + revision_value,
4593 }
4594 }
4595
4596 fn task_launch_issuance(
4597 task_id: HarnessTaskId,
4598 revision_value: u64,
4599 ) -> HarnessTaskLaunchIssuanceV1 {
4600 HarnessTaskLaunchIssuanceV1 {
4601 issuance_id: HarnessTaskLaunchIssuanceId::new(format!(
4602 "hissue_{}",
4603 "f".repeat(24),
4604 )).unwrap(),
4605 revision: revision(revision_value),
4606 digest: HarnessRequestDigest::new(
4607 if revision_value == 1 { "c" } else { "d" }.repeat(64),
4608 ).unwrap(),
4609 task_id,
4610 task_revision: revision(1),
4611 plan: HarnessLaunchPlanRefV1 {
4612 plan_id: HarnessSelectorV1::new("issued").unwrap(),
4613 revision: revision(2),
4614 digest: HarnessRequestDigest::new("a".repeat(64)).unwrap(),
4615 },
4616 target: HarnessLaunchTargetSelectionV1 {
4617 node_id: HarnessSelectorV1::new("node-a").unwrap(),
4618 source_workspace_id: HarnessSelectorV1::new("workspace-a").unwrap(),
4619 worktree: HarnessLaunchWorktreeSelectionV1::Managed {
4620 profile_id: HarnessSelectorV1::new("review-tree").unwrap(),
4621 expected_profile_revision: HarnessSelectorV1::new("revision-7").unwrap(),
4622 },
4623 provider_profile: HarnessSelectorV1::new("claude").unwrap(),
4624 mode: HarnessExecutionModeV1::Pty,
4625 },
4626 context_source: None,
4627 delivery: None,
4628 policy_digest: HarnessRequestDigest::new("b".repeat(64)).unwrap(),
4629 created_at_unix_ms: 20,
4630 updated_at_unix_ms: 20 + revision_value,
4631 }
4632 }
4633
4634 fn issued_execution_spec(
4635 issuance: &HarnessTaskLaunchIssuanceV1,
4636 revision_value: u64,
4637 ) -> HarnessTaskExecutionSpecV2 {
4638 HarnessTaskExecutionSpecV2 {
4639 execution_spec_id: HarnessExecutionSpecId::new(format!(
4640 "hespec_{}",
4641 "f".repeat(24),
4642 )).unwrap(),
4643 revision: revision(revision_value),
4644 task_id: issuance.task_id.clone(),
4645 launch_issuance: issuance.reference(),
4646 review_policy: HarnessTaskReviewPolicyV1::OperatorReview,
4647 created_at_unix_ms: 20,
4648 updated_at_unix_ms: 20 + revision_value,
4649 }
4650 }
4651
4652 fn run(lifecycle: HarnessRunLifecycleV1, revision_value: u64) -> HarnessRunV1 {
4653 let binding = matches!(
4654 lifecycle,
4655 HarnessRunLifecycleV1::Running
4656 | HarnessRunLifecycleV1::Waiting
4657 | HarnessRunLifecycleV1::Completed
4658 ).then(|| HarnessSessionBindingV1 {
4659 node_id: HarnessSelectorV1::new("node-a").unwrap(),
4660 node_incarnation: HarnessSelectorV1::new("incarnation-a").unwrap(),
4661 workspace_id: HarnessSelectorV1::new("workspace-a").unwrap(),
4662 session: HarnessSessionIdentityV1::Managed {
4663 record_id: HarnessSelectorV1::new("record-a").unwrap(),
4664 active_session: Some(HarnessRuntimeIdentityV1 {
4665 instance_id: 1,
4666 generation: 1,
4667 }),
4668 },
4669 });
4670 HarnessRunV1 {
4671 run_id: run_id(),
4672 revision: revision(revision_value),
4673 parent_run_id: None,
4674 task_id: task_id('a'),
4675 operation_id: operation_id('b'),
4676 intent: HarnessRunIntentV1 {
4677 node_id: HarnessSelectorV1::new("node-a").unwrap(),
4678 workspace_id: HarnessSelectorV1::new("workspace-a").unwrap(),
4679 worktree: HarnessWorktreeIntentV1::Existing,
4680 provider_profile: HarnessSelectorV1::new("claude").unwrap(),
4681 mode: HarnessExecutionModeV1::Pty,
4682 delivery_bundle: None,
4683 continuation: None,
4684 },
4685 delivery_receipt: None,
4686 continuation_receipt: None,
4687 context_pack: None,
4688 git_facts: None,
4689 binding,
4690 lifecycle,
4691 result_disposition: match lifecycle {
4692 HarnessRunLifecycleV1::Completed => Some(HarnessResultDispositionV1::Succeeded),
4693 HarnessRunLifecycleV1::Failed => Some(HarnessResultDispositionV1::Failed),
4694 HarnessRunLifecycleV1::Cancelled => Some(HarnessResultDispositionV1::Cancelled),
4695 _ => None,
4696 },
4697 failure: None,
4698 created_at_unix_ms: 10,
4699 updated_at_unix_ms: 10 + revision_value,
4700 }
4701 }
4702
4703 fn original_run_operation(state: HarnessOperationStateV1) -> HarnessOperationV1 {
4704 let dispatched = (state != HarnessOperationStateV1::Prepared).then_some(12);
4705 HarnessOperationV1 {
4706 operation_id: operation_id('b'),
4707 revision: revision(1),
4708 actor: HarnessActorV1::User {
4709 actor_id: HarnessSelectorV1::new("operator").unwrap(),
4710 },
4711 kind: HarnessOperationKindV1::CreateRun,
4712 state,
4713 task_id: Some(task_id('a')),
4714 run_id: Some(run_id()),
4715 grant_id: None,
4716 reconciles_operation_id: None,
4717 expected_revision: Some(revision(1)),
4718 request_digest: HarnessRequestDigest::new("b".repeat(64)).unwrap(),
4719 idempotency_ref: HarnessIdempotencyRef::new(format!(
4720 "hidem_{}",
4721 "b".repeat(24)
4722 )).unwrap(),
4723 failure: None,
4724 outcome_unknown_reason: None,
4725 reconciliation_outcome: None,
4726 created_at_unix_ms: 10,
4727 updated_at_unix_ms: dispatched.unwrap_or(10),
4728 dispatched_at_unix_ms: dispatched,
4729 finished_at_unix_ms: None,
4730 }
4731 }
4732
4733 fn grant(state: SessionGrantStateV1, revision_value: u64) -> SessionGrantV1 {
4734 SessionGrantV1 {
4735 grant_id: grant_id(),
4736 revision: revision(revision_value),
4737 actor_run_id: run_id(),
4738 allowed_targets: vec![HarnessGrantTargetV1 {
4739 node_id: HarnessSelectorV1::new("node-a").unwrap(),
4740 workspace_id: HarnessSelectorV1::new("workspace-a").unwrap(),
4741 provider_profile: HarnessSelectorV1::new("claude").unwrap(),
4742 mode: HarnessExecutionModeV1::Pty,
4743 }],
4744 allowed_delivery_bundles: Vec::new(),
4745 maximum_child_count: 1,
4746 maximum_child_depth: 1,
4747 operation_timeouts: HarnessOperationTimeoutsV1 {
4748 dispatch_ms: 1_000,
4749 wait_ms: 1_000,
4750 reconciliation_ms: 1_000,
4751 },
4752 task_permissions: HarnessTaskPermissionsV1 {
4753 read: true,
4754 create: false,
4755 mutate: false,
4756 request_run: false,
4757 },
4758 read_permissions: HarnessReadPermissionsV1::default(),
4759 monitoring_visibility: HarnessMonitoringVisibilityV1::Summary,
4760 context_permissions: HarnessContextPermissionsV1 {
4761 export: false,
4762 restore: false,
4763 },
4764 state,
4765 created_at_unix_ms: 10,
4766 updated_at_unix_ms: 10 + revision_value,
4767 }
4768 }
4769
4770 fn visibility_run(
4771 id: HarnessRunId,
4772 parent_run_id: Option<HarnessRunId>,
4773 owning_task_id: HarnessTaskId,
4774 ) -> HarnessRunV1 {
4775 let mut value = run(HarnessRunLifecycleV1::Requested, 1);
4776 value.run_id = id;
4777 value.parent_run_id = parent_run_id;
4778 value.task_id = owning_task_id;
4779 value
4780 }
4781
4782 fn visibility_reconcile_operation(
4783 id: HarnessOperationId,
4784 digest_hex: char,
4785 actor_run_id: HarnessRunId,
4786 run_id: HarnessRunId,
4787 target_id: HarnessOperationId,
4788 ) -> HarnessOperationV1 {
4789 let mut operation = task_operation(
4790 id,
4791 numbered_task_id(1),
4792 digest_hex,
4793 Some(revision(1)),
4794 );
4795 operation.actor = HarnessActorV1::ParentRun { run_id: actor_run_id };
4796 operation.kind = HarnessOperationKindV1::Reconcile;
4797 operation.task_id = None;
4798 operation.run_id = Some(run_id);
4799 operation.reconciles_operation_id = Some(target_id);
4800 operation
4801 }
4802
4803 fn visibility_fixture() -> HarnessEngine {
4804 let root_run_id = numbered_run_id(1);
4805 let child_run_id = numbered_run_id(2);
4806 let grandchild_run_id = numbered_run_id(3);
4807 let outside_run_id = numbered_run_id(4);
4808 let root_task_id = numbered_task_id(1);
4809 let child_task_id = numbered_task_id(2);
4810 let grandchild_task_id = numbered_task_id(3);
4811 let outside_task_id = numbered_task_id(4);
4812 let authored_task_id = numbered_task_id(5);
4813 let root_authored_task_id = numbered_task_id(6);
4814 let cross_linked_task_id = numbered_task_id(7);
4815
4816 let mut engine = HarnessEngine::new();
4817 for (id, run_ids, creator) in [
4818 (root_task_id.clone(), vec![root_run_id.clone()], None),
4819 (child_task_id.clone(), vec![child_run_id.clone()], None),
4820 (grandchild_task_id.clone(), vec![grandchild_run_id.clone()], None),
4821 (outside_task_id.clone(), vec![outside_run_id.clone()], None),
4822 (
4823 authored_task_id,
4824 Vec::new(),
4825 Some(HarnessActorV1::ParentRun { run_id: child_run_id.clone() }),
4826 ),
4827 (
4828 root_authored_task_id,
4829 Vec::new(),
4830 Some(HarnessActorV1::ParentRun { run_id: root_run_id.clone() }),
4831 ),
4832 ] {
4833 let mut record = task(id.clone(), 1, "visible fixture");
4834 record.run_ids = run_ids;
4835 if let Some(creator) = creator {
4836 record.creator = creator;
4837 }
4838 engine.tasks.insert(id, record);
4839 }
4840 let mut cross_linked = task(cross_linked_task_id.clone(), 1, "cross-tree link");
4841 cross_linked.parent_task_id = Some(root_task_id.clone());
4842 cross_linked.dependencies = vec![child_task_id.clone()];
4843 engine.tasks.insert(cross_linked_task_id, cross_linked);
4844 for record in [
4845 visibility_run(root_run_id.clone(), None, root_task_id.clone()),
4846 visibility_run(
4847 child_run_id.clone(),
4848 Some(root_run_id.clone()),
4849 child_task_id.clone(),
4850 ),
4851 visibility_run(
4852 grandchild_run_id,
4853 Some(child_run_id.clone()),
4854 grandchild_task_id,
4855 ),
4856 visibility_run(outside_run_id, None, outside_task_id.clone()),
4857 ] {
4858 engine.runs.insert(record.run_id.clone(), record);
4859 }
4860
4861 let mut root_operation = task_operation(
4862 operation_id('1'),
4863 root_task_id,
4864 '1',
4865 None,
4866 );
4867 root_operation.actor = HarnessActorV1::ParentRun {
4868 run_id: root_run_id.clone(),
4869 };
4870 let mut child_operation = task_operation(
4871 operation_id('2'),
4872 child_task_id.clone(),
4873 '2',
4874 None,
4875 );
4876 child_operation.actor = HarnessActorV1::ParentRun {
4877 run_id: child_run_id.clone(),
4878 };
4879 let mut parent_operation = task_operation(
4880 operation_id('3'),
4881 outside_task_id.clone(),
4882 '3',
4883 None,
4884 );
4885 parent_operation.actor = HarnessActorV1::ParentRun {
4886 run_id: child_run_id.clone(),
4887 };
4888 let outside_operation = task_operation(
4889 operation_id('4'),
4890 outside_task_id,
4891 '4',
4892 None,
4893 );
4894 let mut child_run_operation = task_operation(
4895 operation_id('5'),
4896 child_task_id,
4897 '5',
4898 Some(revision(1)),
4899 );
4900 child_run_operation.actor = HarnessActorV1::ParentRun {
4901 run_id: child_run_id.clone(),
4902 };
4903 child_run_operation.kind = HarnessOperationKindV1::BindRun;
4904 child_run_operation.task_id = None;
4905 child_run_operation.run_id = Some(child_run_id);
4906 let mut grant_operation = task_operation(
4907 operation_id('6'),
4908 numbered_task_id(1),
4909 '6',
4910 None,
4911 );
4912 grant_operation.actor = HarnessActorV1::ParentRun {
4913 run_id: root_run_id.clone(),
4914 };
4915 grant_operation.kind = HarnessOperationKindV1::CreateGrant;
4916 grant_operation.task_id = None;
4917 grant_operation.grant_id = Some(grant_id());
4918 let visible_reconcile = visibility_reconcile_operation(
4919 operation_id('7'),
4920 '7',
4921 root_run_id.clone(),
4922 root_run_id.clone(),
4923 operation_id('1'),
4924 );
4925 let visible_reconcile_chain = visibility_reconcile_operation(
4926 operation_id('8'),
4927 '8',
4928 root_run_id.clone(),
4929 root_run_id.clone(),
4930 operation_id('7'),
4931 );
4932 let hidden_reconcile = visibility_reconcile_operation(
4933 operation_id('9'),
4934 '9',
4935 root_run_id.clone(),
4936 root_run_id.clone(),
4937 operation_id('4'),
4938 );
4939 for operation in [
4940 root_operation,
4941 child_operation,
4942 parent_operation,
4943 outside_operation,
4944 child_run_operation,
4945 grant_operation,
4946 visible_reconcile,
4947 visible_reconcile_chain,
4948 hidden_reconcile,
4949 ] {
4950 engine.operations.insert(operation.operation_id.clone(), operation);
4951 }
4952
4953 let mut scoped_grant = grant(SessionGrantStateV1::Active, 1);
4954 scoped_grant.actor_run_id = root_run_id;
4955 scoped_grant.read_permissions = HarnessReadPermissionsV1 {
4956 tasks: HarnessEntityReadScopeV1::Descendants,
4957 runs: HarnessEntityReadScopeV1::Descendants,
4958 operations: HarnessEntityReadScopeV1::Descendants,
4959 };
4960 engine.grants.insert(scoped_grant.grant_id.clone(), scoped_grant);
4961 engine
4962 }
4963
4964 #[test]
4965 fn harness_task_cas_conflict_never_mutates() {
4966 let mut engine = HarnessEngine::new();
4967 let (id, _) = create_task(&mut engine);
4968 let before = engine.checkpoint();
4969 let operation = task_operation(
4970 operation_id('b'),
4971 id.clone(),
4972 'b',
4973 Some(revision(2)),
4974 );
4975 let error = engine.prepare(HarnessMutationV1::ReplaceTask {
4976 operation,
4977 expected_revision: revision(2),
4978 task: task(id, 3, "stale"),
4979 }).unwrap_err();
4980 assert!(matches!(error, HarnessEngineError::ExpectedRevisionMismatch { .. }));
4981 assert_eq!(engine.checkpoint(), before);
4982 }
4983
4984 #[test]
4985 fn harness_execution_spec_cas_replays_and_restores_from_checkpoint() {
4986 let mut engine = HarnessEngine::new();
4987 let (task_id, _) = create_task(&mut engine);
4988 let mut operation = task_operation(
4989 operation_id('b'),
4990 task_id.clone(),
4991 'b',
4992 Some(revision(1)),
4993 );
4994 operation.kind = HarnessOperationKindV1::MutateExecutionSpec;
4995 let first_spec = execution_spec(task_id.clone(), 1);
4996 let first = HarnessMutationV1::PutExecutionSpec {
4997 operation: operation.clone(),
4998 expected_task_revision: revision(1),
4999 expected_spec_revision: HarnessExpectedExecutionSpecRevisionV1::Absent,
5000 spec: first_spec.clone(),
5001 };
5002 let prepared = engine.prepare(first.clone()).unwrap();
5003 assert_eq!(prepared.outcome(), HarnessApplyOutcome::Applied);
5004 engine.accept(prepared);
5005 assert_eq!(
5006 engine.prepare(first).unwrap().outcome(),
5007 HarnessApplyOutcome::Replayed,
5008 );
5009
5010 let before = engine.checkpoint();
5011 let mut stale_operation = task_operation(
5012 operation_id('c'),
5013 task_id.clone(),
5014 'c',
5015 Some(revision(1)),
5016 );
5017 stale_operation.kind = HarnessOperationKindV1::MutateExecutionSpec;
5018 assert!(matches!(
5019 engine.prepare(HarnessMutationV1::PutExecutionSpec {
5020 operation: stale_operation,
5021 expected_task_revision: revision(1),
5022 expected_spec_revision: HarnessExpectedExecutionSpecRevisionV1::Absent,
5023 spec: first_spec.clone(),
5024 }),
5025 Err(HarnessEngineError::ExpectedRevisionMismatch {
5026 entity: "execution spec",
5027 ..
5028 }),
5029 ));
5030 assert_eq!(engine.checkpoint(), before);
5031
5032 let mut replacement = first_spec;
5033 replacement.revision = revision(2);
5034 replacement.updated_at_unix_ms = 22;
5035 let mut replace_operation = task_operation(
5036 operation_id('d'),
5037 task_id.clone(),
5038 'd',
5039 Some(revision(1)),
5040 );
5041 replace_operation.kind = HarnessOperationKindV1::MutateExecutionSpec;
5042 let prepared = engine.prepare(HarnessMutationV1::PutExecutionSpec {
5043 operation: replace_operation,
5044 expected_task_revision: revision(1),
5045 expected_spec_revision: HarnessExpectedExecutionSpecRevisionV1::Exact(revision(1)),
5046 spec: replacement.clone(),
5047 }).unwrap();
5048 engine.accept(prepared);
5049 let restored = HarnessEngine::restore(engine.checkpoint()).unwrap();
5050 assert_eq!(restored.execution_spec(&task_id), Some(&replacement));
5051 }
5052
5053 #[test]
5054 fn issued_execution_spec_cas_is_atomic_replayable_and_restore_exact() {
5055 let mut engine = HarnessEngine::new();
5056 let (task_id, _) = create_task(&mut engine);
5057 let mut operation = task_operation(
5058 operation_id('b'),
5059 task_id.clone(),
5060 'b',
5061 Some(revision(1)),
5062 );
5063 operation.kind = HarnessOperationKindV1::MutateExecutionSpec;
5064 let issuance = task_launch_issuance(task_id.clone(), 1);
5065 let spec = issued_execution_spec(&issuance, 1);
5066 let mutation = HarnessMutationV1::PutIssuedExecutionSpec {
5067 operation: operation.clone(),
5068 expected_task_revision: revision(1),
5069 expected_spec_revision: HarnessExpectedExecutionSpecRevisionV1::Absent,
5070 expected_issuance_revision: HarnessExpectedExecutionSpecRevisionV1::Absent,
5071 issuance: issuance.clone(),
5072 spec: spec.clone(),
5073 };
5074 let prepared = engine.prepare(mutation.clone()).unwrap();
5075 assert_eq!(prepared.outcome(), HarnessApplyOutcome::Applied);
5076 engine.accept(prepared);
5077 assert_eq!(
5078 engine.prepare(mutation).unwrap().outcome(),
5079 HarnessApplyOutcome::Replayed,
5080 );
5081 assert_eq!(engine.task_launch_issuance(&task_id), Some(&issuance));
5082 assert_eq!(engine.task_execution_spec_v2(&task_id), Some(&spec));
5083
5084 let restored = HarnessEngine::restore(engine.checkpoint()).unwrap();
5085 assert_eq!(restored.task_launch_issuance(&task_id), Some(&issuance));
5086 assert_eq!(restored.task_execution_spec_v2(&task_id), Some(&spec));
5087
5088 let before = engine.checkpoint();
5089 let mut mismatched_operation = task_operation(
5090 operation_id('c'),
5091 task_id.clone(),
5092 'c',
5093 Some(revision(1)),
5094 );
5095 mismatched_operation.kind = HarnessOperationKindV1::MutateExecutionSpec;
5096 let replacement_issuance = task_launch_issuance(task_id.clone(), 2);
5097 let replacement_spec = issued_execution_spec(&replacement_issuance, 2);
5098 assert!(matches!(
5099 engine.prepare(HarnessMutationV1::PutIssuedExecutionSpec {
5100 operation: mismatched_operation,
5101 expected_task_revision: revision(1),
5102 expected_spec_revision: HarnessExpectedExecutionSpecRevisionV1::Exact(revision(1)),
5103 expected_issuance_revision: HarnessExpectedExecutionSpecRevisionV1::Absent,
5104 issuance: replacement_issuance,
5105 spec: replacement_spec,
5106 }),
5107 Err(HarnessEngineError::IssuedExecutionCasMismatch { .. }),
5108 ));
5109 assert_eq!(engine.checkpoint(), before);
5110 }
5111
5112 #[test]
5113 fn issued_execution_restore_rejects_dual_orphan_and_ref_mismatch() {
5114 let mut engine = HarnessEngine::new();
5115 let (task_id, _) = create_task(&mut engine);
5116 let issuance = task_launch_issuance(task_id.clone(), 1);
5117 let spec = issued_execution_spec(&issuance, 1);
5118 let mut checkpoint = engine.checkpoint();
5119 checkpoint.issuances.push(issuance.clone());
5120 checkpoint.execution_specs_v2.push(spec.clone());
5121 HarnessEngine::restore(checkpoint.clone()).unwrap();
5122
5123 let mut dual = checkpoint.clone();
5124 dual.execution_specs.push(execution_spec(task_id.clone(), 1));
5125 assert!(matches!(
5126 HarnessEngine::restore(dual),
5127 Err(HarnessEngineError::DualExecutionSpec(_)),
5128 ));
5129
5130 let mut orphan_spec = checkpoint.clone();
5131 orphan_spec.issuances.clear();
5132 assert!(matches!(
5133 HarnessEngine::restore(orphan_spec),
5134 Err(HarnessEngineError::OrphanIssuedExecutionSpec),
5135 ));
5136
5137 let mut orphan_issuance = checkpoint.clone();
5138 orphan_issuance.execution_specs_v2.clear();
5139 assert!(matches!(
5140 HarnessEngine::restore(orphan_issuance),
5141 Err(HarnessEngineError::OrphanIssuance),
5142 ));
5143
5144 let mut mismatched = checkpoint;
5145 mismatched.execution_specs_v2[0].launch_issuance.digest =
5146 HarnessRequestDigest::new("e".repeat(64)).unwrap();
5147 assert!(matches!(
5148 HarnessEngine::restore(mismatched),
5149 Err(HarnessEngineError::InvalidIssuanceLink),
5150 ));
5151
5152 let mut divergent_revision = engine.checkpoint();
5153 divergent_revision.issuances.push(issuance);
5154 let mut divergent_spec = spec;
5155 divergent_spec.revision = revision(2);
5156 divergent_spec.updated_at_unix_ms = 22;
5157 divergent_revision.execution_specs_v2.push(divergent_spec);
5158 assert!(matches!(
5159 HarnessEngine::restore(divergent_revision),
5160 Err(HarnessEngineError::InvalidIssuanceLink),
5161 ));
5162 }
5163
5164 #[test]
5165 fn create_issued_run_is_atomic_exact_replayable_and_restart_safe() {
5166 let mut engine = HarnessEngine::new();
5167 let (task_id, _) = create_task(&mut engine);
5168 engine.tasks.get_mut(&task_id).unwrap().state = HarnessTaskStateV1::Ready;
5169
5170 let mut issuance = task_launch_issuance(task_id.clone(), 1);
5171 issuance.delivery = Some(HarnessDeliveryBundleSelectionV1 {
5172 bundle: delivery_bundle(),
5173 component_counts: vec![HarnessDeliveryComponentCountV1 {
5174 kind: HarnessDeliveryComponentKindV1::Skill,
5175 workspace_count: 1,
5176 session_count: 0,
5177 }],
5178 });
5179 let spec = issued_execution_spec(&issuance, 1);
5180 engine.issuances.insert(task_id.clone(), issuance.clone());
5181 engine.execution_specs_v2.insert(task_id.clone(), spec);
5182
5183 let mut operation = original_run_operation(HarnessOperationStateV1::Prepared);
5184 operation.expected_revision = Some(revision(1));
5185 let mut requested_run = run(HarnessRunLifecycleV1::Requested, 1);
5186 requested_run.intent.worktree = HarnessWorktreeIntentV1::ManagedProfile {
5187 profile_id: HarnessSelectorV1::new("review-tree").unwrap(),
5188 expected_profile_revision: HarnessSelectorV1::new("revision-7").unwrap(),
5189 };
5190 requested_run.intent.delivery_bundle = Some(
5191 issuance.delivery.as_ref().unwrap().bundle.selector.clone(),
5192 );
5193 let mut running_task = engine.task(&task_id).unwrap().clone();
5194 running_task.revision = revision(2);
5195 running_task.state = HarnessTaskStateV1::Running;
5196 running_task.run_ids = vec![requested_run.run_id.clone()];
5197 running_task.updated_at_unix_ms = 20;
5198 let mut prepared_delivery = delivery(HarnessDeliveryStateV1::Prepared, 1);
5199 prepared_delivery.authority = HarnessTransferAuthorityRefV1::OperatorIssuance {
5200 issuance: issuance.reference(),
5201 };
5202 let mutation = HarnessMutationV1::CreateIssuedRun {
5203 operation: operation.clone(),
5204 expected_task_revision: revision(1),
5205 expected_execution_spec_revision: revision(1),
5206 expected_issuance_revision: revision(1),
5207 task: running_task.clone(),
5208 run: requested_run.clone(),
5209 delivery: Some(prepared_delivery.clone()),
5210 continuation: None,
5211 };
5212
5213 let before = engine.checkpoint();
5214 let prepared = engine.prepare(mutation.clone()).unwrap();
5215 assert_eq!(engine.checkpoint(), before);
5216 assert_eq!(prepared.outcome(), HarnessApplyOutcome::Applied);
5217 engine.accept(prepared);
5218 assert_eq!(engine.task(&task_id), Some(&running_task));
5219 assert_eq!(engine.run(&requested_run.run_id), Some(&requested_run));
5220 assert_eq!(engine.delivery(&prepared_delivery.delivery_ref), Some(&prepared_delivery));
5221 assert_eq!(
5222 engine.prepare(mutation.clone()).unwrap().outcome(),
5223 HarnessApplyOutcome::Replayed,
5224 );
5225 let restored = HarnessEngine::restore(engine.checkpoint()).unwrap();
5226 assert_eq!(restored.task(&task_id), Some(&running_task));
5227 assert_eq!(restored.run(&requested_run.run_id), Some(&requested_run));
5228 assert_eq!(
5229 restored.delivery(&prepared_delivery.delivery_ref),
5230 Some(&prepared_delivery),
5231 );
5232
5233 let mut stale = mutation.clone();
5234 if let HarnessMutationV1::CreateIssuedRun {
5235 operation,
5236 expected_issuance_revision,
5237 run,
5238 delivery: Some(delivery),
5239 ..
5240 } = &mut stale
5241 {
5242 operation.operation_id = operation_id('c');
5243 run.operation_id = operation.operation_id.clone();
5244 delivery.operation_id = operation.operation_id.clone();
5245 operation.idempotency_ref = HarnessIdempotencyRef::new(format!(
5246 "hidem_{}",
5247 "c".repeat(24),
5248 )).unwrap();
5249 operation.request_digest = HarnessRequestDigest::new("c".repeat(64)).unwrap();
5250 *expected_issuance_revision = revision(2);
5251 }
5252 let stale_error = HarnessEngine::restore(before.clone()).unwrap()
5253 .prepare(stale).unwrap_err();
5254 assert!(
5255 matches!(stale_error, HarnessEngineError::IssuedExecutionCasMismatch { .. }),
5256 "unexpected stale error: {stale_error:?}",
5257 );
5258
5259 let mut wrong_authority = mutation;
5260 if let HarnessMutationV1::CreateIssuedRun {
5261 operation,
5262 run,
5263 delivery: Some(delivery),
5264 ..
5265 } = &mut wrong_authority
5266 {
5267 operation.operation_id = operation_id('d');
5268 run.operation_id = operation.operation_id.clone();
5269 delivery.operation_id = operation.operation_id.clone();
5270 operation.idempotency_ref = HarnessIdempotencyRef::new(format!(
5271 "hidem_{}",
5272 "d".repeat(24),
5273 )).unwrap();
5274 operation.request_digest = HarnessRequestDigest::new("d".repeat(64)).unwrap();
5275 delivery.authority = HarnessTransferAuthorityRefV1::ParentGrant {
5276 grant_id: grant_id(),
5277 revision: revision(1),
5278 };
5279 }
5280 assert!(matches!(
5281 HarnessEngine::restore(before).unwrap().prepare(wrong_authority),
5282 Err(HarnessEngineError::IssuedExecutionTransferMismatch),
5283 ));
5284 }
5285
5286 #[test]
5287 fn create_issued_run_accepts_concrete_continuation_provider_and_rejects_wrong_route() {
5288 let mut engine = HarnessEngine::new();
5289 let (task_id, _) = create_task(&mut engine);
5290 engine.tasks.get_mut(&task_id).unwrap().state = HarnessTaskStateV1::Ready;
5291
5292 let source_task_id = numbered_task_id(2);
5293 let source_run_id = numbered_run_id(2);
5294 let source_operation_id = operation_id('e');
5295 let mut source_task = task(source_task_id.clone(), 1, "source");
5296 source_task.state = HarnessTaskStateV1::Running;
5297 source_task.run_ids = vec![source_run_id.clone()];
5298 let mut source_operation = original_run_operation(HarnessOperationStateV1::Succeeded);
5299 source_operation.operation_id = source_operation_id.clone();
5300 source_operation.task_id = Some(source_task_id.clone());
5301 source_operation.run_id = Some(source_run_id.clone());
5302 source_operation.finished_at_unix_ms = Some(12);
5303 let mut source_run = run(HarnessRunLifecycleV1::Running, 1);
5304 source_run.task_id = source_task_id.clone();
5305 source_run.run_id = source_run_id.clone();
5306 source_run.operation_id = source_operation_id.clone();
5307 source_run.intent.provider_profile = HarnessSelectorV1::new("source-claude").unwrap();
5308 let source_binding = source_run.binding.clone().unwrap();
5309 engine.tasks.insert(source_task_id, source_task);
5310 engine.operations.insert(source_operation_id, source_operation);
5311 engine.runs.insert(source_run_id.clone(), source_run.clone());
5312
5313 let (record_id, active_session) = match &source_binding.session {
5314 HarnessSessionIdentityV1::Managed {
5315 record_id,
5316 active_session: Some(active_session),
5317 } => (record_id.clone(), active_session.clone()),
5318 _ => unreachable!(),
5319 };
5320 let mut issuance = task_launch_issuance(task_id.clone(), 1);
5321 issuance.context_source = Some(HarnessContextSourceSelectionV1 {
5322 source_run_id: source_run_id.clone(),
5323 source_run_revision: source_run.revision,
5324 observed_at_unix_ms: 20,
5325 metadata_digest: HarnessRequestDigest::new("e".repeat(64)).unwrap(),
5326 node_id: source_binding.node_id.clone(),
5327 node_incarnation: source_binding.node_incarnation.clone(),
5328 workspace_id: source_binding.workspace_id.clone(),
5329 session_record_id: record_id,
5330 active_session: Some(active_session),
5331 message_count: 3,
5332 message_count_exact: true,
5333 completed_turn_count: Some(1),
5334 total_tokens: Some(100),
5335 availability: HarnessContextSourceAvailabilityV1::Live,
5336 context_pack: None,
5337 });
5338 let spec = issued_execution_spec(&issuance, 1);
5339 engine.issuances.insert(task_id.clone(), issuance.clone());
5340 engine.execution_specs_v2.insert(task_id.clone(), spec);
5341
5342 let operation = original_run_operation(HarnessOperationStateV1::Prepared);
5343 let mut requested_run = run(HarnessRunLifecycleV1::Requested, 1);
5344 requested_run.intent.worktree = HarnessWorktreeIntentV1::ManagedProfile {
5345 profile_id: HarnessSelectorV1::new("review-tree").unwrap(),
5346 expected_profile_revision: HarnessSelectorV1::new("revision-7").unwrap(),
5347 };
5348 requested_run.intent.continuation = Some(
5349 HarnessSelectorV1::new(source_run_id.to_string()).unwrap(),
5350 );
5351 let mut running_task = engine.task(&task_id).unwrap().clone();
5352 running_task.revision = revision(2);
5353 running_task.state = HarnessTaskStateV1::Running;
5354 running_task.run_ids = vec![requested_run.run_id.clone()];
5355 running_task.updated_at_unix_ms = 20;
5356 let continuation = HarnessContinuationV1 {
5357 continuation_ref: HarnessContinuationRef::new(format!(
5358 "hcontinuation_{}",
5359 "f".repeat(24),
5360 )).unwrap(),
5361 receipt_ref: HarnessReceiptRef::new(format!(
5362 "hreceipt_{}",
5363 "f".repeat(24),
5364 )).unwrap(),
5365 revision: revision(1),
5366 state: HarnessContinuationStateV1::Prepared,
5367 authority: HarnessTransferAuthorityRefV1::OperatorIssuance {
5368 issuance: issuance.reference(),
5369 },
5370 source_run_id: source_run_id.clone(),
5371 target_run_id: requested_run.run_id.clone(),
5372 operation_id: operation.operation_id.clone(),
5373 node_id: source_binding.node_id.clone(),
5374 node_incarnation: source_binding.node_incarnation.clone(),
5375 workspace_id: source_binding.workspace_id.clone(),
5376 source_provider: HarnessSelectorV1::new("claude").unwrap(),
5377 source_binding,
5378 context: None,
5379 target_binding: None,
5380 prepared_at_unix_ms: 20,
5381 exporting_at_unix_ms: None,
5382 exported_at_unix_ms: None,
5383 bound_at_unix_ms: None,
5384 expired_at_unix_ms: None,
5385 outcome_unknown_at_unix_ms: None,
5386 outcome_unknown_reason: None,
5387 cleanup_state: HarnessContinuationCleanupStateV1::Retained,
5388 created_at_unix_ms: 20,
5389 updated_at_unix_ms: 20,
5390 };
5391 let mutation = HarnessMutationV1::CreateIssuedRun {
5392 operation,
5393 expected_task_revision: revision(1),
5394 expected_execution_spec_revision: revision(1),
5395 expected_issuance_revision: revision(1),
5396 task: running_task,
5397 run: requested_run.clone(),
5398 delivery: None,
5399 continuation: Some(continuation.clone()),
5400 };
5401 let mut wrong_route = mutation.clone();
5402 if let HarnessMutationV1::CreateIssuedRun {
5403 continuation: Some(continuation),
5404 ..
5405 } = &mut wrong_route
5406 {
5407 continuation.node_id = HarnessSelectorV1::new("node-b").unwrap();
5408 continuation.source_binding.node_id = continuation.node_id.clone();
5409 }
5410 assert!(matches!(
5411 engine.prepare(wrong_route),
5412 Err(HarnessEngineError::InvalidContinuationLink),
5413 ));
5414
5415 let before = engine.checkpoint();
5416 let prepared = engine.prepare(mutation).unwrap();
5417 assert_eq!(engine.checkpoint(), before);
5418 engine.accept(prepared);
5419 assert_eq!(
5420 engine.continuation(&continuation.continuation_ref),
5421 Some(&continuation),
5422 );
5423 assert_eq!(
5424 HarnessEngine::restore(engine.checkpoint()).unwrap()
5425 .continuation(&continuation.continuation_ref),
5426 Some(&continuation),
5427 );
5428 }
5429
5430 #[test]
5431 fn harness_scheduler_exact_id_selects_b_while_a_sorts_first() {
5432 let mut engine = HarnessEngine::new();
5433 let mut task_a = task(numbered_task_id(1), 1, "a");
5434 task_a.state = HarnessTaskStateV1::Ready;
5435 let mut task_b = task(numbered_task_id(2), 1, "b");
5436 task_b.state = HarnessTaskStateV1::Ready;
5437 engine.tasks.insert(task_a.task_id.clone(), task_a.clone());
5438 engine.tasks.insert(task_b.task_id.clone(), task_b.clone());
5439 assert_eq!(
5440 engine.scheduler_ready_task().unwrap().map(|task| &task.task_id),
5441 Some(&task_a.task_id),
5442 );
5443 assert_eq!(
5444 engine.scheduler_ready_task_by_id(&task_b.task_id)
5445 .unwrap()
5446 .map(|task| &task.task_id),
5447 Some(&task_b.task_id),
5448 );
5449 }
5450
5451 #[test]
5452 fn harness_scheduler_selects_lowest_ready_task_with_done_dependencies() {
5453 let mut engine = HarnessEngine::new();
5454 let mut dependency = task(numbered_task_id(1), 1, "dependency");
5455 dependency.state = HarnessTaskStateV1::Done;
5456 let mut blocked = task(numbered_task_id(2), 1, "blocked");
5457 blocked.state = HarnessTaskStateV1::Ready;
5458 blocked.dependencies = vec![numbered_task_id(4)];
5459 let mut eligible_low = task(numbered_task_id(3), 1, "eligible-low");
5460 eligible_low.state = HarnessTaskStateV1::Ready;
5461 eligible_low.dependencies = vec![dependency.task_id.clone()];
5462 let mut unfinished = task(numbered_task_id(4), 1, "unfinished");
5463 unfinished.state = HarnessTaskStateV1::Backlog;
5464 let mut eligible_high = task(numbered_task_id(5), 1, "eligible-high");
5465 eligible_high.state = HarnessTaskStateV1::Ready;
5466 for item in [dependency, blocked, eligible_low.clone(), unfinished, eligible_high] {
5467 engine.tasks.insert(item.task_id.clone(), item);
5468 }
5469
5470 assert_eq!(
5471 engine.scheduler_ready_task().unwrap().map(|task| task.task_id.clone()),
5472 Some(eligible_low.task_id),
5473 );
5474 }
5475
5476 #[test]
5477 fn harness_scheduler_requires_every_dependency_done() {
5478 let mut engine = HarnessEngine::new();
5479 let dependency_id = numbered_task_id(1);
5480 let mut dependency = task(dependency_id.clone(), 1, "dependency");
5481 dependency.state = HarnessTaskStateV1::Review;
5482 let mut candidate = task(numbered_task_id(2), 1, "candidate");
5483 candidate.state = HarnessTaskStateV1::Ready;
5484 candidate.dependencies = vec![dependency_id.clone()];
5485 engine.tasks.insert(dependency_id.clone(), dependency);
5486 engine.tasks.insert(candidate.task_id.clone(), candidate.clone());
5487 assert!(engine.scheduler_ready_task().unwrap().is_none());
5488
5489 engine.tasks.get_mut(&dependency_id).unwrap().state = HarnessTaskStateV1::Done;
5490 assert_eq!(
5491 engine.scheduler_ready_task().unwrap().map(|task| task.task_id.clone()),
5492 Some(candidate.task_id),
5493 );
5494 }
5495
5496 #[test]
5507 fn harness_scheduler_ready_task_scan_bound_is_categorical() {
5508 let mut engine = HarnessEngine::new();
5509 for index in 0..=HARNESS_SCHEDULER_SCAN_MAX {
5510 let item = task(numbered_task_id(index + 1), 1, "bounded");
5511 engine.tasks.insert(item.task_id.clone(), item);
5512 }
5513 assert!(matches!(
5517 engine.scheduler_ready_task(),
5518 Err(HarnessEngineError::SchedulerResourceExhausted),
5519 ));
5520 }
5521
5522 #[test]
5523 fn harness_scheduler_pending_dispatch_not_bound_by_task_count() {
5524 let mut engine = HarnessEngine::new();
5525 for index in 0..=HARNESS_SCHEDULER_SCAN_MAX {
5526 let item = task(numbered_task_id(index + 1), 1, "bounded");
5527 engine.tasks.insert(item.task_id.clone(), item);
5528 }
5529 assert_eq!(engine.scheduler_pending_dispatch().unwrap(), None);
5533 }
5534
5535 #[test]
5536 fn harness_scheduler_pending_dispatch_scan_bound_is_categorical() {
5537 let mut engine = HarnessEngine::new();
5538 for index in 0..=HARNESS_SCHEDULER_SCAN_MAX {
5539 let mut item = run(HarnessRunLifecycleV1::Failed, 1);
5540 item.run_id = numbered_run_id(index + 1);
5541 engine.runs.insert(item.run_id.clone(), item);
5542 }
5543 assert!(matches!(
5547 engine.scheduler_pending_dispatch(),
5548 Err(HarnessEngineError::SchedulerResourceExhausted),
5549 ));
5550 }
5551
5552 #[test]
5553 fn harness_scheduler_ready_task_by_id_survives_oversized_store() {
5554 let mut engine = HarnessEngine::new();
5555 let filler_tasks = HARNESS_SCHEDULER_SCAN_MAX / 2 + 100;
5556 let filler_runs = HARNESS_SCHEDULER_SCAN_MAX / 2 + 100;
5557 assert!(filler_tasks + filler_runs > HARNESS_SCHEDULER_SCAN_MAX);
5558 for index in 0..filler_tasks {
5559 let item = task(numbered_task_id(index + 1_000), 1, "filler");
5560 engine.tasks.insert(item.task_id.clone(), item);
5561 }
5562 for index in 0..filler_runs {
5563 let mut item = run(HarnessRunLifecycleV1::Failed, 1);
5564 item.run_id = numbered_run_id(index + 1_000);
5565 engine.runs.insert(item.run_id.clone(), item);
5566 }
5567
5568 let eligible_task_id = numbered_task_id(1);
5569 let eligible_run_id = numbered_run_id(1);
5570 let mut eligible_run = run(HarnessRunLifecycleV1::Completed, 1);
5571 eligible_run.run_id = eligible_run_id.clone();
5572 eligible_run.task_id = eligible_task_id.clone();
5573 engine.runs.insert(eligible_run_id.clone(), eligible_run);
5574
5575 let mut eligible_task = task(eligible_task_id.clone(), 1, "eligible");
5576 eligible_task.state = HarnessTaskStateV1::Ready;
5577 eligible_task.run_ids = vec![eligible_run_id];
5578 engine.tasks.insert(eligible_task_id.clone(), eligible_task);
5579
5580 assert_eq!(
5583 engine.scheduler_ready_task_by_id(&eligible_task_id)
5584 .unwrap()
5585 .map(|task| task.task_id.clone()),
5586 Some(eligible_task_id),
5587 );
5588 }
5589
5590 #[test]
5591 fn harness_scheduler_ready_task_by_id_still_refuses_ineligible_in_oversized_store() {
5592 let mut engine = HarnessEngine::new();
5593 let filler_tasks = HARNESS_SCHEDULER_SCAN_MAX / 2 + 100;
5594 let filler_runs = HARNESS_SCHEDULER_SCAN_MAX / 2 + 100;
5595 assert!(filler_tasks + filler_runs > HARNESS_SCHEDULER_SCAN_MAX);
5596 for index in 0..filler_tasks {
5597 let item = task(numbered_task_id(index + 1_000), 1, "filler");
5598 engine.tasks.insert(item.task_id.clone(), item);
5599 }
5600 for index in 0..filler_runs {
5601 let mut item = run(HarnessRunLifecycleV1::Failed, 1);
5602 item.run_id = numbered_run_id(index + 1_000);
5603 engine.runs.insert(item.run_id.clone(), item);
5604 }
5605
5606 let backlog_task_id = numbered_task_id(1);
5608 let backlog_task = task(backlog_task_id.clone(), 1, "still-backlog");
5609 engine.tasks.insert(backlog_task_id.clone(), backlog_task);
5610 assert_eq!(engine.scheduler_ready_task_by_id(&backlog_task_id).unwrap(), None);
5611
5612 let unmet_task_id = numbered_task_id(2);
5614 let mut unmet_task = task(unmet_task_id.clone(), 1, "unmet-dependency");
5615 unmet_task.state = HarnessTaskStateV1::Ready;
5616 unmet_task.dependencies = vec![numbered_task_id(3)];
5617 engine.tasks.insert(unmet_task_id.clone(), unmet_task);
5618 assert_eq!(engine.scheduler_ready_task_by_id(&unmet_task_id).unwrap(), None);
5619
5620 let running_task_id = numbered_task_id(4);
5622 let running_run_id = numbered_run_id(4);
5623 let mut running_run = run(HarnessRunLifecycleV1::Running, 1);
5624 running_run.run_id = running_run_id.clone();
5625 running_run.task_id = running_task_id.clone();
5626 engine.runs.insert(running_run_id.clone(), running_run);
5627 let mut running_task = task(running_task_id.clone(), 1, "nonterminal-run");
5628 running_task.state = HarnessTaskStateV1::Ready;
5629 running_task.run_ids = vec![running_run_id];
5630 engine.tasks.insert(running_task_id.clone(), running_task);
5631 assert_eq!(engine.scheduler_ready_task_by_id(&running_task_id).unwrap(), None);
5632 }
5633
5634 #[test]
5635 fn harness_task_has_nonterminal_run_not_bound_by_store_size() {
5636 let mut engine = HarnessEngine::new();
5637 let filler_tasks = HARNESS_SCHEDULER_SCAN_MAX / 2 + 100;
5638 let filler_runs = HARNESS_SCHEDULER_SCAN_MAX / 2 + 100;
5639 assert!(filler_tasks + filler_runs > HARNESS_SCHEDULER_SCAN_MAX);
5640 for index in 0..filler_tasks {
5641 let item = task(numbered_task_id(index + 1_000), 1, "filler");
5642 engine.tasks.insert(item.task_id.clone(), item);
5643 }
5644 for index in 0..filler_runs {
5645 let mut item = run(HarnessRunLifecycleV1::Failed, 1);
5646 item.run_id = numbered_run_id(index + 1_000);
5647 engine.runs.insert(item.run_id.clone(), item);
5648 }
5649
5650 let owner_task_id = numbered_task_id(1);
5651 let owned_run_id = numbered_run_id(1);
5652 let mut owned_run = run(HarnessRunLifecycleV1::Running, 1);
5653 owned_run.run_id = owned_run_id.clone();
5654 owned_run.task_id = owner_task_id.clone();
5655 engine.runs.insert(owned_run_id.clone(), owned_run);
5656
5657 let mut owner = task(owner_task_id.clone(), 1, "owner");
5658 owner.run_ids = vec![owned_run_id.clone()];
5659
5660 assert!(engine.task_has_nonterminal_run(&owner).unwrap());
5662
5663 engine.runs.get_mut(&owned_run_id).unwrap().lifecycle = HarnessRunLifecycleV1::Completed;
5664 assert!(!engine.task_has_nonterminal_run(&owner).unwrap());
5665 }
5666
5667 #[test]
5668 fn harness_request_idempotency_replays_or_conflicts() {
5669 let mut engine = HarnessEngine::new();
5670 let id = task_id('a');
5671 let operation = task_operation(operation_id('a'), id.clone(), 'a', None);
5672 let prepared = engine.prepare(HarnessMutationV1::CreateTask {
5673 operation: operation.clone(),
5674 task: task(id.clone(), 1, "first"),
5675 }).unwrap();
5676 engine.accept(prepared);
5677
5678 let replay = engine.prepare(HarnessMutationV1::CreateTask {
5679 operation: operation.clone(),
5680 task: task(id.clone(), 1, "first"),
5681 }).unwrap();
5682 assert_eq!(replay.outcome(), HarnessApplyOutcome::Replayed);
5683
5684 let mut wrong_digest = operation.clone();
5685 wrong_digest.request_digest = HarnessRequestDigest::new("b".repeat(64)).unwrap();
5686 assert!(matches!(
5687 engine.prepare(HarnessMutationV1::CreateTask {
5688 operation: wrong_digest,
5689 task: task(id.clone(), 1, "conflict"),
5690 }),
5691 Err(HarnessEngineError::OperationIdConflict { .. })
5692 ));
5693
5694 let mut wrong_actor = operation;
5695 wrong_actor.actor = HarnessActorV1::User {
5696 actor_id: HarnessSelectorV1::new("different-operator").unwrap(),
5697 };
5698 assert!(matches!(
5699 engine.prepare(HarnessMutationV1::CreateTask {
5700 operation: wrong_actor,
5701 task: task(id, 1, "conflict"),
5702 }),
5703 Err(HarnessEngineError::OperationIdConflict { .. })
5704 ));
5705 }
5706
5707 #[test]
5708 fn observation_never_creates_or_mutates_harness_task() {
5709 let mut engine = HarnessEngine::new();
5710 create_task(&mut engine);
5711 let before = engine.checkpoint();
5712 assert!(
5713 !include_str!("../Cargo.toml").contains("hatchery-observation"),
5714 "the mutation engine must not acquire a monitoring dependency",
5715 );
5716 let observation_only_projection = serde_json::json!({
5717 "kind": "turn-activity",
5718 "sequence": 42,
5719 });
5720 assert!(observation_only_projection.is_object());
5721 assert_eq!(engine.checkpoint(), before);
5722 }
5723
5724 #[test]
5725 fn restore_rejects_multi_node_task_cycle() {
5726 let first = task_id('a');
5727 let second = task_id('b');
5728 let mut first_task = task(first.clone(), 1, "first");
5729 first_task.dependencies = vec![second.clone()];
5730 let mut second_task = task(second, 1, "second");
5731 second_task.dependencies = vec![first];
5732 let checkpoint = HarnessEngineCheckpointV1 {
5733 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
5734 tasks: vec![first_task, second_task],
5735 runs: Vec::new(),
5736 grants: Vec::new(),
5737 operations: Vec::new(),
5738 execution_specs: Vec::new(),
5739 issuances: Vec::new(),
5740 execution_specs_v2: Vec::new(),
5741 deliveries: Vec::new(),
5742 continuations: Vec::new(),
5743 };
5744 assert!(matches!(HarnessEngine::restore(checkpoint), Err(HarnessEngineError::Cycle(_))));
5745 }
5746
5747 #[test]
5748 fn harness_run_lifecycle_is_exact_terminal_and_unknown_frozen() {
5749 let unknown = run(HarnessRunLifecycleV1::OutcomeUnknown, 1);
5750 let running = run(HarnessRunLifecycleV1::Running, 2);
5751 assert!(matches!(
5752 validate_run_lifecycle_transition(&unknown, &running),
5753 Err(HarnessEngineError::OutcomeUnknownRunFrozen)
5754 ));
5755
5756 let completed = run(HarnessRunLifecycleV1::Completed, 3);
5757 let mut rewritten = completed.clone();
5758 rewritten.revision = revision(4);
5759 rewritten.updated_at_unix_ms = 20;
5760 assert!(matches!(
5761 validate_run_lifecycle_transition(&completed, &rewritten),
5762 Err(HarnessEngineError::TerminalRunMutation)
5763 ));
5764
5765 let requested = run(HarnessRunLifecycleV1::Requested, 1);
5766 let mut changed_intent = run(HarnessRunLifecycleV1::Requested, 2);
5767 changed_intent.intent.workspace_id = HarnessSelectorV1::new("workspace-b").unwrap();
5768 assert!(matches!(
5769 validate_run_immutable(&requested, &changed_intent),
5770 Err(HarnessEngineError::MismatchedIdentity("immutable run request"))
5771 ));
5772 }
5773
5774 #[test]
5775 fn run_event_commit_is_atomic_exact_replay_and_fail_closed() {
5776 for (current_lifecycle, current_task_state, next_lifecycle, task_state, marker) in [
5777 (
5778 HarnessRunLifecycleV1::Running,
5779 HarnessTaskStateV1::Running,
5780 HarnessRunLifecycleV1::Waiting,
5781 HarnessTaskStateV1::Waiting,
5782 'c',
5783 ),
5784 (
5785 HarnessRunLifecycleV1::Waiting,
5786 HarnessTaskStateV1::Waiting,
5787 HarnessRunLifecycleV1::Running,
5788 HarnessTaskStateV1::Running,
5789 'd',
5790 ),
5791 (
5792 HarnessRunLifecycleV1::Running,
5793 HarnessTaskStateV1::Running,
5794 HarnessRunLifecycleV1::Completed,
5795 HarnessTaskStateV1::Review,
5796 'e',
5797 ),
5798 (
5799 HarnessRunLifecycleV1::Running,
5800 HarnessTaskStateV1::Running,
5801 HarnessRunLifecycleV1::Failed,
5802 HarnessTaskStateV1::Failed,
5803 'f',
5804 ),
5805 (
5806 HarnessRunLifecycleV1::Running,
5807 HarnessTaskStateV1::Running,
5808 HarnessRunLifecycleV1::Cancelled,
5809 HarnessTaskStateV1::Cancelled,
5810 '9',
5811 ),
5812 ] {
5813 let mut current_task = task(task_id('a'), 2, "event task");
5814 current_task.state = current_task_state;
5815 current_task.run_ids = vec![run_id()];
5816 let current_run = run(current_lifecycle, 3);
5817 let mut create_run = original_run_operation(HarnessOperationStateV1::Succeeded);
5818 create_run.revision = revision(3);
5819 create_run.updated_at_unix_ms = current_run.updated_at_unix_ms;
5820 create_run.finished_at_unix_ms = Some(current_run.updated_at_unix_ms);
5821 let engine = HarnessEngine::restore(HarnessEngineCheckpointV1 {
5822 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
5823 tasks: vec![current_task.clone()],
5824 runs: vec![current_run.clone()],
5825 grants: Vec::new(),
5826 operations: vec![create_run],
5827 execution_specs: Vec::new(),
5828 issuances: Vec::new(),
5829 execution_specs_v2: Vec::new(),
5830 deliveries: Vec::new(),
5831 continuations: Vec::new(),
5832 }).unwrap();
5833 let before = engine.checkpoint();
5834 let mut next_run = run(next_lifecycle, 4);
5835 next_run.updated_at_unix_ms = 20;
5836 if next_lifecycle == HarnessRunLifecycleV1::Failed {
5837 next_run.failure = Some(HarnessFailureV1 {
5838 category: HarnessFailureCategoryV1::Internal,
5839 retryable: false,
5840 });
5841 }
5842 let mut next_task = current_task.clone();
5843 next_task.revision = revision(3);
5844 next_task.state = task_state;
5845 next_task.updated_at_unix_ms = 20;
5846 let operation = HarnessOperationV1 {
5847 operation_id: operation_id(marker),
5848 revision: revision(1),
5849 actor: HarnessActorV1::ParentRun { run_id: run_id() },
5850 kind: HarnessOperationKindV1::MutateRun,
5851 state: HarnessOperationStateV1::Succeeded,
5852 task_id: None,
5853 run_id: Some(run_id()),
5854 grant_id: None,
5855 reconciles_operation_id: None,
5856 expected_revision: Some(revision(3)),
5857 request_digest: HarnessRequestDigest::new(
5858 marker.to_string().repeat(64),
5859 ).unwrap(),
5860 idempotency_ref: HarnessIdempotencyRef::new(format!(
5861 "hidem_{}",
5862 marker.to_string().repeat(24),
5863 )).unwrap(),
5864 failure: None,
5865 outcome_unknown_reason: None,
5866 reconciliation_outcome: None,
5867 created_at_unix_ms: 20,
5868 updated_at_unix_ms: 20,
5869 dispatched_at_unix_ms: None,
5870 finished_at_unix_ms: Some(20),
5871 };
5872 let prepared = engine.prepare_run_event_commit(
5873 operation.clone(),
5874 revision(3),
5875 next_run.clone(),
5876 revision(2),
5877 next_task.clone(),
5878 ).unwrap();
5879 assert_eq!(prepared.outcome(), HarnessApplyOutcome::Applied);
5880 assert_eq!(engine.checkpoint(), before);
5881 let mut committed = engine.clone();
5882 committed.accept(prepared);
5883 let committed_snapshot = committed.checkpoint();
5884 assert_eq!(committed.prepare_run_event_commit(
5885 operation.clone(),
5886 revision(3),
5887 next_run.clone(),
5888 revision(2),
5889 next_task.clone(),
5890 ).unwrap().outcome(), HarnessApplyOutcome::Replayed);
5891 assert_eq!(committed.checkpoint(), committed_snapshot);
5892
5893 let mut changed_task = next_task.clone();
5894 changed_task.title.push_str(" changed");
5895 assert!(matches!(
5896 committed.prepare_run_event_commit(
5897 operation.clone(),
5898 revision(3),
5899 next_run.clone(),
5900 revision(2),
5901 changed_task,
5902 ),
5903 Err(HarnessEngineError::OperationIdConflict { .. })
5904 ));
5905 assert_eq!(committed.checkpoint(), committed_snapshot);
5906
5907 assert!(engine.prepare_run_event_commit(
5908 operation.clone(),
5909 revision(2),
5910 next_run.clone(),
5911 revision(2),
5912 next_task.clone(),
5913 ).is_err());
5914 let mut cross_operation = operation;
5915 cross_operation.task_id = Some(task_id('f'));
5916 assert!(engine.prepare_run_event_commit(
5917 cross_operation,
5918 revision(3),
5919 next_run,
5920 revision(2),
5921 next_task,
5922 ).is_err());
5923 assert_eq!(engine.checkpoint(), before);
5924 }
5925 }
5926
5927 fn context_pack_fixture() -> HarnessEngine {
5928 let mut current_task = task(task_id('a'), 2, "context pack task");
5929 current_task.state = HarnessTaskStateV1::Review;
5930 current_task.run_ids = vec![run_id()];
5931 let current_run = run(HarnessRunLifecycleV1::Completed, 3);
5932 let mut create_run = original_run_operation(HarnessOperationStateV1::Succeeded);
5933 create_run.updated_at_unix_ms = current_run.updated_at_unix_ms;
5934 create_run.finished_at_unix_ms = Some(current_run.updated_at_unix_ms);
5935 HarnessEngine::restore(HarnessEngineCheckpointV1 {
5936 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
5937 tasks: vec![current_task],
5938 runs: vec![current_run],
5939 grants: Vec::new(),
5940 operations: vec![create_run],
5941 execution_specs: Vec::new(),
5942 issuances: Vec::new(),
5943 execution_specs_v2: Vec::new(),
5944 deliveries: Vec::new(),
5945 continuations: Vec::new(),
5946 }).unwrap()
5947 }
5948
5949 fn context_pack_receipt(digest_hex: char) -> HarnessResolvedContextPackReceiptV1 {
5950 HarnessResolvedContextPackReceiptV1 {
5951 id: HarnessSelectorV1::new("context-a").unwrap(),
5952 digest: format!("sha256:{}", digest_hex.to_string().repeat(64)),
5953 lineage: HarnessContextPackLineageV1 {
5954 source_node_id: HarnessSelectorV1::new("node-a").unwrap(),
5955 source_workspace_id: HarnessSelectorV1::new("workspace-a").unwrap(),
5956 source_instance_id: 1,
5957 source_generation: 1,
5958 source_provider: HarnessSelectorV1::new("claude").unwrap(),
5959 },
5960 source_message_count: 4,
5961 retained_message_count: 4,
5962 byte_len: 64,
5963 truncated: false,
5964 }
5965 }
5966
5967 fn context_pack_record_operation(
5968 marker: char,
5969 expected_revision: HarnessRevision,
5970 at_unix_ms: u64,
5971 ) -> HarnessOperationV1 {
5972 HarnessOperationV1 {
5973 operation_id: operation_id(marker),
5974 revision: revision(1),
5975 actor: HarnessActorV1::ParentRun { run_id: run_id() },
5976 kind: HarnessOperationKindV1::RecordRunContextPack,
5977 state: HarnessOperationStateV1::Succeeded,
5978 task_id: None,
5979 run_id: Some(run_id()),
5980 grant_id: None,
5981 reconciles_operation_id: None,
5982 expected_revision: Some(expected_revision),
5983 request_digest: HarnessRequestDigest::new(marker.to_string().repeat(64)).unwrap(),
5984 idempotency_ref: HarnessIdempotencyRef::new(format!(
5985 "hidem_{}",
5986 marker.to_string().repeat(24),
5987 )).unwrap(),
5988 failure: None,
5989 outcome_unknown_reason: None,
5990 reconciliation_outcome: None,
5991 created_at_unix_ms: at_unix_ms,
5992 updated_at_unix_ms: at_unix_ms,
5993 dispatched_at_unix_ms: None,
5994 finished_at_unix_ms: Some(at_unix_ms),
5995 }
5996 }
5997
5998 #[test]
5999 fn context_pack_record_freezes_every_field_except_pack_revision_and_updated_at() {
6000 let engine = context_pack_fixture();
6001 let current_run = engine.run(&run_id()).unwrap().clone();
6002 let operation = context_pack_record_operation('1', current_run.revision, 30);
6003
6004 let mut tampered = current_run.clone();
6005 tampered.context_pack = Some(context_pack_receipt('a'));
6006 tampered.revision = revision(current_run.revision.get() + 1);
6007 tampered.updated_at_unix_ms = 30;
6008 tampered.parent_run_id = Some(numbered_run_id(999));
6009 let before = engine.checkpoint();
6010 assert!(matches!(
6011 engine.prepare_run_context_pack_record(operation.clone(), current_run.revision, tampered),
6012 Err(HarnessEngineError::InvalidContextPackRecordProjection),
6013 ));
6014 assert_eq!(engine.checkpoint(), before);
6015
6016 let mut clean = current_run.clone();
6017 clean.context_pack = Some(context_pack_receipt('a'));
6018 clean.revision = revision(current_run.revision.get() + 1);
6019 clean.updated_at_unix_ms = 30;
6020 let prepared = engine.prepare_run_context_pack_record(
6021 operation,
6022 current_run.revision,
6023 clean.clone(),
6024 ).unwrap();
6025 assert_eq!(prepared.outcome(), HarnessApplyOutcome::Applied);
6026 let mut committed = engine.clone();
6027 committed.accept(prepared);
6028 let recorded = committed.run(&run_id()).unwrap().clone();
6029 let mut expected = current_run;
6030 expected.context_pack = clean.context_pack;
6031 expected.revision = clean.revision;
6032 expected.updated_at_unix_ms = clean.updated_at_unix_ms;
6033 assert_eq!(recorded, expected);
6034 }
6035
6036 #[test]
6037 fn context_pack_record_rejects_a_second_different_receipt_as_conflict() {
6038 let engine = context_pack_fixture();
6039 let current_run = engine.run(&run_id()).unwrap().clone();
6040 let first_operation = context_pack_record_operation('2', current_run.revision, 30);
6041 let mut first_next = current_run.clone();
6042 first_next.context_pack = Some(context_pack_receipt('a'));
6043 first_next.revision = revision(current_run.revision.get() + 1);
6044 first_next.updated_at_unix_ms = 30;
6045 let prepared = engine.prepare_run_context_pack_record(
6046 first_operation,
6047 current_run.revision,
6048 first_next,
6049 ).unwrap();
6050 let mut committed = engine.clone();
6051 committed.accept(prepared);
6052 let before = committed.checkpoint();
6053
6054 let recorded_run = committed.run(&run_id()).unwrap().clone();
6055 let second_operation = context_pack_record_operation('3', recorded_run.revision, 40);
6056 let mut second_next = recorded_run.clone();
6057 second_next.context_pack = Some(context_pack_receipt('b'));
6058 second_next.revision = revision(recorded_run.revision.get() + 1);
6059 second_next.updated_at_unix_ms = 40;
6060 assert!(matches!(
6061 committed.prepare_run_context_pack_record(
6062 second_operation,
6063 recorded_run.revision,
6064 second_next,
6065 ),
6066 Err(HarnessEngineError::TerminalRunMutation),
6067 ));
6068 assert_eq!(committed.checkpoint(), before);
6069 }
6070
6071 #[test]
6072 fn context_pack_record_is_idempotent_on_the_same_digest() {
6073 let engine = context_pack_fixture();
6074 let current_run = engine.run(&run_id()).unwrap().clone();
6075 let operation = context_pack_record_operation('4', current_run.revision, 30);
6076 let mut next = current_run.clone();
6077 next.context_pack = Some(context_pack_receipt('a'));
6078 next.revision = revision(current_run.revision.get() + 1);
6079 next.updated_at_unix_ms = 30;
6080 let prepared = engine.prepare_run_context_pack_record(
6081 operation,
6082 current_run.revision,
6083 next,
6084 ).unwrap();
6085 let mut committed = engine.clone();
6086 committed.accept(prepared);
6087 let committed_snapshot = committed.checkpoint();
6088
6089 let replay_operation = context_pack_record_operation('5', current_run.revision, 31);
6093 let mut replay_next = current_run.clone();
6094 replay_next.context_pack = Some(context_pack_receipt('a'));
6095 replay_next.revision = revision(current_run.revision.get() + 1);
6096 replay_next.updated_at_unix_ms = 31;
6097 let replayed = committed.prepare_run_context_pack_record(
6098 replay_operation,
6099 current_run.revision,
6100 replay_next,
6101 ).unwrap();
6102 assert_eq!(replayed.outcome(), HarnessApplyOutcome::Replayed);
6103 assert_eq!(replayed.checkpoint(), committed_snapshot);
6104 }
6105
6106 #[test]
6107 fn context_pack_record_keeps_outcome_unknown_run_frozen() {
6108 let mut current_task = task(task_id('a'), 2, "frozen task");
6109 current_task.state = HarnessTaskStateV1::Waiting;
6110 current_task.run_ids = vec![run_id()];
6111 let mut current_run = run(HarnessRunLifecycleV1::Requested, 3);
6112 current_run.lifecycle = HarnessRunLifecycleV1::OutcomeUnknown;
6113 let mut create_run = original_run_operation(HarnessOperationStateV1::OutcomeUnknown);
6114 create_run.outcome_unknown_reason = Some(HarnessOutcomeUnknownReasonV1::Timeout);
6115 let engine = HarnessEngine::restore(HarnessEngineCheckpointV1 {
6116 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
6117 tasks: vec![current_task],
6118 runs: vec![current_run.clone()],
6119 grants: Vec::new(),
6120 operations: vec![create_run],
6121 execution_specs: Vec::new(),
6122 issuances: Vec::new(),
6123 execution_specs_v2: Vec::new(),
6124 deliveries: Vec::new(),
6125 continuations: Vec::new(),
6126 }).unwrap();
6127 let operation = context_pack_record_operation('6', current_run.revision, 30);
6128 let before = engine.checkpoint();
6129 assert!(matches!(
6130 engine.prepare_run_context_pack_record(operation, current_run.revision, current_run),
6131 Err(HarnessEngineError::OutcomeUnknownRunFrozen),
6132 ));
6133 assert_eq!(engine.checkpoint(), before);
6134 }
6135
6136 #[test]
6137 fn context_pack_record_never_touches_task_state() {
6138 let engine = context_pack_fixture();
6139 let current_run = engine.run(&run_id()).unwrap().clone();
6140 let original_task = engine.task(&task_id('a')).unwrap().clone();
6141 let operation = context_pack_record_operation('7', current_run.revision, 30);
6142 let mut next = current_run.clone();
6143 next.context_pack = Some(context_pack_receipt('a'));
6144 next.revision = revision(current_run.revision.get() + 1);
6145 next.updated_at_unix_ms = 30;
6146 let prepared = engine.prepare_run_context_pack_record(
6147 operation,
6148 current_run.revision,
6149 next,
6150 ).unwrap();
6151 let mut committed = engine.clone();
6152 committed.accept(prepared);
6153 assert_eq!(committed.task(&task_id('a')).unwrap(), &original_task);
6154 }
6155
6156 #[test]
6157 fn context_pack_record_freezes_git_facts_too() {
6158 let engine = context_pack_fixture();
6164 let current_run = engine.run(&run_id()).unwrap().clone();
6165 let operation = context_pack_record_operation('8', current_run.revision, 30);
6166 let mut tampered = current_run.clone();
6167 tampered.context_pack = Some(context_pack_receipt('a'));
6168 tampered.git_facts = Some(git_facts('a', 30));
6169 tampered.revision = revision(current_run.revision.get() + 1);
6170 tampered.updated_at_unix_ms = 30;
6171 let before = engine.checkpoint();
6172 assert!(matches!(
6173 engine.prepare_run_context_pack_record(operation, current_run.revision, tampered),
6174 Err(HarnessEngineError::InvalidContextPackRecordProjection),
6175 ));
6176 assert_eq!(engine.checkpoint(), before);
6177 }
6178
6179 fn git_facts_summary(marker: char) -> HarnessRunGitSummaryV1 {
6180 HarnessRunGitSummaryV1 {
6181 is_repository: true,
6182 branch: Some(format!("branch-{marker}")),
6183 status: vec![HarnessRunGitStatusEntryV1 {
6184 index_status: HarnessRunGitStatusCodeV1::Unmodified,
6185 worktree_status: HarnessRunGitStatusCodeV1::Modified,
6186 path: "src/lib.rs".to_owned(),
6187 previous_path: None,
6188 }],
6189 recent_commits: vec![HarnessRunGitCommitSummaryV1 {
6190 id: marker.to_string().repeat(40),
6191 summary: "commit summary".to_owned(),
6192 }],
6193 truncated: false,
6194 }
6195 }
6196
6197 fn git_facts(marker: char, at_unix_ms: u64) -> HarnessRunGitFactsV1 {
6198 HarnessRunGitFactsV1 {
6199 captured_at_unix_ms: at_unix_ms,
6200 outcome: HarnessRunGitFactsOutcomeV1::Captured(git_facts_summary(marker)),
6201 }
6202 }
6203
6204 fn git_facts_record_operation(
6205 marker: char,
6206 expected_revision: HarnessRevision,
6207 at_unix_ms: u64,
6208 ) -> HarnessOperationV1 {
6209 HarnessOperationV1 {
6210 operation_id: operation_id(marker),
6211 revision: revision(1),
6212 actor: HarnessActorV1::ParentRun { run_id: run_id() },
6213 kind: HarnessOperationKindV1::RecordRunGitFacts,
6214 state: HarnessOperationStateV1::Succeeded,
6215 task_id: None,
6216 run_id: Some(run_id()),
6217 grant_id: None,
6218 reconciles_operation_id: None,
6219 expected_revision: Some(expected_revision),
6220 request_digest: HarnessRequestDigest::new(marker.to_string().repeat(64)).unwrap(),
6221 idempotency_ref: HarnessIdempotencyRef::new(format!(
6222 "hidem_{}",
6223 marker.to_string().repeat(24),
6224 )).unwrap(),
6225 failure: None,
6226 outcome_unknown_reason: None,
6227 reconciliation_outcome: None,
6228 created_at_unix_ms: at_unix_ms,
6229 updated_at_unix_ms: at_unix_ms,
6230 dispatched_at_unix_ms: None,
6231 finished_at_unix_ms: Some(at_unix_ms),
6232 }
6233 }
6234
6235 #[test]
6236 fn git_facts_record_freezes_every_field_except_facts_revision_and_updated_at() {
6237 let engine = context_pack_fixture();
6238 let current_run = engine.run(&run_id()).unwrap().clone();
6239 let operation = git_facts_record_operation('1', current_run.revision, 30);
6240
6241 let mut tampered = current_run.clone();
6242 tampered.git_facts = Some(git_facts('a', 30));
6243 tampered.revision = revision(current_run.revision.get() + 1);
6244 tampered.updated_at_unix_ms = 30;
6245 tampered.parent_run_id = Some(numbered_run_id(999));
6246 let before = engine.checkpoint();
6247 assert!(matches!(
6248 engine.prepare_run_git_facts_record(operation.clone(), current_run.revision, tampered),
6249 Err(HarnessEngineError::InvalidGitFactsRecordProjection),
6250 ));
6251 assert_eq!(engine.checkpoint(), before);
6252
6253 let mut clean = current_run.clone();
6254 clean.git_facts = Some(git_facts('a', 30));
6255 clean.revision = revision(current_run.revision.get() + 1);
6256 clean.updated_at_unix_ms = 30;
6257 let prepared = engine.prepare_run_git_facts_record(
6258 operation,
6259 current_run.revision,
6260 clean.clone(),
6261 ).unwrap();
6262 assert_eq!(prepared.outcome(), HarnessApplyOutcome::Applied);
6263 let mut committed = engine.clone();
6264 committed.accept(prepared);
6265 let recorded = committed.run(&run_id()).unwrap().clone();
6266 let mut expected = current_run;
6267 expected.git_facts = clean.git_facts;
6268 expected.revision = clean.revision;
6269 expected.updated_at_unix_ms = clean.updated_at_unix_ms;
6270 assert_eq!(recorded, expected);
6271 }
6272
6273 #[test]
6274 fn git_facts_record_rejects_a_second_different_value_as_conflict() {
6275 let engine = context_pack_fixture();
6276 let current_run = engine.run(&run_id()).unwrap().clone();
6277 let first_operation = git_facts_record_operation('2', current_run.revision, 30);
6278 let mut first_next = current_run.clone();
6279 first_next.git_facts = Some(git_facts('a', 30));
6280 first_next.revision = revision(current_run.revision.get() + 1);
6281 first_next.updated_at_unix_ms = 30;
6282 let prepared = engine.prepare_run_git_facts_record(
6283 first_operation,
6284 current_run.revision,
6285 first_next,
6286 ).unwrap();
6287 let mut committed = engine.clone();
6288 committed.accept(prepared);
6289 let before = committed.checkpoint();
6290
6291 let recorded_run = committed.run(&run_id()).unwrap().clone();
6292 let second_operation = git_facts_record_operation('3', recorded_run.revision, 40);
6293 let mut second_next = recorded_run.clone();
6294 second_next.git_facts = Some(git_facts('b', 40));
6295 second_next.revision = revision(recorded_run.revision.get() + 1);
6296 second_next.updated_at_unix_ms = 40;
6297 assert!(matches!(
6298 committed.prepare_run_git_facts_record(
6299 second_operation,
6300 recorded_run.revision,
6301 second_next,
6302 ),
6303 Err(HarnessEngineError::TerminalRunMutation),
6304 ));
6305 assert_eq!(committed.checkpoint(), before);
6306 }
6307
6308 #[test]
6309 fn git_facts_record_is_idempotent_on_the_same_value() {
6310 let engine = context_pack_fixture();
6311 let current_run = engine.run(&run_id()).unwrap().clone();
6312 let operation = git_facts_record_operation('4', current_run.revision, 30);
6313 let mut next = current_run.clone();
6314 next.git_facts = Some(git_facts('a', 30));
6315 next.revision = revision(current_run.revision.get() + 1);
6316 next.updated_at_unix_ms = 30;
6317 let prepared = engine.prepare_run_git_facts_record(
6318 operation,
6319 current_run.revision,
6320 next,
6321 ).unwrap();
6322 let mut committed = engine.clone();
6323 committed.accept(prepared);
6324 let committed_snapshot = committed.checkpoint();
6325
6326 let replay_operation = git_facts_record_operation('5', current_run.revision, 31);
6331 let mut replay_next = current_run.clone();
6332 replay_next.git_facts = Some(git_facts('a', 30));
6333 replay_next.revision = revision(current_run.revision.get() + 1);
6334 replay_next.updated_at_unix_ms = 31;
6335 let replayed = committed.prepare_run_git_facts_record(
6336 replay_operation,
6337 current_run.revision,
6338 replay_next,
6339 ).unwrap();
6340 assert_eq!(replayed.outcome(), HarnessApplyOutcome::Replayed);
6341 assert_eq!(replayed.checkpoint(), committed_snapshot);
6342 }
6343
6344 #[test]
6345 fn git_facts_record_keeps_outcome_unknown_run_frozen() {
6346 let mut current_task = task(task_id('a'), 2, "frozen task");
6347 current_task.state = HarnessTaskStateV1::Waiting;
6348 current_task.run_ids = vec![run_id()];
6349 let mut current_run = run(HarnessRunLifecycleV1::Requested, 3);
6350 current_run.lifecycle = HarnessRunLifecycleV1::OutcomeUnknown;
6351 let mut create_run = original_run_operation(HarnessOperationStateV1::OutcomeUnknown);
6352 create_run.outcome_unknown_reason = Some(HarnessOutcomeUnknownReasonV1::Timeout);
6353 let engine = HarnessEngine::restore(HarnessEngineCheckpointV1 {
6354 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
6355 tasks: vec![current_task],
6356 runs: vec![current_run.clone()],
6357 grants: Vec::new(),
6358 operations: vec![create_run],
6359 execution_specs: Vec::new(),
6360 issuances: Vec::new(),
6361 execution_specs_v2: Vec::new(),
6362 deliveries: Vec::new(),
6363 continuations: Vec::new(),
6364 }).unwrap();
6365 let operation = git_facts_record_operation('6', current_run.revision, 30);
6366 let before = engine.checkpoint();
6367 assert!(matches!(
6368 engine.prepare_run_git_facts_record(operation, current_run.revision, current_run),
6369 Err(HarnessEngineError::OutcomeUnknownRunFrozen),
6370 ));
6371 assert_eq!(engine.checkpoint(), before);
6372 }
6373
6374 #[test]
6375 fn git_facts_record_never_touches_task_state() {
6376 let engine = context_pack_fixture();
6377 let current_run = engine.run(&run_id()).unwrap().clone();
6378 let original_task = engine.task(&task_id('a')).unwrap().clone();
6379 let operation = git_facts_record_operation('7', current_run.revision, 30);
6380 let mut next = current_run.clone();
6381 next.git_facts = Some(git_facts('a', 30));
6382 next.revision = revision(current_run.revision.get() + 1);
6383 next.updated_at_unix_ms = 30;
6384 let prepared = engine.prepare_run_git_facts_record(
6385 operation,
6386 current_run.revision,
6387 next,
6388 ).unwrap();
6389 let mut committed = engine.clone();
6390 committed.accept(prepared);
6391 assert_eq!(committed.task(&task_id('a')).unwrap(), &original_task);
6392 }
6393
6394 fn result_ref_fixture() -> HarnessEngine {
6395 let mut current_task = task(task_id('a'), 2, "result ref task");
6396 current_task.state = HarnessTaskStateV1::Review;
6397 current_task.run_ids = vec![numbered_run_id(1), numbered_run_id(2)];
6398
6399 let mut run_one = run(HarnessRunLifecycleV1::Completed, 3);
6400 run_one.run_id = numbered_run_id(1);
6401 run_one.operation_id = operation_id('e');
6402 let mut create_run_one = original_run_operation(HarnessOperationStateV1::Succeeded);
6403 create_run_one.operation_id = operation_id('e');
6404 create_run_one.run_id = Some(numbered_run_id(1));
6405 create_run_one.request_digest = HarnessRequestDigest::new("e".repeat(64)).unwrap();
6406 create_run_one.idempotency_ref = HarnessIdempotencyRef::new(format!(
6407 "hidem_{}",
6408 "e".repeat(24),
6409 )).unwrap();
6410 create_run_one.updated_at_unix_ms = run_one.updated_at_unix_ms;
6411 create_run_one.finished_at_unix_ms = Some(run_one.updated_at_unix_ms);
6412
6413 let mut run_two = run(HarnessRunLifecycleV1::Completed, 3);
6414 run_two.run_id = numbered_run_id(2);
6415 run_two.operation_id = operation_id('f');
6416 let mut create_run_two = original_run_operation(HarnessOperationStateV1::Succeeded);
6417 create_run_two.operation_id = operation_id('f');
6418 create_run_two.run_id = Some(numbered_run_id(2));
6419 create_run_two.request_digest = HarnessRequestDigest::new("f".repeat(64)).unwrap();
6420 create_run_two.idempotency_ref = HarnessIdempotencyRef::new(format!(
6421 "hidem_{}",
6422 "f".repeat(24),
6423 )).unwrap();
6424 create_run_two.updated_at_unix_ms = run_two.updated_at_unix_ms;
6425 create_run_two.finished_at_unix_ms = Some(run_two.updated_at_unix_ms);
6426
6427 HarnessEngine::restore(HarnessEngineCheckpointV1 {
6428 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
6429 tasks: vec![current_task],
6430 runs: vec![run_one, run_two],
6431 grants: Vec::new(),
6432 operations: vec![create_run_one, create_run_two],
6433 execution_specs: Vec::new(),
6434 issuances: Vec::new(),
6435 execution_specs_v2: Vec::new(),
6436 deliveries: Vec::new(),
6437 continuations: Vec::new(),
6438 }).unwrap()
6439 }
6440
6441 fn result_ref_record_operation(
6442 marker: char,
6443 for_run_id: HarnessRunId,
6444 expected_task_revision: HarnessRevision,
6445 at_unix_ms: u64,
6446 ) -> HarnessOperationV1 {
6447 HarnessOperationV1 {
6448 operation_id: operation_id(marker),
6449 revision: revision(1),
6450 actor: HarnessActorV1::ParentRun { run_id: for_run_id },
6451 kind: HarnessOperationKindV1::RecordTaskResultRef,
6452 state: HarnessOperationStateV1::Succeeded,
6453 task_id: Some(task_id('a')),
6454 run_id: None,
6455 grant_id: None,
6456 reconciles_operation_id: None,
6457 expected_revision: Some(expected_task_revision),
6458 request_digest: HarnessRequestDigest::new(marker.to_string().repeat(64)).unwrap(),
6459 idempotency_ref: HarnessIdempotencyRef::new(format!(
6460 "hidem_{}",
6461 marker.to_string().repeat(24),
6462 )).unwrap(),
6463 failure: None,
6464 outcome_unknown_reason: None,
6465 reconciliation_outcome: None,
6466 created_at_unix_ms: at_unix_ms,
6467 updated_at_unix_ms: at_unix_ms,
6468 dispatched_at_unix_ms: None,
6469 finished_at_unix_ms: Some(at_unix_ms),
6470 }
6471 }
6472
6473 #[test]
6474 fn result_ref_record_freezes_every_field_except_refs_revision_and_updated_at() {
6475 let engine = result_ref_fixture();
6476 let current_task = engine.task(&task_id('a')).unwrap().clone();
6477 let candidate = HarnessResultRef::for_run(&numbered_run_id(1));
6478 let operation = result_ref_record_operation(
6479 '1',
6480 numbered_run_id(1),
6481 current_task.revision,
6482 30,
6483 );
6484
6485 let mut tampered = current_task.clone();
6488 tampered.result_refs = vec![candidate.clone()];
6489 tampered.revision = revision(current_task.revision.get() + 1);
6490 tampered.updated_at_unix_ms = 30;
6491 tampered.state = HarnessTaskStateV1::Done;
6492 let before = engine.checkpoint();
6493 assert!(matches!(
6494 engine.prepare_task_result_ref_record(operation.clone(), current_task.revision, tampered),
6495 Err(HarnessEngineError::InvalidResultRefRecordProjection),
6496 ));
6497 assert_eq!(engine.checkpoint(), before);
6498
6499 let mut clean = current_task.clone();
6500 clean.result_refs = vec![candidate.clone()];
6501 clean.revision = revision(current_task.revision.get() + 1);
6502 clean.updated_at_unix_ms = 30;
6503 let prepared = engine.prepare_task_result_ref_record(
6504 operation,
6505 current_task.revision,
6506 clean.clone(),
6507 ).unwrap();
6508 assert_eq!(prepared.outcome(), HarnessApplyOutcome::Applied);
6509 let mut committed = engine.clone();
6510 committed.accept(prepared);
6511 let recorded = committed.task(&task_id('a')).unwrap().clone();
6512 let mut expected = current_task;
6513 expected.result_refs = clean.result_refs;
6514 expected.revision = clean.revision;
6515 expected.updated_at_unix_ms = clean.updated_at_unix_ms;
6516 assert_eq!(recorded, expected);
6517 }
6518
6519 #[test]
6520 fn result_ref_record_inserts_at_sorted_position() {
6521 let engine = result_ref_fixture();
6522 let current_task = engine.task(&task_id('a')).unwrap().clone();
6523 let ref_one = HarnessResultRef::for_run(&numbered_run_id(1));
6524 let ref_two = HarnessResultRef::for_run(&numbered_run_id(2));
6525 assert!(ref_one < ref_two, "fixture run ids must sort ref_one before ref_two");
6526
6527 let first_operation = result_ref_record_operation(
6529 '1',
6530 numbered_run_id(2),
6531 current_task.revision,
6532 30,
6533 );
6534 let mut first_next = current_task.clone();
6535 first_next.result_refs = vec![ref_two.clone()];
6536 first_next.revision = revision(current_task.revision.get() + 1);
6537 first_next.updated_at_unix_ms = 30;
6538 let prepared = engine.prepare_task_result_ref_record(
6539 first_operation,
6540 current_task.revision,
6541 first_next,
6542 ).unwrap();
6543 assert_eq!(prepared.outcome(), HarnessApplyOutcome::Applied);
6544 let mut committed = engine.clone();
6545 committed.accept(prepared);
6546
6547 let after_first = committed.task(&task_id('a')).unwrap().clone();
6551 let second_operation = result_ref_record_operation(
6552 '2',
6553 numbered_run_id(1),
6554 after_first.revision,
6555 40,
6556 );
6557 let mut second_next = after_first.clone();
6558 second_next.result_refs = vec![ref_one.clone(), ref_two.clone()];
6559 second_next.revision = revision(after_first.revision.get() + 1);
6560 second_next.updated_at_unix_ms = 40;
6561 let prepared = committed.prepare_task_result_ref_record(
6562 second_operation,
6563 after_first.revision,
6564 second_next,
6565 ).unwrap();
6566 assert_eq!(prepared.outcome(), HarnessApplyOutcome::Applied);
6567 committed.accept(prepared);
6568 assert_eq!(
6569 committed.task(&task_id('a')).unwrap().result_refs,
6570 vec![ref_one, ref_two],
6571 );
6572 }
6573
6574 #[test]
6575 fn result_ref_record_is_idempotent_on_replay() {
6576 let engine = result_ref_fixture();
6577 let current_task = engine.task(&task_id('a')).unwrap().clone();
6578 let candidate = HarnessResultRef::for_run(&numbered_run_id(1));
6579 let operation = result_ref_record_operation(
6580 '1',
6581 numbered_run_id(1),
6582 current_task.revision,
6583 30,
6584 );
6585 let mut next = current_task.clone();
6586 next.result_refs = vec![candidate.clone()];
6587 next.revision = revision(current_task.revision.get() + 1);
6588 next.updated_at_unix_ms = 30;
6589 let prepared = engine.prepare_task_result_ref_record(
6590 operation,
6591 current_task.revision,
6592 next,
6593 ).unwrap();
6594 let mut committed = engine.clone();
6595 committed.accept(prepared);
6596 let committed_snapshot = committed.checkpoint();
6597
6598 let replay_operation = result_ref_record_operation(
6603 '2',
6604 numbered_run_id(1),
6605 current_task.revision,
6606 31,
6607 );
6608 let mut replay_next = current_task.clone();
6609 replay_next.result_refs = vec![candidate];
6610 replay_next.revision = revision(current_task.revision.get() + 1);
6611 replay_next.updated_at_unix_ms = 31;
6612 let replayed = committed.prepare_task_result_ref_record(
6613 replay_operation,
6614 current_task.revision,
6615 replay_next,
6616 ).unwrap();
6617 assert_eq!(replayed.outcome(), HarnessApplyOutcome::Replayed);
6618 assert_eq!(replayed.checkpoint(), committed_snapshot);
6619 }
6620
6621 #[test]
6622 fn result_ref_record_is_a_no_op_at_capacity() {
6623 let mut current_task = task(task_id('a'), 2, "full task");
6624 current_task.state = HarnessTaskStateV1::Review;
6625 current_task.result_refs = (0..HARNESS_RESULTS_MAX)
6626 .map(numbered_run_id)
6627 .map(|run_id| HarnessResultRef::for_run(&run_id))
6628 .collect();
6629 let engine = HarnessEngine::restore(HarnessEngineCheckpointV1 {
6630 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
6631 tasks: vec![current_task.clone()],
6632 runs: Vec::new(),
6633 grants: Vec::new(),
6634 operations: Vec::new(),
6635 execution_specs: Vec::new(),
6636 issuances: Vec::new(),
6637 execution_specs_v2: Vec::new(),
6638 deliveries: Vec::new(),
6639 continuations: Vec::new(),
6640 }).unwrap();
6641
6642 let replay_operation = result_ref_record_operation(
6646 '1',
6647 numbered_run_id(0),
6648 current_task.revision,
6649 30,
6650 );
6651 let mut replay_next = current_task.clone();
6652 replay_next.revision = revision(current_task.revision.get() + 1);
6653 replay_next.updated_at_unix_ms = 30;
6654 let replayed = engine.prepare_task_result_ref_record(
6655 replay_operation,
6656 current_task.revision,
6657 replay_next,
6658 ).unwrap();
6659 assert_eq!(replayed.outcome(), HarnessApplyOutcome::Replayed);
6660 assert_eq!(replayed.checkpoint(), engine.checkpoint());
6661
6662 let substitute_operation = result_ref_record_operation(
6666 '2',
6667 numbered_run_id(HARNESS_RESULTS_MAX),
6668 current_task.revision,
6669 30,
6670 );
6671 let mut substitute_next = current_task.clone();
6672 substitute_next.result_refs.pop();
6673 substitute_next.result_refs.push(
6674 HarnessResultRef::for_run(&numbered_run_id(HARNESS_RESULTS_MAX)),
6675 );
6676 substitute_next.result_refs.sort();
6677 substitute_next.revision = revision(current_task.revision.get() + 1);
6678 substitute_next.updated_at_unix_ms = 30;
6679 let before = engine.checkpoint();
6680 assert!(matches!(
6681 engine.prepare_task_result_ref_record(
6682 substitute_operation,
6683 current_task.revision,
6684 substitute_next,
6685 ),
6686 Err(HarnessEngineError::InvalidResultRefRecordProjection),
6687 ));
6688 assert_eq!(engine.checkpoint(), before);
6689 }
6690
6691 #[test]
6692 fn result_ref_record_rejects_removal_or_reorder() {
6693 let mut current_task = task(task_id('a'), 2, "seeded task");
6694 current_task.state = HarnessTaskStateV1::Review;
6695 let ref_one = HarnessResultRef::for_run(&numbered_run_id(1));
6696 let ref_two = HarnessResultRef::for_run(&numbered_run_id(2));
6697 let ref_three = HarnessResultRef::for_run(&numbered_run_id(3));
6698 current_task.result_refs = vec![ref_one.clone(), ref_two.clone()];
6699 let engine = HarnessEngine::restore(HarnessEngineCheckpointV1 {
6700 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
6701 tasks: vec![current_task.clone()],
6702 runs: Vec::new(),
6703 grants: Vec::new(),
6704 operations: Vec::new(),
6705 execution_specs: Vec::new(),
6706 issuances: Vec::new(),
6707 execution_specs_v2: Vec::new(),
6708 deliveries: Vec::new(),
6709 continuations: Vec::new(),
6710 }).unwrap();
6711 let before = engine.checkpoint();
6712
6713 let removal_operation = result_ref_record_operation(
6715 '1',
6716 numbered_run_id(1),
6717 current_task.revision,
6718 30,
6719 );
6720 let mut removed = current_task.clone();
6721 removed.result_refs = vec![ref_one.clone()];
6722 removed.revision = revision(current_task.revision.get() + 1);
6723 removed.updated_at_unix_ms = 30;
6724 assert!(matches!(
6725 engine.prepare_task_result_ref_record(removal_operation, current_task.revision, removed),
6726 Err(HarnessEngineError::InvalidResultRefRecordProjection),
6727 ));
6728
6729 let reorder_operation = result_ref_record_operation(
6733 '2',
6734 numbered_run_id(1),
6735 current_task.revision,
6736 30,
6737 );
6738 let mut reordered = current_task.clone();
6739 reordered.result_refs = vec![ref_two.clone(), ref_one.clone()];
6740 reordered.revision = revision(current_task.revision.get() + 1);
6741 reordered.updated_at_unix_ms = 30;
6742 assert!(matches!(
6743 engine.prepare_task_result_ref_record(reorder_operation, current_task.revision, reordered),
6744 Err(HarnessEngineError::Validation(HarnessValidationError::CollectionNotCanonical {
6745 field: "result_refs",
6746 })),
6747 ));
6748
6749 let substitution_operation = result_ref_record_operation(
6752 '3',
6753 numbered_run_id(1),
6754 current_task.revision,
6755 30,
6756 );
6757 let mut substituted = current_task.clone();
6758 substituted.result_refs = vec![ref_one, ref_three];
6759 substituted.revision = revision(current_task.revision.get() + 1);
6760 substituted.updated_at_unix_ms = 30;
6761 assert!(matches!(
6762 engine.prepare_task_result_ref_record(
6763 substitution_operation,
6764 current_task.revision,
6765 substituted,
6766 ),
6767 Err(HarnessEngineError::InvalidResultRefRecordProjection),
6768 ));
6769
6770 assert_eq!(engine.checkpoint(), before);
6771 }
6772
6773 #[test]
6774 fn dispatch_outcome_commit_is_atomic_exact_replay_and_fail_closed() {
6775 for outcome_unknown in [false, true] {
6776 let mut current_task = task(task_id('a'), 2, "dispatch task");
6777 current_task.state = HarnessTaskStateV1::Running;
6778 current_task.run_ids = vec![run_id()];
6779 let current_run = run(HarnessRunLifecycleV1::Dispatching, 2);
6780 let mut current_operation = original_run_operation(
6781 HarnessOperationStateV1::Dispatching,
6782 );
6783 current_operation.revision = revision(2);
6784 let engine = HarnessEngine::restore(HarnessEngineCheckpointV1 {
6785 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
6786 tasks: vec![current_task.clone()],
6787 runs: vec![current_run],
6788 grants: Vec::new(),
6789 operations: vec![current_operation.clone()],
6790 execution_specs: Vec::new(),
6791 issuances: Vec::new(),
6792 execution_specs_v2: Vec::new(),
6793 deliveries: Vec::new(),
6794 continuations: Vec::new(),
6795 }).unwrap();
6796 let before = engine.checkpoint();
6797 let next_lifecycle = if outcome_unknown {
6798 HarnessRunLifecycleV1::OutcomeUnknown
6799 } else {
6800 HarnessRunLifecycleV1::Failed
6801 };
6802 let mut next_run = run(next_lifecycle, 3);
6803 next_run.updated_at_unix_ms = 20;
6804 let mut next_operation = current_operation;
6805 next_operation.revision = revision(3);
6806 next_operation.updated_at_unix_ms = 20;
6807 let mut next_task = current_task;
6808 next_task.revision = revision(3);
6809 next_task.updated_at_unix_ms = 20;
6810 if outcome_unknown {
6811 next_operation.state = HarnessOperationStateV1::OutcomeUnknown;
6812 next_operation.outcome_unknown_reason = Some(
6813 HarnessOutcomeUnknownReasonV1::ReplyLost,
6814 );
6815 next_task.state = HarnessTaskStateV1::Waiting;
6816 } else {
6817 let failure = HarnessFailureV1 {
6818 category: HarnessFailureCategoryV1::Rejected,
6819 retryable: false,
6820 };
6821 next_run.failure = Some(failure.clone());
6822 next_operation.state = HarnessOperationStateV1::Failed;
6823 next_operation.failure = Some(failure);
6824 next_operation.finished_at_unix_ms = Some(20);
6825 next_task.state = HarnessTaskStateV1::Failed;
6826 }
6827 let prepared = engine.prepare_dispatch_outcome_commit(
6828 revision(2),
6829 next_run.clone(),
6830 revision(2),
6831 next_operation.clone(),
6832 revision(2),
6833 next_task.clone(),
6834 ).unwrap();
6835 assert_eq!(prepared.outcome(), HarnessApplyOutcome::Applied);
6836 assert_eq!(engine.checkpoint(), before);
6837 let mut committed = engine.clone();
6838 committed.accept(prepared);
6839 let committed_snapshot = committed.checkpoint();
6840 assert_eq!(committed.prepare_dispatch_outcome_commit(
6841 revision(2),
6842 next_run.clone(),
6843 revision(2),
6844 next_operation.clone(),
6845 revision(2),
6846 next_task.clone(),
6847 ).unwrap().outcome(), HarnessApplyOutcome::Replayed);
6848 assert_eq!(committed.checkpoint(), committed_snapshot);
6849
6850 let mut changed_task = next_task.clone();
6851 changed_task.title.push_str(" drift");
6852 assert!(committed.prepare_dispatch_outcome_commit(
6853 revision(2),
6854 next_run.clone(),
6855 revision(2),
6856 next_operation.clone(),
6857 revision(2),
6858 changed_task,
6859 ).is_err());
6860 assert_eq!(committed.checkpoint(), committed_snapshot);
6861 assert!(engine.prepare_dispatch_outcome_commit(
6862 revision(1),
6863 next_run.clone(),
6864 revision(2),
6865 next_operation.clone(),
6866 revision(2),
6867 next_task.clone(),
6868 ).is_err());
6869 let mut cross_operation = next_operation;
6870 cross_operation.run_id = Some(numbered_run_id(99));
6871 assert!(engine.prepare_dispatch_outcome_commit(
6872 revision(2),
6873 next_run,
6874 revision(2),
6875 cross_operation,
6876 revision(2),
6877 next_task,
6878 ).is_err());
6879 assert_eq!(engine.checkpoint(), before);
6880 }
6881 }
6882
6883 #[test]
6884 fn harness_grant_revoke_is_terminal_and_cannot_reactivate() {
6885 let active = grant(SessionGrantStateV1::Active, 1);
6886 let still_active = grant(SessionGrantStateV1::Active, 2);
6887 assert!(matches!(
6888 validate_grant_transition(
6889 &active,
6890 &still_active,
6891 HarnessOperationKindV1::RevokeGrant,
6892 ),
6893 Err(HarnessEngineError::RevokeGrantMustRevoke)
6894 ));
6895 let revoked = grant(SessionGrantStateV1::Revoked, 2);
6896 assert!(validate_grant_transition(
6897 &active,
6898 &revoked,
6899 HarnessOperationKindV1::RevokeGrant,
6900 ).is_ok());
6901 assert!(matches!(
6902 validate_grant_transition(
6903 &revoked,
6904 &grant(SessionGrantStateV1::Active, 3),
6905 HarnessOperationKindV1::MutateGrant,
6906 ),
6907 Err(HarnessEngineError::RevokedGrantMutation)
6908 ));
6909 }
6910
6911 #[test]
6912 fn harness_mutation_prevalidates_bounds_before_serialization() {
6913 let id = task_id('a');
6914 let mut unbounded = task(id.clone(), 1, "valid");
6915 unbounded.title = "x".repeat(
6916 hatchery_harness_protocol::HARNESS_TITLE_MAX_BYTES + 1,
6917 );
6918 let mutation = HarnessMutationV1::CreateTask {
6919 operation: task_operation(operation_id('a'), id, 'a', None),
6920 task: unbounded,
6921 };
6922 assert!(matches!(
6923 mutation.validate_payload(),
6924 Err(HarnessEngineError::Validation(
6925 hatchery_harness_protocol::HarnessValidationError::InvalidTitle
6926 ))
6927 ));
6928 }
6929
6930 #[test]
6931 fn harness_entity_mutation_rejects_non_authoritative_operation_state() {
6932 let mut engine = HarnessEngine::new();
6933 let (id, _) = create_task(&mut engine);
6934 let before = engine.checkpoint();
6935 let mut operation = task_operation(
6936 operation_id('b'),
6937 id.clone(),
6938 'b',
6939 Some(revision(1)),
6940 );
6941 operation.state = HarnessOperationStateV1::Prepared;
6942 operation.finished_at_unix_ms = None;
6943 assert!(matches!(
6944 engine.prepare(HarnessMutationV1::ReplaceTask {
6945 operation,
6946 expected_revision: revision(1),
6947 task: task(id, 2, "changed"),
6948 }),
6949 Err(HarnessEngineError::WrongOperationState { .. })
6950 ));
6951 assert_eq!(engine.checkpoint(), before);
6952 }
6953
6954 #[test]
6955 fn harness_run_operation_authority_rejects_generic_dispatch_shortcut() {
6956 let requested = run(HarnessRunLifecycleV1::Requested, 1);
6957 let dispatching = run(HarnessRunLifecycleV1::Dispatching, 2);
6958 let mut generic = task_operation(
6959 operation_id('c'),
6960 task_id('a'),
6961 'c',
6962 Some(revision(1)),
6963 );
6964 generic.kind = HarnessOperationKindV1::MutateRun;
6965 generic.task_id = None;
6966 generic.run_id = Some(run_id());
6967 assert!(matches!(
6968 validate_generic_run_operation(&requested, &dispatching, &generic),
6969 Err(HarnessEngineError::InvalidGenericRunOperation)
6970 ));
6971 generic.kind = HarnessOperationKindV1::BindRun;
6972 generic.state = HarnessOperationStateV1::Succeeded;
6973 assert!(matches!(
6974 validate_generic_run_operation(&requested, &dispatching, &generic),
6975 Err(HarnessEngineError::InvalidGenericRunOperation)
6976 ));
6977
6978 let mut coupled = generic;
6979 coupled.kind = HarnessOperationKindV1::CreateRun;
6980 coupled.state = HarnessOperationStateV1::Dispatching;
6981 assert!(validate_coupled_run_operation(
6982 &requested,
6983 &dispatching,
6984 &coupled,
6985 ).is_ok());
6986 coupled.state = HarnessOperationStateV1::Prepared;
6987 assert!(matches!(
6988 validate_coupled_run_operation(&requested, &dispatching, &coupled),
6989 Err(HarnessEngineError::InvalidCoupledRunOperation)
6990 ));
6991 }
6992
6993 #[test]
6994 fn harness_restore_rejects_orphan_and_incoherent_run_authority() {
6995 let mut linked_task = task(task_id('a'), 1, "linked");
6996 linked_task.run_ids = vec![run_id()];
6997 let requested = run(HarnessRunLifecycleV1::Requested, 1);
6998 let mut orphan = requested.clone();
6999 orphan.operation_id = operation_id('c');
7000 let orphan_checkpoint = HarnessEngineCheckpointV1 {
7001 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
7002 tasks: vec![linked_task.clone()],
7003 runs: vec![orphan],
7004 grants: Vec::new(),
7005 operations: vec![original_run_operation(HarnessOperationStateV1::Prepared)],
7006 execution_specs: Vec::new(),
7007 issuances: Vec::new(),
7008 execution_specs_v2: Vec::new(),
7009 deliveries: Vec::new(),
7010 continuations: Vec::new(),
7011 };
7012 assert!(matches!(
7013 HarnessEngine::restore(orphan_checkpoint),
7014 Err(HarnessEngineError::NotFound(_))
7015 ));
7016
7017 let incoherent_checkpoint = HarnessEngineCheckpointV1 {
7018 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
7019 tasks: vec![linked_task],
7020 runs: vec![requested],
7021 grants: Vec::new(),
7022 operations: vec![original_run_operation(HarnessOperationStateV1::Dispatching)],
7023 execution_specs: Vec::new(),
7024 issuances: Vec::new(),
7025 execution_specs_v2: Vec::new(),
7026 deliveries: Vec::new(),
7027 continuations: Vec::new(),
7028 };
7029 assert!(matches!(
7030 HarnessEngine::restore(incoherent_checkpoint),
7031 Err(HarnessEngineError::RunOperationStateIncoherent { .. })
7032 ));
7033 }
7034
7035 #[test]
7036 fn harness_restore_rejects_missing_parent_run_actor() {
7037 let mut authored = task(task_id('a'), 1, "authored");
7038 authored.creator = HarnessActorV1::ParentRun { run_id: run_id() };
7039 let checkpoint = HarnessEngineCheckpointV1 {
7040 version: HARNESS_ENGINE_CHECKPOINT_VERSION_V1,
7041 tasks: vec![authored],
7042 runs: Vec::new(),
7043 grants: Vec::new(),
7044 operations: Vec::new(),
7045 execution_specs: Vec::new(),
7046 issuances: Vec::new(),
7047 execution_specs_v2: Vec::new(),
7048 deliveries: Vec::new(),
7049 continuations: Vec::new(),
7050 };
7051 assert!(matches!(
7052 HarnessEngine::restore(checkpoint),
7053 Err(HarnessEngineError::NotFound(_))
7054 ));
7055 }
7056
7057 #[test]
7058 fn harness_read_visibility_allows_descendants_and_denies_cross_tree() {
7059 let engine = visibility_fixture();
7060 let visibility = engine.read_visibility(&grant_id()).unwrap();
7061
7062 for run_id in [numbered_run_id(1), numbered_run_id(2), numbered_run_id(3)] {
7063 assert!(visibility.run_visible(&run_id));
7064 }
7065 assert!(!visibility.run_visible(&numbered_run_id(4)));
7066 for task_id in [
7067 numbered_task_id(1),
7068 numbered_task_id(2),
7069 numbered_task_id(3),
7070 numbered_task_id(5),
7071 numbered_task_id(6),
7072 ] {
7073 assert!(visibility.task_visible(&task_id));
7074 }
7075 assert!(!visibility.task_visible(&numbered_task_id(4)));
7076 assert!(!visibility.task_visible(&numbered_task_id(7)));
7077 for operation_id in [
7078 operation_id('1'),
7079 operation_id('2'),
7080 operation_id('5'),
7081 operation_id('7'),
7082 operation_id('8'),
7083 ] {
7084 assert!(visibility.operation_visible(&operation_id));
7085 }
7086 assert!(!visibility.operation_visible(&operation_id('3')));
7087 assert!(!visibility.operation_visible(&operation_id('4')));
7088 assert!(!visibility.operation_visible(&operation_id('6')));
7089 assert!(!visibility.operation_visible(&operation_id('9')));
7090
7091 assert_eq!(visibility.run_ids().count(), 3);
7092 assert_eq!(visibility.task_ids().count(), 5);
7093 assert_eq!(visibility.operation_ids().count(), 5);
7094 }
7095
7096 #[test]
7097 fn harness_read_visibility_self_only_requires_direct_root_attribution() {
7098 let mut engine = visibility_fixture();
7099 engine.grants.get_mut(&grant_id()).unwrap().read_permissions =
7100 HarnessReadPermissionsV1 {
7101 tasks: HarnessEntityReadScopeV1::SelfOnly,
7102 runs: HarnessEntityReadScopeV1::SelfOnly,
7103 operations: HarnessEntityReadScopeV1::SelfOnly,
7104 };
7105
7106 let visibility = engine.read_visibility(&grant_id()).unwrap();
7107 assert!(visibility.run_visible(&numbered_run_id(1)));
7108 assert!(!visibility.run_visible(&numbered_run_id(2)));
7109 assert!(visibility.task_visible(&numbered_task_id(1)));
7110 assert!(visibility.task_visible(&numbered_task_id(6)));
7111 assert!(!visibility.task_visible(&numbered_task_id(2)));
7112 assert!(!visibility.task_visible(&numbered_task_id(5)));
7113 assert!(!visibility.task_visible(&numbered_task_id(7)));
7114 assert!(visibility.operation_visible(&operation_id('1')));
7115 assert!(!visibility.operation_visible(&operation_id('2')));
7116 assert!(!visibility.operation_visible(&operation_id('3')));
7117 assert!(!visibility.operation_visible(&operation_id('5')));
7118 }
7119
7120 #[test]
7121 fn harness_read_visibility_mixed_scopes_do_not_cross_authorize_targets() {
7122 let mut engine = visibility_fixture();
7123 engine.grants.get_mut(&grant_id()).unwrap().read_permissions =
7124 HarnessReadPermissionsV1 {
7125 tasks: HarnessEntityReadScopeV1::SelfOnly,
7126 runs: HarnessEntityReadScopeV1::Descendants,
7127 operations: HarnessEntityReadScopeV1::Descendants,
7128 };
7129
7130 let visibility = engine.read_visibility(&grant_id()).unwrap();
7131 assert!(visibility.run_visible(&numbered_run_id(2)));
7132 assert!(!visibility.task_visible(&numbered_task_id(2)));
7133 assert!(!visibility.operation_visible(&operation_id('2')));
7134 assert!(visibility.operation_visible(&operation_id('5')));
7135
7136 engine.grants.get_mut(&grant_id()).unwrap().read_permissions =
7137 HarnessReadPermissionsV1 {
7138 tasks: HarnessEntityReadScopeV1::Descendants,
7139 runs: HarnessEntityReadScopeV1::SelfOnly,
7140 operations: HarnessEntityReadScopeV1::Descendants,
7141 };
7142 let visibility = engine.read_visibility(&grant_id()).unwrap();
7143 assert!(visibility.task_visible(&numbered_task_id(2)));
7144 assert!(!visibility.run_visible(&numbered_run_id(2)));
7145 assert!(visibility.operation_visible(&operation_id('2')));
7146 assert!(!visibility.operation_visible(&operation_id('5')));
7147 }
7148
7149 #[test]
7150 fn harness_read_visibility_hidden_cross_tree_target_hides_operation() {
7151 let engine = visibility_fixture();
7152 let visibility = engine.read_visibility(&grant_id()).unwrap();
7153
7154 assert!(!visibility.task_visible(&numbered_task_id(4)));
7155 assert!(!visibility.operation_visible(&operation_id('3')));
7156 }
7157
7158 #[test]
7159 fn harness_read_visibility_hides_grant_operations_and_hidden_reconciliations() {
7160 let engine = visibility_fixture();
7161 let visibility = engine.read_visibility(&grant_id()).unwrap();
7162
7163 assert!(!visibility.operation_visible(&operation_id('6')));
7164 assert!(visibility.operation_visible(&operation_id('7')));
7165 assert!(visibility.operation_visible(&operation_id('8')));
7166 assert!(!visibility.operation_visible(&operation_id('4')));
7167 assert!(!visibility.operation_visible(&operation_id('9')));
7168 }
7169
7170 #[test]
7171 fn harness_read_visibility_reconcile_cycle_fails_closed() {
7172 let mut engine = visibility_fixture();
7173 let root_run_id = numbered_run_id(1);
7174 let first = visibility_reconcile_operation(
7175 operation_id('a'),
7176 'a',
7177 root_run_id.clone(),
7178 root_run_id.clone(),
7179 operation_id('b'),
7180 );
7181 let second = visibility_reconcile_operation(
7182 operation_id('b'),
7183 'b',
7184 root_run_id.clone(),
7185 root_run_id,
7186 operation_id('a'),
7187 );
7188 engine.operations.insert(first.operation_id.clone(), first);
7189 engine.operations.insert(second.operation_id.clone(), second);
7190
7191 assert!(matches!(
7192 engine.read_visibility(&grant_id()),
7193 Err(HarnessEngineError::ReadVisibilityInvalidGraph),
7194 ));
7195 }
7196
7197 #[test]
7198 fn harness_read_visibility_missing_reconcile_target_fails_closed() {
7199 let mut engine = visibility_fixture();
7200 let root_run_id = numbered_run_id(1);
7201 let invalid = visibility_reconcile_operation(
7202 operation_id('c'),
7203 'c',
7204 root_run_id.clone(),
7205 root_run_id,
7206 operation_id('d'),
7207 );
7208 engine.operations.insert(invalid.operation_id.clone(), invalid);
7209
7210 assert!(matches!(
7211 engine.read_visibility(&grant_id()),
7212 Err(HarnessEngineError::ReadVisibilityInvalidGraph),
7213 ));
7214 }
7215
7216 #[test]
7217 fn harness_read_visibility_revoked_invalid_and_missing_grants_are_empty() {
7218 let mut engine = visibility_fixture();
7219 assert!(engine.read_visibility(
7220 &SessionGrantId::new(format!("hgrant_{}", "b".repeat(24))).unwrap(),
7221 ).unwrap().is_empty());
7222
7223 let stored = engine.grants.get_mut(&grant_id()).unwrap();
7224 stored.state = SessionGrantStateV1::Revoked;
7225 assert!(engine.read_visibility(&grant_id()).unwrap().is_empty());
7226
7227 let stored = engine.grants.get_mut(&grant_id()).unwrap();
7228 stored.state = SessionGrantStateV1::Active;
7229 stored.task_permissions.read = false;
7230 assert!(engine.read_visibility(&grant_id()).unwrap().is_empty());
7231 }
7232
7233 #[test]
7234 fn harness_read_visibility_cycle_fails_closed() {
7235 let mut engine = visibility_fixture();
7236 engine.runs.get_mut(&numbered_run_id(1)).unwrap().parent_run_id =
7237 Some(numbered_run_id(2));
7238 assert!(matches!(
7239 engine.read_visibility(&grant_id()),
7240 Err(HarnessEngineError::ReadVisibilityInvalidGraph),
7241 ));
7242 }
7243
7244 #[test]
7245 fn harness_read_visibility_depth_fails_closed() {
7246 let mut engine = visibility_fixture();
7247 engine.runs.clear();
7248 let owning_task_id = numbered_task_id(1);
7249 let mut parent = None;
7250 for index in 0..=usize::from(HARNESS_CHILD_DEPTH_MAX) + 1 {
7251 let id = numbered_run_id(index + 1);
7252 let record = visibility_run(id.clone(), parent, owning_task_id.clone());
7253 engine.runs.insert(id.clone(), record);
7254 parent = Some(id);
7255 }
7256 engine.grants.get_mut(&grant_id()).unwrap().actor_run_id = numbered_run_id(1);
7257 assert!(matches!(
7258 engine.read_visibility(&grant_id()),
7259 Err(HarnessEngineError::ReadVisibilityInvalidGraph),
7260 ));
7261 }
7262
7263 #[test]
7282 fn harness_read_visibility_survives_a_store_past_the_old_scan_ceiling() {
7283 let mut engine = visibility_fixture();
7284 engine.tasks.clear();
7285 for index in 0..HARNESS_VISIBILITY_SCAN_MAX {
7286 let id = numbered_task_id(index + 1);
7287 engine.tasks.insert(id.clone(), task(id, 1, "bounded scan"));
7288 }
7289 assert!(
7290 engine.tasks.len() + engine.runs.len() + engine.operations.len()
7291 > HARNESS_VISIBILITY_SCAN_MAX,
7292 );
7293 assert!(engine.read_visibility(&grant_id()).is_ok());
7294 }
7295
7296 #[test]
7297 fn harness_task_subtree_three_level_tree_and_depth_bound_cuts() {
7298 let root_id = numbered_task_id(30);
7305 let child_id = numbered_task_id(20);
7306 let grandchild_id = numbered_task_id(10);
7307 let stranger_id = numbered_task_id(99);
7308
7309 let mut engine = HarnessEngine::new();
7310 let root = task(root_id.clone(), 1, "root");
7311 let mut child = task(child_id.clone(), 1, "child");
7312 child.parent_task_id = Some(root_id.clone());
7313 let mut grandchild = task(grandchild_id.clone(), 1, "grandchild");
7314 grandchild.parent_task_id = Some(child_id.clone());
7315 let stranger = task(stranger_id.clone(), 1, "stranger");
7316 for record in [root, child, grandchild, stranger] {
7317 engine.tasks.insert(record.task_id.clone(), record);
7318 }
7319
7320 let root_subtree = engine.task_subtree(&root_id, HARNESS_CHILD_DEPTH_MAX);
7321 assert_eq!(
7322 root_subtree,
7323 BTreeSet::from([root_id.clone(), child_id.clone(), grandchild_id.clone()]),
7324 );
7325
7326 let leaf_subtree = engine.task_subtree(&grandchild_id, HARNESS_CHILD_DEPTH_MAX);
7327 assert_eq!(leaf_subtree, BTreeSet::from([grandchild_id.clone()]));
7328
7329 assert!(!engine.task_is_strict_descendant(&stranger_id, &root_id));
7330 assert!(engine.task_is_strict_descendant(&child_id, &root_id));
7331 assert!(engine.task_is_strict_descendant(&grandchild_id, &root_id));
7332 assert!(!engine.task_is_strict_descendant(&root_id, &root_id), "a root is not its own strict descendant");
7333
7334 let bounded = engine.task_subtree(&root_id, 0);
7335 assert_eq!(
7336 bounded,
7337 BTreeSet::from([root_id, child_id]),
7338 "a zero depth bound cuts the grandchild generation",
7339 );
7340 }
7341
7342 #[test]
7349 fn harness_task_child_count_and_subtree_depth_observe_two_level_tree() {
7350 let root_id = numbered_task_id(30);
7351 let child_id = numbered_task_id(20);
7352 let grandchild_id = numbered_task_id(10);
7353 let stranger_id = numbered_task_id(99);
7354
7355 let mut engine = HarnessEngine::new();
7356 let root = task(root_id.clone(), 1, "root");
7357 let mut child = task(child_id.clone(), 1, "child");
7358 child.parent_task_id = Some(root_id.clone());
7359 let mut grandchild = task(grandchild_id.clone(), 1, "grandchild");
7360 grandchild.parent_task_id = Some(child_id.clone());
7361 let stranger = task(stranger_id.clone(), 1, "stranger");
7362 for record in [root, child, grandchild, stranger] {
7363 engine.tasks.insert(record.task_id.clone(), record);
7364 }
7365
7366 assert_eq!(engine.task_child_count(&root_id), 1);
7370 assert_eq!(engine.task_subtree_depth(&root_id), 2);
7371
7372 assert_eq!(engine.task_child_count(&child_id), 1);
7373 assert_eq!(engine.task_subtree_depth(&child_id), 1);
7374
7375 assert_eq!(engine.task_child_count(&grandchild_id), 0);
7378 assert_eq!(engine.task_subtree_depth(&grandchild_id), 0);
7379 assert_eq!(engine.task_child_count(&stranger_id), 0);
7380 assert_eq!(engine.task_subtree_depth(&stranger_id), 0);
7381 }
7382
7383 #[test]
7384 fn harness_tasks_attributed_to_runs_includes_a_task_whose_creator_is_that_run() {
7385 let mut engine = HarnessEngine::new();
7386 let attributing_run = run_id();
7387 let mut authored = task(task_id('a'), 1, "authored by its own run");
7388 authored.creator = HarnessActorV1::ParentRun { run_id: attributing_run.clone() };
7389 engine.tasks.insert(authored.task_id.clone(), authored.clone());
7390
7391 let attributed = engine.tasks_attributed_to_runs(&BTreeSet::from([attributing_run]));
7392 assert!(attributed.contains(&authored.task_id));
7393
7394 assert!(engine.tasks_attributed_to_runs(&BTreeSet::new()).is_empty());
7395
7396 let other_run = numbered_run_id(2);
7397 assert!(!engine.tasks_attributed_to_runs(&BTreeSet::from([other_run])).contains(&authored.task_id));
7398 }
7399
7400 #[test]
7401 fn harness_operations_for_task_are_newest_first_bounded_and_store_the_prepared_actor() {
7402 let placeholder_task_id = task_id('e');
7408 let authoring_run_id = run_id();
7409 let replacing_run_id = numbered_run_id(9);
7410 let mut engine = HarnessEngine::new();
7411 let mut placeholder_task = task(placeholder_task_id.clone(), 1, "placeholder");
7412 placeholder_task.run_ids = vec![replacing_run_id.clone(), authoring_run_id.clone()];
7416 engine.tasks.insert(placeholder_task_id.clone(), placeholder_task);
7417
7418 let mut authoring_run = run(HarnessRunLifecycleV1::Requested, 1);
7419 authoring_run.task_id = placeholder_task_id.clone();
7420 engine.runs.insert(authoring_run.run_id.clone(), authoring_run);
7421 let mut authoring_run_operation = original_run_operation(HarnessOperationStateV1::Prepared);
7422 authoring_run_operation.task_id = Some(placeholder_task_id.clone());
7423 engine.operations.insert(operation_id('b'), authoring_run_operation);
7424
7425 let mut replacing_run = run(HarnessRunLifecycleV1::Requested, 1);
7426 replacing_run.run_id = replacing_run_id.clone();
7427 replacing_run.task_id = placeholder_task_id.clone();
7428 replacing_run.operation_id = operation_id('c');
7429 engine.runs.insert(replacing_run_id.clone(), replacing_run);
7430 let mut replacing_run_operation = original_run_operation(HarnessOperationStateV1::Prepared);
7431 replacing_run_operation.operation_id = operation_id('c');
7432 replacing_run_operation.run_id = Some(replacing_run_id.clone());
7433 replacing_run_operation.task_id = Some(placeholder_task_id);
7434 engine.operations.insert(operation_id('c'), replacing_run_operation);
7435
7436 let id = task_id('a');
7437 let mut created_task = task(id.clone(), 1, "authored by its own run");
7438 created_task.creator = HarnessActorV1::ParentRun { run_id: authoring_run_id.clone() };
7439 let mut create_operation = task_operation(operation_id('f'), id.clone(), 'f', None);
7445 create_operation.actor = HarnessActorV1::ParentRun { run_id: authoring_run_id.clone() };
7446 let prepared = engine.prepare(HarnessMutationV1::CreateTask {
7447 operation: create_operation.clone(),
7448 task: created_task,
7449 }).unwrap();
7450 engine.accept(prepared);
7451
7452 let mut replaced_task = task(id.clone(), 2, "authored by its own run, replaced");
7453 replaced_task.creator = HarnessActorV1::ParentRun { run_id: authoring_run_id.clone() };
7454 let mut replace_operation = task_operation(operation_id('a'), id.clone(), 'd', Some(revision(1)));
7455 replace_operation.actor = HarnessActorV1::ParentRun { run_id: replacing_run_id.clone() };
7456 replace_operation.created_at_unix_ms = 20;
7457 replace_operation.updated_at_unix_ms = 22;
7458 replace_operation.finished_at_unix_ms = Some(22);
7459 let prepared = engine.prepare(HarnessMutationV1::ReplaceTask {
7460 operation: replace_operation.clone(),
7461 expected_revision: revision(1),
7462 task: replaced_task,
7463 }).unwrap();
7464 engine.accept(prepared);
7465
7466 let ledger = engine.operations_for_task(&id, 10);
7467 assert_eq!(ledger.len(), 2, "only the two operations naming this task, not the seeded run operations");
7468 assert_eq!(ledger[0].operation_id, replace_operation.operation_id, "newest (highest created_at_unix_ms) first");
7469 assert_eq!(ledger[1].operation_id, create_operation.operation_id);
7470 assert_eq!(ledger[0].actor, HarnessActorV1::ParentRun { run_id: replacing_run_id });
7471 assert_eq!(ledger[1].actor, HarnessActorV1::ParentRun { run_id: authoring_run_id });
7472
7473 let bounded = engine.operations_for_task(&id, 1);
7474 assert_eq!(bounded.len(), 1);
7475 assert_eq!(bounded[0].operation_id, replace_operation.operation_id);
7476 }
7477}