1use std::sync::Arc;
9
10use ag_session::IntegrationApproach;
11use async_trait::async_trait;
12use sqlx::SqlitePool;
13
14use super::status;
15use crate::timestamp::TimestampSource;
16use crate::{DbError, DbResultExt};
17
18#[derive(Clone, Debug, Eq, PartialEq)]
20pub struct SessionOrchestrationRow {
21 pub controller_project_id: i64,
23 pub controller_session_id: String,
25 pub goal_statement: String,
27 pub id: i64,
29 pub max_parallelism: i64,
31 pub relayed_question_task_id: Option<i64>,
33 pub status: String,
35 pub verification_generation: i64,
37}
38
39#[derive(Clone, Debug, Eq, PartialEq)]
41pub struct SessionOrchestrationTaskRow {
42 pub acceptance_criteria: String,
44 pub area_violations: String,
46 pub areas_compliant: Option<bool>,
48 pub attempt_count: i64,
50 pub child_added_lines: i64,
52 pub child_answer: Option<String>,
54 pub child_deleted_lines: i64,
56 pub child_focused_review_status: Option<String>,
58 pub child_focused_review_text: Option<String>,
60 pub child_has_diff: Option<bool>,
62 pub child_input_tokens: i64,
64 pub child_output_tokens: i64,
66 pub child_questions: Option<String>,
68 pub child_session_id: Option<String>,
70 pub child_status: Option<String>,
72 pub child_summary: Option<String>,
74 pub continuation_generation: i64,
76 pub continuation_prompt: Option<String>,
78 pub id: i64,
80 pub infrastructure_retry_count: i64,
82 pub kind: String,
84 pub last_error: Option<String>,
86 pub merge_position: i64,
88 pub prompt: String,
90 pub research_report: Option<String>,
92 pub result_summary: Option<String>,
94 pub review_iteration: i64,
96 pub status: String,
98 pub task_key: String,
100 pub touched_areas: String,
102 pub title: String,
104 pub verification_reason: Option<String>,
106 pub verification_verdict: Option<String>,
108}
109
110#[derive(Clone, Debug, Eq, PartialEq)]
112pub struct OrchestrationTaskScopeRow {
113 pub base_branch: String,
115 pub id: i64,
117 pub touched_areas: String,
119}
120
121#[derive(Clone, Debug, Eq, PartialEq)]
123pub struct SessionOrchestrationMetadataRow {
124 pub controller_session_id: Option<String>,
126 pub orchestration_status: Option<String>,
129 pub running_task_count: i64,
131 pub session_id: String,
133 pub waiting_task_count: i64,
135}
136
137pub struct PersistedOrchestrationTask {
144 pub acceptance_criteria: String,
146 pub kind: String,
148 pub merge_position: i64,
150 pub prompt: String,
152 pub session_orchestration_id: i64,
154 pub task_key: String,
156 pub title: String,
158 pub touched_areas: String,
160}
161
162#[cfg_attr(any(test, feature = "test-utils"), mockall::automock)]
167#[async_trait]
168pub trait OrchestrationRepository: Send + Sync {
169 async fn insert_orchestration(
171 &self,
172 controller_session_id: &str,
173 status: &str,
174 max_parallelism: i64,
175 ) -> Result<i64, DbError>;
176
177 async fn upsert_orchestration_task(
185 &self,
186 task: PersistedOrchestrationTask,
187 ) -> Result<i64, DbError>;
188
189 async fn load_orchestration_for_controller(
191 &self,
192 controller_session_id: &str,
193 ) -> Result<Option<SessionOrchestrationRow>, DbError>;
194
195 async fn load_active_orchestrations(&self) -> Result<Vec<SessionOrchestrationRow>, DbError>;
197
198 async fn load_recoverable_focused_review_session_ids(
201 &self,
202 project_id: i64,
203 ) -> Result<Vec<String>, DbError>;
204
205 async fn load_session_metadata_for_project(
208 &self,
209 project_id: i64,
210 ) -> Result<Vec<SessionOrchestrationMetadataRow>, DbError>;
211
212 async fn load_orchestration_tasks(
214 &self,
215 session_orchestration_id: i64,
216 ) -> Result<Vec<SessionOrchestrationTaskRow>, DbError>;
217
218 async fn load_orchestration_integration_approach(&self, id: i64) -> Result<String, DbError>;
220
221 async fn load_orchestration_task_scope_for_child(
223 &self,
224 child_session_id: &str,
225 ) -> Result<Option<OrchestrationTaskScopeRow>, DbError>;
226
227 async fn load_child_session_id_for_task(&self, task_id: i64)
229 -> Result<Option<String>, DbError>;
230
231 async fn begin_orchestration_cancellation(&self, id: i64) -> Result<bool, DbError>;
233
234 async fn claim_orchestration_task(&self, id: i64) -> Result<bool, DbError>;
236
237 async fn claim_orchestration_review_application(
240 &self,
241 id: i64,
242 prompt: &str,
243 iteration_limit: i64,
244 ) -> Result<bool, DbError>;
245
246 async fn claim_orchestration_rollup(&self, id: i64) -> Result<bool, DbError>;
248
249 async fn complete_orchestration_rollup(&self, id: i64) -> Result<bool, DbError>;
251
252 async fn record_orchestration_verdict(
254 &self,
255 id: i64,
256 task_key: &str,
257 is_pass: bool,
258 reason: &str,
259 ) -> Result<bool, DbError>;
260
261 async fn complete_orchestration_campaign(&self, id: i64) -> Result<bool, DbError>;
263
264 async fn load_rollup_operation_status(
266 &self,
267 operation_id: &str,
268 ) -> Result<Option<String>, DbError>;
269
270 async fn update_orchestration_status(&self, id: i64, status: &str) -> Result<(), DbError>;
272
273 async fn approve_orchestration_plan(&self, id: i64) -> Result<bool, DbError>;
275
276 async fn approve_orchestration_integration(
278 &self,
279 id: i64,
280 approach: IntegrationApproach,
281 ) -> Result<bool, DbError>;
282
283 async fn update_orchestration_plan(
285 &self,
286 id: i64,
287 goal_statement: &str,
288 max_parallelism: i64,
289 ) -> Result<(), DbError>;
290
291 async fn queue_orchestration_continuation(
293 &self,
294 id: i64,
295 prompt: &str,
296 acceptance_criteria: &str,
297 touched_areas: &str,
298 ) -> Result<bool, DbError>;
299
300 async fn reset_orchestration_verification(&self, id: i64) -> Result<(), DbError>;
302
303 async fn record_orchestration_spawn_failure(
305 &self,
306 id: i64,
307 error: &str,
308 retry_limit: i64,
309 ) -> Result<String, DbError>;
310
311 async fn detach_orchestration_child(&self, child_session_id: &str) -> Result<bool, DbError>;
313
314 async fn surface_orchestration_questions(
317 &self,
318 session_orchestration_id: i64,
319 task_id: i64,
320 questions: &str,
321 ) -> Result<bool, DbError>;
322
323 async fn clear_orchestration_questions(
325 &self,
326 session_orchestration_id: i64,
327 ) -> Result<(), DbError>;
328
329 async fn link_orchestration_task_child(
331 &self,
332 id: i64,
333 child_session_id: &str,
334 ) -> Result<bool, DbError>;
335
336 async fn update_orchestration_task_status(
338 &self,
339 id: i64,
340 status: &str,
341 last_error: Option<String>,
342 ) -> Result<(), DbError>;
343
344 async fn update_orchestration_task_result_summary(
346 &self,
347 id: i64,
348 result_summary: &str,
349 ) -> Result<(), DbError>;
350
351 async fn update_orchestration_task_research_report(
354 &self,
355 id: i64,
356 research_report: &str,
357 ) -> Result<(), DbError>;
358
359 async fn update_orchestration_task_area_compliance(
361 &self,
362 id: i64,
363 areas_compliant: Option<bool>,
364 area_violations: &str,
365 ) -> Result<(), DbError>;
366}
367
368#[derive(Clone)]
370pub(crate) struct SqliteOrchestrationRepository(SqlitePool, Arc<dyn TimestampSource>);
371
372struct OrchestrationTransition {
373 context: &'static str,
374 from_orchestration_status: &'static str,
375 from_task_status: &'static str,
376 require_pass_verdict: bool,
377 to_orchestration_status: &'static str,
378 to_task_status: &'static str,
379}
380
381impl SqliteOrchestrationRepository {
382 pub(crate) fn new(pool: SqlitePool, timestamp_source: Arc<dyn TimestampSource>) -> Self {
384 Self(pool, timestamp_source)
385 }
386
387 fn now(&self) -> i64 {
389 self.1.now_timestamp_seconds()
390 }
391
392 async fn transition_orchestration_and_tasks(
393 &self,
394 id: i64,
395 transition: OrchestrationTransition,
396 ) -> Result<bool, DbError> {
397 let now = self.now();
398 let mut transaction = self.0.begin().await.db_context(transition.context)?;
399 let result = sqlx::query(
400 r"
401UPDATE session_orchestration
402SET status = ?,
403 updated_at = ?
404WHERE id = ?
405 AND status = ?
406",
407 )
408 .bind(transition.to_orchestration_status)
409 .bind(now)
410 .bind(id)
411 .bind(transition.from_orchestration_status)
412 .execute(&mut *transaction)
413 .await
414 .db_context(transition.context)?;
415 if result.rows_affected() == 1 {
416 sqlx::query(
417 r"
418UPDATE session_orchestration_task
419SET status = ?,
420 updated_at = ?
421WHERE session_orchestration_id = ?
422 AND status = ?
423 AND (? = 0 OR verification_verdict = 'Pass')
424",
425 )
426 .bind(transition.to_task_status)
427 .bind(now)
428 .bind(id)
429 .bind(transition.from_task_status)
430 .bind(i64::from(transition.require_pass_verdict))
431 .execute(&mut *transaction)
432 .await
433 .db_context(transition.context)?;
434 }
435 transaction.commit().await.db_context(transition.context)?;
436
437 Ok(result.rows_affected() == 1)
438 }
439}
440
441#[async_trait]
442impl OrchestrationRepository for SqliteOrchestrationRepository {
443 async fn insert_orchestration(
444 &self,
445 controller_session_id: &str,
446 status: &str,
447 max_parallelism: i64,
448 ) -> Result<i64, DbError> {
449 status::validate_orchestration(status)?;
450 let now = self.now();
451
452 let row = sqlx::query!(
453 r#"
454INSERT INTO session_orchestration (
455 controller_session_id,
456 goal_statement,
457 status,
458 max_parallelism,
459 created_at,
460 updated_at
461)
462VALUES (?, '', ?, ?, ?, ?)
463RETURNING id AS "id!: i64"
464"#,
465 controller_session_id,
466 status,
467 max_parallelism,
468 now,
469 now
470 )
471 .fetch_one(&self.0)
472 .await?;
473
474 Ok(row.id)
475 }
476
477 async fn upsert_orchestration_task(
478 &self,
479 task: PersistedOrchestrationTask,
480 ) -> Result<i64, DbError> {
481 let PersistedOrchestrationTask {
482 acceptance_criteria,
483 kind,
484 merge_position,
485 prompt,
486 session_orchestration_id,
487 task_key,
488 title,
489 touched_areas,
490 } = task;
491 kind.parse::<ag_session::OrchestrationTaskKind>()
492 .map_err(|_| DbError::InvalidData {
493 entity: "orchestration task kind",
494 reason: format!("unknown persisted kind `{kind}`"),
495 })?;
496 let now = self.now();
497 let mut transaction = self
498 .0
499 .begin()
500 .await
501 .db_context("upsert orchestration task")?;
502
503 let row = sqlx::query!(
504 r#"
505INSERT INTO session_orchestration_task (
506 session_orchestration_id,
507 task_key,
508 title,
509 prompt,
510 touched_areas,
511 acceptance_criteria,
512 kind,
513 merge_position,
514 status,
515 created_at,
516 updated_at
517)
518VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'Planned', ?, ?)
519ON CONFLICT(session_orchestration_id, task_key) DO UPDATE
520SET title = excluded.title,
521 prompt = excluded.prompt,
522 kind = excluded.kind,
523 touched_areas = excluded.touched_areas,
524 acceptance_criteria = excluded.acceptance_criteria,
525 merge_position = excluded.merge_position,
526 status = 'Planned',
527 child_session_id = NULL,
528 continuation_prompt = NULL,
529 review_iteration = 0,
530 research_report = NULL,
531 result_summary = NULL,
532 verification_reason = NULL,
533 verification_verdict = NULL,
534 last_error = NULL,
535 updated_at = excluded.updated_at
536RETURNING id AS "id!: i64"
537"#,
538 session_orchestration_id,
539 task_key,
540 title,
541 prompt,
542 touched_areas,
543 acceptance_criteria,
544 kind,
545 merge_position,
546 now,
547 now
548 )
549 .fetch_one(&mut *transaction)
550 .await
551 .db_context("upsert orchestration task")?;
552
553 sqlx::query!(
554 r"
555UPDATE session
556SET orchestration_task_id = NULL,
557 updated_at = ?
558WHERE orchestration_task_id = ?
559",
560 now,
561 row.id
562 )
563 .execute(&mut *transaction)
564 .await
565 .db_context("upsert orchestration task")?;
566
567 transaction
568 .commit()
569 .await
570 .db_context("upsert orchestration task")?;
571
572 Ok(row.id)
573 }
574
575 async fn load_orchestration_for_controller(
576 &self,
577 controller_session_id: &str,
578 ) -> Result<Option<SessionOrchestrationRow>, DbError> {
579 let row = sqlx::query_as!(
580 SessionOrchestrationRow,
581 r#"
582SELECT orchestration.id AS "id!: i64",
583 session.project_id AS "controller_project_id!: i64",
584 orchestration.controller_session_id,
585 orchestration.goal_statement,
586 orchestration.relayed_question_task_id,
587 orchestration.status,
588 orchestration.max_parallelism,
589 orchestration.verification_generation
590FROM session_orchestration AS orchestration
591INNER JOIN session
592ON session.id = orchestration.controller_session_id
593WHERE orchestration.controller_session_id = ?
594ORDER BY orchestration.id DESC
595LIMIT 1
596"#,
597 controller_session_id
598 )
599 .fetch_optional(&self.0)
600 .await?;
601 if let Some(row) = &row {
602 status::validate_orchestration(&row.status)?;
603 }
604
605 Ok(row)
606 }
607
608 async fn load_active_orchestrations(&self) -> Result<Vec<SessionOrchestrationRow>, DbError> {
609 let rows = sqlx::query_as!(
610 SessionOrchestrationRow,
611 r#"
612SELECT orchestration.id AS "id!: i64",
613 session.project_id AS "controller_project_id!: i64",
614 orchestration.controller_session_id,
615 orchestration.goal_statement,
616 orchestration.relayed_question_task_id,
617 orchestration.status,
618 orchestration.max_parallelism,
619 orchestration.verification_generation
620FROM session_orchestration AS orchestration
621INNER JOIN session
622ON session.id = orchestration.controller_session_id
623WHERE orchestration.status IN (
624 'AwaitingApproval',
625 'Running',
626 'Verifying',
627 'AwaitingIntegration',
628 'Integrating',
629 'Canceling'
630)
631ORDER BY orchestration.id
632"#
633 )
634 .fetch_all(&self.0)
635 .await?;
636 for row in &rows {
637 status::validate_orchestration(&row.status)?;
638 }
639
640 Ok(rows)
641 }
642
643 async fn load_recoverable_focused_review_session_ids(
644 &self,
645 project_id: i64,
646 ) -> Result<Vec<String>, DbError> {
647 let session_ids = sqlx::query_scalar!(
648 r#"
649SELECT child.id AS "id!: String"
650FROM session_orchestration_task AS task
651INNER JOIN session_orchestration AS orchestration
652ON orchestration.id = task.session_orchestration_id
653INNER JOIN session AS child
654ON child.id = task.child_session_id
655WHERE child.project_id = ?
656 AND orchestration.status IN ('AwaitingApproval', 'Running')
657 AND task.status = 'Reviewing'
658 AND child.status IN ('Review', 'AgentReview')
659 AND (
660 child.focused_review_status IS NULL
661 OR child.focused_review_status = 'Pending'
662 )
663ORDER BY task.id
664"#,
665 project_id
666 )
667 .fetch_all(&self.0)
668 .await?;
669
670 Ok(session_ids)
671 }
672
673 async fn load_session_metadata_for_project(
674 &self,
675 project_id: i64,
676 ) -> Result<Vec<SessionOrchestrationMetadataRow>, DbError> {
677 let rows = sqlx::query_as!(
678 SessionOrchestrationMetadataRow,
679 r#"
680WITH latest_orchestration_id AS (
681 SELECT controller_session_id,
682 MAX(id) AS orchestration_id
683 FROM session_orchestration
684 GROUP BY controller_session_id
685),
686latest_orchestration AS (
687 SELECT orchestration.id,
688 orchestration.controller_session_id,
689 orchestration.status
690 FROM session_orchestration AS orchestration
691 INNER JOIN latest_orchestration_id AS latest
692 ON latest.orchestration_id = orchestration.id
693),
694controller_metadata AS (
695 SELECT orchestration.controller_session_id AS session_id,
696 orchestration.status AS orchestration_status,
697 COALESCE(SUM(
698 CASE WHEN task.status IN (
699 'Creating',
700 'Running',
701 'Reviewing',
702 'ReviewApplying',
703 'ContinuationPending'
704 )
705 THEN 1 ELSE 0 END
706 ), 0) AS running_task_count,
707 COALESCE(SUM(
708 CASE WHEN task.status = 'WaitingForInput' THEN 1 ELSE 0 END
709 ), 0) AS waiting_task_count
710 FROM latest_orchestration AS orchestration
711 LEFT JOIN session_orchestration_task AS task
712 ON task.session_orchestration_id = orchestration.id
713 GROUP BY orchestration.id
714),
715child_metadata AS (
716 SELECT task.child_session_id AS session_id,
717 orchestration.controller_session_id
718 FROM session_orchestration_task AS task
719 INNER JOIN session_orchestration AS orchestration
720 ON orchestration.id = task.session_orchestration_id
721 WHERE task.child_session_id IS NOT NULL
722)
723SELECT session.id AS "session_id!: String",
724 child_metadata.controller_session_id,
725 controller_metadata.orchestration_status,
726 COALESCE(controller_metadata.running_task_count, 0) AS "running_task_count!: i64",
727 COALESCE(controller_metadata.waiting_task_count, 0) AS "waiting_task_count!: i64"
728FROM session
729LEFT JOIN controller_metadata
730ON controller_metadata.session_id = session.id
731LEFT JOIN child_metadata
732ON child_metadata.session_id = session.id
733WHERE session.project_id = ?
734 AND (
735 controller_metadata.session_id IS NOT NULL
736 OR child_metadata.session_id IS NOT NULL
737 )
738ORDER BY session.id
739"#,
740 project_id
741 )
742 .fetch_all(&self.0)
743 .await?;
744 for row in &rows {
745 if let Some(orchestration_status) = &row.orchestration_status {
746 status::validate_orchestration(orchestration_status)?;
747 }
748 }
749
750 Ok(rows)
751 }
752
753 async fn load_orchestration_tasks(
754 &self,
755 session_orchestration_id: i64,
756 ) -> Result<Vec<SessionOrchestrationTaskRow>, DbError> {
757 let rows = sqlx::query_as!(
758 SessionOrchestrationTaskRow,
759 r#"
760SELECT task.id AS "id!: i64",
761 task.acceptance_criteria,
762 task.area_violations,
763 task.areas_compliant AS "areas_compliant?: bool",
764 task.attempt_count,
765 COALESCE(child.added_lines, 0) AS "child_added_lines!: i64",
766 (
767 SELECT message.content
768 FROM session_message AS message
769 WHERE message.session_id = task.child_session_id
770 AND message.kind = 'assistant_answer'
771 ORDER BY message.position DESC
772 LIMIT 1
773 ) AS child_answer,
774 COALESCE(child.deleted_lines, 0) AS "child_deleted_lines!: i64",
775 child.focused_review_status AS child_focused_review_status,
776 child.focused_review_text AS child_focused_review_text,
777 child.has_diff AS "child_has_diff?: bool",
778 COALESCE(child.input_tokens, 0) AS "child_input_tokens!: i64",
779 COALESCE(child.output_tokens, 0) AS "child_output_tokens!: i64",
780 task.child_session_id,
781 child.status AS child_status,
782 child.questions AS child_questions,
783 child.summary AS child_summary,
784 task.continuation_generation,
785 task.continuation_prompt,
786 task.infrastructure_retry_count,
787 task.kind,
788 task.last_error,
789 task.merge_position,
790 task.prompt,
791 task.research_report,
792 task.result_summary,
793 task.review_iteration,
794 task.status,
795 task.task_key,
796 task.touched_areas,
797 task.title,
798 task.verification_reason,
799 task.verification_verdict
800FROM session_orchestration_task AS task
801LEFT JOIN session AS child
802ON child.id = task.child_session_id
803WHERE task.session_orchestration_id = ?
804ORDER BY task.merge_position, task.id
805"#,
806 session_orchestration_id
807 )
808 .fetch_all(&self.0)
809 .await?;
810 for row in &rows {
811 status::validate_orchestration_task(&row.status)?;
812 row.kind
813 .parse::<ag_session::OrchestrationTaskKind>()
814 .map_err(|_| DbError::InvalidData {
815 entity: "orchestration task kind",
816 reason: format!("unknown persisted kind `{}`", row.kind),
817 })?;
818 if let Some(child_status) = &row.child_status {
819 status::validate_session(child_status)?;
820 }
821 }
822
823 Ok(rows)
824 }
825
826 async fn load_orchestration_integration_approach(&self, id: i64) -> Result<String, DbError> {
827 sqlx::query_scalar::<_, String>(
828 "SELECT integration_approach FROM session_orchestration WHERE id = ?",
829 )
830 .bind(id)
831 .fetch_one(&self.0)
832 .await
833 .db_context("load orchestration integration approach")
834 }
835
836 async fn load_orchestration_task_scope_for_child(
837 &self,
838 child_session_id: &str,
839 ) -> Result<Option<OrchestrationTaskScopeRow>, DbError> {
840 sqlx::query_as!(
841 OrchestrationTaskScopeRow,
842 r#"
843SELECT child.base_branch,
844 task.id AS "id!: i64",
845 task.touched_areas
846FROM session_orchestration_task AS task
847INNER JOIN session AS child
848ON child.id = task.child_session_id
849WHERE child.id = ?
850 AND child.role = 'OrchestrationWorker'
851 AND task.kind = 'Implementation'
852"#,
853 child_session_id
854 )
855 .fetch_optional(&self.0)
856 .await
857 .db_context("load orchestration task scope for child")
858 }
859
860 async fn load_child_session_id_for_task(
861 &self,
862 task_id: i64,
863 ) -> Result<Option<String>, DbError> {
864 let row = sqlx::query!(
865 r#"
866SELECT id AS "id!: String"
867FROM session
868WHERE orchestration_task_id = ?
869"#,
870 task_id
871 )
872 .fetch_optional(&self.0)
873 .await?;
874
875 Ok(row.map(|row| row.id))
876 }
877
878 async fn begin_orchestration_cancellation(&self, id: i64) -> Result<bool, DbError> {
879 let now = self.now();
880
881 let result = sqlx::query!(
882 r"
883UPDATE session_orchestration
884SET status = 'Canceling',
885 updated_at = ?
886WHERE id = ?
887 AND status IN (
888 'AwaitingApproval',
889 'Running',
890 'Verifying',
891 'AwaitingIntegration',
892 'Integrating',
893 'Canceling'
894 )
895",
896 now,
897 id
898 )
899 .execute(&self.0)
900 .await
901 .db_context("begin orchestration cancellation")?;
902
903 Ok(result.rows_affected() == 1)
904 }
905
906 async fn claim_orchestration_task(&self, id: i64) -> Result<bool, DbError> {
907 let now = self.now();
908
909 let result = sqlx::query!(
910 r"
911UPDATE session_orchestration_task
912SET status = 'Creating',
913 last_error = NULL,
914 updated_at = ?
915WHERE id = ?
916 AND status = 'Planned'
917 AND EXISTS (
918 SELECT 1
919 FROM session_orchestration
920 WHERE session_orchestration.id = session_orchestration_task.session_orchestration_id
921 AND session_orchestration.status = 'Running'
922 )
923",
924 now,
925 id
926 )
927 .execute(&self.0)
928 .await
929 .db_context("claim orchestration task")?;
930
931 Ok(result.rows_affected() == 1)
932 }
933
934 async fn claim_orchestration_review_application(
935 &self,
936 id: i64,
937 prompt: &str,
938 iteration_limit: i64,
939 ) -> Result<bool, DbError> {
940 let now = self.now();
941 let mut transaction = self
942 .0
943 .begin()
944 .await
945 .db_context("claim orchestration review application")?;
946 let claim = sqlx::query!(
947 r"
948UPDATE session_orchestration_task
949SET continuation_generation = continuation_generation + 1,
950 continuation_prompt = ?,
951 review_iteration = review_iteration + 1,
952 status = 'ReviewApplying',
953 updated_at = ?
954WHERE id = ?
955 AND status = 'Reviewing'
956 AND review_iteration < ?
957 AND child_session_id IS NOT NULL
958",
959 prompt,
960 now,
961 id,
962 iteration_limit
963 )
964 .execute(&mut *transaction)
965 .await
966 .db_context("claim orchestration review application")?;
967 if claim.rows_affected() == 0 {
968 transaction
969 .rollback()
970 .await
971 .db_context("claim orchestration review application")?;
972
973 return Ok(false);
974 }
975
976 sqlx::query!(
977 r"
978UPDATE session
979SET focused_review_status = NULL,
980 focused_review_diff_hash = NULL,
981 focused_review_text = NULL,
982 updated_at = ?
983WHERE id = (
984 SELECT child_session_id
985 FROM session_orchestration_task
986 WHERE id = ?
987)
988",
989 now,
990 id
991 )
992 .execute(&mut *transaction)
993 .await
994 .db_context("claim orchestration review application")?;
995 transaction
996 .commit()
997 .await
998 .db_context("claim orchestration review application")?;
999
1000 Ok(true)
1001 }
1002
1003 async fn claim_orchestration_rollup(&self, id: i64) -> Result<bool, DbError> {
1004 let now = self.now();
1005
1006 let result = sqlx::query!(
1007 r"
1008UPDATE session_orchestration
1009SET status = 'Verifying',
1010 verification_generation = verification_generation + 1,
1011 updated_at = ?
1012WHERE id = ?
1013 AND status = 'Running'
1014",
1015 now,
1016 id
1017 )
1018 .execute(&self.0)
1019 .await
1020 .db_context("claim orchestration rollup")?;
1021
1022 Ok(result.rows_affected() == 1)
1023 }
1024
1025 async fn complete_orchestration_rollup(&self, id: i64) -> Result<bool, DbError> {
1026 self.transition_orchestration_and_tasks(
1027 id,
1028 OrchestrationTransition {
1029 context: "complete orchestration rollup",
1030 from_orchestration_status: "Verifying",
1031 from_task_status: "Ready",
1032 require_pass_verdict: true,
1033 to_orchestration_status: "AwaitingIntegration",
1034 to_task_status: "AwaitingIntegration",
1035 },
1036 )
1037 .await
1038 }
1039
1040 async fn record_orchestration_verdict(
1041 &self,
1042 id: i64,
1043 task_key: &str,
1044 is_pass: bool,
1045 reason: &str,
1046 ) -> Result<bool, DbError> {
1047 let now = self.now();
1048 let verdict = if is_pass { "Pass" } else { "Flag" };
1049 let result = sqlx::query!(
1050 r"
1051UPDATE session_orchestration_task
1052SET verification_reason = ?,
1053 verification_verdict = ?,
1054 updated_at = ?
1055WHERE session_orchestration_id = ?
1056 AND task_key = ?
1057 AND status IN ('Ready', 'Reported')
1058 AND EXISTS (
1059 SELECT 1
1060 FROM session_orchestration
1061 WHERE id = ?
1062 AND status = 'Verifying'
1063 )
1064",
1065 reason,
1066 verdict,
1067 now,
1068 id,
1069 task_key,
1070 id
1071 )
1072 .execute(&self.0)
1073 .await
1074 .db_context("record orchestration verdict")?;
1075
1076 Ok(result.rows_affected() == 1)
1077 }
1078
1079 async fn complete_orchestration_campaign(&self, id: i64) -> Result<bool, DbError> {
1080 let now = self.now();
1081 let mut transaction = self
1082 .0
1083 .begin()
1084 .await
1085 .db_context("complete orchestration campaign")?;
1086 let orchestration = sqlx::query!(
1087 r#"
1088UPDATE session_orchestration
1089SET status = 'Done',
1090 updated_at = ?
1091WHERE id = ?
1092 AND status IN ('AwaitingIntegration', 'Integrating')
1093RETURNING controller_session_id AS "controller_session_id!: String"
1094"#,
1095 now,
1096 id
1097 )
1098 .fetch_optional(&mut *transaction)
1099 .await
1100 .db_context("complete orchestration campaign")?;
1101 if let Some(orchestration) = &orchestration {
1102 sqlx::query!(
1103 r"
1104UPDATE session
1105SET status = 'Done',
1106 questions = '',
1107 updated_at = ?
1108WHERE id = ?
1109 AND role = 'Orchestrator'
1110 AND status IN ('Review', 'Question')
1111",
1112 now,
1113 orchestration.controller_session_id
1114 )
1115 .execute(&mut *transaction)
1116 .await
1117 .db_context("complete orchestration campaign")?;
1118 }
1119 transaction
1120 .commit()
1121 .await
1122 .db_context("complete orchestration campaign")?;
1123
1124 Ok(orchestration.is_some())
1125 }
1126
1127 async fn load_rollup_operation_status(
1128 &self,
1129 operation_id: &str,
1130 ) -> Result<Option<String>, DbError> {
1131 let row = sqlx::query!(
1132 r#"
1133SELECT status AS "status!: String"
1134FROM session_operation
1135WHERE id = ?
1136"#,
1137 operation_id
1138 )
1139 .fetch_optional(&self.0)
1140 .await?;
1141 if let Some(row) = &row {
1142 status::validate_operation(&row.status)?;
1143 }
1144
1145 Ok(row.map(|row| row.status))
1146 }
1147
1148 async fn update_orchestration_status(&self, id: i64, status: &str) -> Result<(), DbError> {
1149 status::validate_orchestration(status)?;
1150 let now = self.now();
1151
1152 sqlx::query!(
1153 r"
1154UPDATE session_orchestration
1155SET status = ?,
1156 updated_at = ?
1157WHERE id = ?
1158",
1159 status,
1160 now,
1161 id
1162 )
1163 .execute(&self.0)
1164 .await?;
1165
1166 Ok(())
1167 }
1168
1169 async fn approve_orchestration_plan(&self, id: i64) -> Result<bool, DbError> {
1170 self.transition_orchestration_and_tasks(
1171 id,
1172 OrchestrationTransition {
1173 context: "approve orchestration plan",
1174 from_orchestration_status: "AwaitingApproval",
1175 from_task_status: "Proposed",
1176 require_pass_verdict: false,
1177 to_orchestration_status: "Running",
1178 to_task_status: "Planned",
1179 },
1180 )
1181 .await
1182 }
1183
1184 async fn approve_orchestration_integration(
1185 &self,
1186 id: i64,
1187 approach: IntegrationApproach,
1188 ) -> Result<bool, DbError> {
1189 let approach = approach.to_string();
1190 let now = self.now();
1191 let result = sqlx::query(
1192 r"
1193UPDATE session_orchestration
1194SET integration_approach = ?,
1195 status = 'Integrating',
1196 updated_at = ?
1197WHERE id = ?
1198 AND status = 'AwaitingIntegration'
1199",
1200 )
1201 .bind(approach)
1202 .bind(now)
1203 .bind(id)
1204 .execute(&self.0)
1205 .await
1206 .db_context("approve orchestration integration")?;
1207
1208 Ok(result.rows_affected() == 1)
1209 }
1210
1211 async fn update_orchestration_plan(
1212 &self,
1213 id: i64,
1214 goal_statement: &str,
1215 max_parallelism: i64,
1216 ) -> Result<(), DbError> {
1217 let now = self.now();
1218
1219 sqlx::query!(
1220 r"
1221UPDATE session_orchestration
1222SET goal_statement = ?,
1223 max_parallelism = ?,
1224 updated_at = ?
1225WHERE id = ?
1226 AND status = 'AwaitingApproval'
1227",
1228 goal_statement,
1229 max_parallelism,
1230 now,
1231 id
1232 )
1233 .execute(&self.0)
1234 .await?;
1235
1236 Ok(())
1237 }
1238
1239 async fn queue_orchestration_continuation(
1240 &self,
1241 id: i64,
1242 prompt: &str,
1243 acceptance_criteria: &str,
1244 touched_areas: &str,
1245 ) -> Result<bool, DbError> {
1246 let now = self.now();
1247 let mut transaction = self
1248 .0
1249 .begin()
1250 .await
1251 .db_context("queue orchestration continuation")?;
1252 let result = sqlx::query!(
1253 r"
1254UPDATE session_orchestration_task
1255SET acceptance_criteria = ?,
1256 area_violations = '[]',
1257 areas_compliant = NULL,
1258 continuation_generation = continuation_generation + 1,
1259 continuation_prompt = ?,
1260 review_iteration = 0,
1261 status = 'ContinuationPending',
1262 touched_areas = ?,
1263 result_summary = NULL,
1264 verification_verdict = NULL,
1265 verification_reason = NULL,
1266 last_error = NULL,
1267 updated_at = ?
1268WHERE id = ?
1269 AND child_session_id IS NOT NULL
1270 AND status IN ('Ready', 'AwaitingIntegration', 'IntegrationFailed')
1271",
1272 acceptance_criteria,
1273 prompt,
1274 touched_areas,
1275 now,
1276 id
1277 )
1278 .execute(&mut *transaction)
1279 .await?;
1280
1281 if result.rows_affected() == 1 {
1282 sqlx::query!(
1283 r"
1284UPDATE session
1285SET focused_review_status = NULL,
1286 focused_review_diff_hash = NULL,
1287 focused_review_text = NULL,
1288 updated_at = ?
1289WHERE id = (
1290 SELECT child_session_id
1291 FROM session_orchestration_task
1292 WHERE id = ?
1293)
1294",
1295 now,
1296 id
1297 )
1298 .execute(&mut *transaction)
1299 .await?;
1300 }
1301 transaction.commit().await?;
1302
1303 Ok(result.rows_affected() == 1)
1304 }
1305
1306 async fn reset_orchestration_verification(&self, id: i64) -> Result<(), DbError> {
1307 let now = self.now();
1308
1309 sqlx::query!(
1310 r"
1311UPDATE session_orchestration_task
1312SET status = 'Ready',
1313 verification_reason = NULL,
1314 verification_verdict = NULL,
1315 updated_at = ?
1316WHERE session_orchestration_id = ?
1317 AND status = 'AwaitingIntegration'
1318",
1319 now,
1320 id
1321 )
1322 .execute(&self.0)
1323 .await
1324 .db_context("reset orchestration verification")?;
1325
1326 Ok(())
1327 }
1328
1329 async fn record_orchestration_spawn_failure(
1330 &self,
1331 id: i64,
1332 error: &str,
1333 retry_limit: i64,
1334 ) -> Result<String, DbError> {
1335 let now = self.now();
1336 let mut transaction = self
1337 .0
1338 .begin()
1339 .await
1340 .db_context("record orchestration spawn failure")?;
1341 sqlx::query!(
1342 r"
1343UPDATE session
1344SET orchestration_task_id = NULL,
1345 updated_at = ?
1346WHERE orchestration_task_id = ?
1347",
1348 now,
1349 id
1350 )
1351 .execute(&mut *transaction)
1352 .await
1353 .db_context("record orchestration spawn failure")?;
1354 let row = sqlx::query!(
1355 r#"
1356UPDATE session_orchestration_task
1357SET child_session_id = NULL,
1358 infrastructure_retry_count = infrastructure_retry_count + 1,
1359 status = CASE
1360 WHEN infrastructure_retry_count < ? THEN 'Planned'
1361 ELSE 'Failed'
1362 END,
1363 last_error = ?,
1364 updated_at = ?
1365WHERE id = ?
1366RETURNING status AS "status!: String"
1367"#,
1368 retry_limit,
1369 error,
1370 now,
1371 id
1372 )
1373 .fetch_one(&mut *transaction)
1374 .await
1375 .db_context("record orchestration spawn failure")?;
1376 transaction
1377 .commit()
1378 .await
1379 .db_context("record orchestration spawn failure")?;
1380 status::validate_orchestration_task(&row.status)?;
1381
1382 Ok(row.status)
1383 }
1384
1385 async fn detach_orchestration_child(&self, child_session_id: &str) -> Result<bool, DbError> {
1386 let now = self.now();
1387 let mut transaction = self
1388 .0
1389 .begin()
1390 .await
1391 .db_context("detach orchestration child")?;
1392 let task = sqlx::query!(
1393 r#"
1394SELECT orchestration_task_id AS "task_id!: i64"
1395FROM session
1396WHERE id = ?
1397 AND role = 'OrchestrationWorker'
1398 AND orchestration_task_id IS NOT NULL
1399"#,
1400 child_session_id
1401 )
1402 .fetch_optional(&mut *transaction)
1403 .await
1404 .db_context("detach orchestration child")?;
1405 let Some(task) = task else {
1406 transaction
1407 .commit()
1408 .await
1409 .db_context("detach orchestration child")?;
1410
1411 return Ok(false);
1412 };
1413
1414 sqlx::query!(
1415 r"
1416UPDATE session_orchestration_task
1417SET child_session_id = NULL,
1418 status = 'Detached',
1419 updated_at = ?
1420WHERE id = ?
1421",
1422 now,
1423 task.task_id
1424 )
1425 .execute(&mut *transaction)
1426 .await
1427 .db_context("detach orchestration child")?;
1428 sqlx::query!(
1429 r"
1430UPDATE session
1431SET orchestration_task_id = NULL,
1432 role = 'Worker',
1433 updated_at = ?
1434WHERE id = ?
1435",
1436 now,
1437 child_session_id
1438 )
1439 .execute(&mut *transaction)
1440 .await
1441 .db_context("detach orchestration child")?;
1442 transaction
1443 .commit()
1444 .await
1445 .db_context("detach orchestration child")?;
1446
1447 Ok(true)
1448 }
1449
1450 async fn surface_orchestration_questions(
1451 &self,
1452 session_orchestration_id: i64,
1453 task_id: i64,
1454 questions: &str,
1455 ) -> Result<bool, DbError> {
1456 let now = self.now();
1457 let mut transaction = self
1458 .0
1459 .begin()
1460 .await
1461 .db_context("surface orchestration questions")?;
1462 let claim = sqlx::query!(
1463 r"
1464UPDATE session_orchestration
1465SET relayed_question_task_id = ?,
1466 updated_at = ?
1467WHERE id = ?
1468 AND relayed_question_task_id IS NULL
1469 AND EXISTS (
1470 SELECT 1
1471 FROM session_orchestration_task AS task
1472 WHERE task.id = ?
1473 AND task.session_orchestration_id = session_orchestration.id
1474 AND task.status = 'WaitingForInput'
1475 AND task.child_session_id IS NOT NULL
1476 )
1477 AND EXISTS (
1478 SELECT 1
1479 FROM session AS controller
1480 WHERE controller.id = session_orchestration.controller_session_id
1481 AND controller.role = 'Orchestrator'
1482 AND controller.status IN ('Review', 'Question')
1483 AND COALESCE(controller.questions, '') = ''
1484 )
1485",
1486 task_id,
1487 now,
1488 session_orchestration_id,
1489 task_id
1490 )
1491 .execute(&mut *transaction)
1492 .await
1493 .db_context("surface orchestration questions")?;
1494 if claim.rows_affected() == 0 {
1495 transaction
1496 .rollback()
1497 .await
1498 .db_context("surface orchestration questions")?;
1499
1500 return Ok(false);
1501 }
1502 sqlx::query!(
1503 r"
1504UPDATE session
1505SET questions = ?,
1506 status = 'Question',
1507 updated_at = ?
1508WHERE id = (
1509 SELECT controller_session_id
1510 FROM session_orchestration
1511 WHERE id = ?
1512)
1513",
1514 questions,
1515 now,
1516 session_orchestration_id
1517 )
1518 .execute(&mut *transaction)
1519 .await
1520 .db_context("surface orchestration questions")?;
1521 transaction
1522 .commit()
1523 .await
1524 .db_context("surface orchestration questions")?;
1525
1526 Ok(true)
1527 }
1528
1529 async fn clear_orchestration_questions(
1530 &self,
1531 session_orchestration_id: i64,
1532 ) -> Result<(), DbError> {
1533 let now = self.now();
1534 let mut transaction = self
1535 .0
1536 .begin()
1537 .await
1538 .db_context("clear orchestration questions")?;
1539 sqlx::query!(
1540 r"
1541UPDATE session
1542SET questions = '',
1543 status = 'Review',
1544 updated_at = ?
1545WHERE id = (
1546 SELECT controller_session_id
1547 FROM session_orchestration
1548 WHERE id = ?
1549 AND relayed_question_task_id IS NOT NULL
1550)
1551 AND role = 'Orchestrator'
1552 AND status = 'Question'
1553",
1554 now,
1555 session_orchestration_id
1556 )
1557 .execute(&mut *transaction)
1558 .await
1559 .db_context("clear orchestration questions")?;
1560 sqlx::query!(
1561 r"
1562UPDATE session_orchestration
1563SET relayed_question_task_id = NULL,
1564 updated_at = ?
1565WHERE id = ?
1566 AND relayed_question_task_id IS NOT NULL
1567",
1568 now,
1569 session_orchestration_id
1570 )
1571 .execute(&mut *transaction)
1572 .await
1573 .db_context("clear orchestration questions")?;
1574 transaction
1575 .commit()
1576 .await
1577 .db_context("clear orchestration questions")?;
1578
1579 Ok(())
1580 }
1581
1582 async fn link_orchestration_task_child(
1583 &self,
1584 id: i64,
1585 child_session_id: &str,
1586 ) -> Result<bool, DbError> {
1587 let now = self.now();
1588
1589 let result = sqlx::query!(
1590 r"
1591UPDATE session_orchestration_task
1592SET child_session_id = ?,
1593 status = 'Running',
1594 attempt_count = attempt_count + 1,
1595 updated_at = ?
1596WHERE id = ?
1597 AND status = 'Creating'
1598 AND EXISTS (
1599 SELECT 1
1600 FROM session_orchestration
1601 WHERE session_orchestration.id = session_orchestration_task.session_orchestration_id
1602 AND session_orchestration.status = 'Running'
1603 )
1604",
1605 child_session_id,
1606 now,
1607 id
1608 )
1609 .execute(&self.0)
1610 .await?;
1611
1612 Ok(result.rows_affected() == 1)
1613 }
1614
1615 async fn update_orchestration_task_status(
1616 &self,
1617 id: i64,
1618 status: &str,
1619 last_error: Option<String>,
1620 ) -> Result<(), DbError> {
1621 status::validate_orchestration_task(status)?;
1622 let now = self.now();
1623
1624 sqlx::query!(
1625 r"
1626UPDATE session_orchestration_task
1627SET status = ?,
1628 last_error = ?,
1629 updated_at = ?
1630WHERE id = ?
1631",
1632 status,
1633 last_error,
1634 now,
1635 id
1636 )
1637 .execute(&self.0)
1638 .await?;
1639
1640 Ok(())
1641 }
1642
1643 async fn update_orchestration_task_result_summary(
1644 &self,
1645 id: i64,
1646 result_summary: &str,
1647 ) -> Result<(), DbError> {
1648 let now = self.now();
1649
1650 sqlx::query!(
1651 r"
1652UPDATE session_orchestration_task
1653SET result_summary = ?,
1654 updated_at = ?
1655WHERE id = ?
1656",
1657 result_summary,
1658 now,
1659 id
1660 )
1661 .execute(&self.0)
1662 .await?;
1663
1664 Ok(())
1665 }
1666
1667 async fn update_orchestration_task_research_report(
1668 &self,
1669 id: i64,
1670 research_report: &str,
1671 ) -> Result<(), DbError> {
1672 let now = self.now();
1673
1674 sqlx::query!(
1675 r"
1676UPDATE session_orchestration_task
1677SET research_report = ?,
1678 updated_at = ?
1679WHERE id = ?
1680 AND kind = 'Research'
1681",
1682 research_report,
1683 now,
1684 id
1685 )
1686 .execute(&self.0)
1687 .await?;
1688
1689 Ok(())
1690 }
1691
1692 async fn update_orchestration_task_area_compliance(
1693 &self,
1694 id: i64,
1695 areas_compliant: Option<bool>,
1696 area_violations: &str,
1697 ) -> Result<(), DbError> {
1698 let now = self.now();
1699
1700 sqlx::query!(
1701 r"
1702UPDATE session_orchestration_task
1703SET area_violations = ?,
1704 areas_compliant = ?,
1705 updated_at = ?
1706WHERE id = ?
1707",
1708 area_violations,
1709 areas_compliant,
1710 now,
1711 id
1712 )
1713 .execute(&self.0)
1714 .await?;
1715
1716 Ok(())
1717 }
1718}
1719
1720#[cfg(test)]
1721mod tests {
1722 use ag_agent::{AgentKind, ReasoningLevel, SpeedMode};
1723 use ag_session::{
1724 FocusedReviewStatus, OrchestrationStatus, OrchestrationTaskKind, OrchestrationTaskStatus,
1725 SessionMessageKind,
1726 };
1727
1728 use super::*;
1729 use crate::{AppRepositories, PersistedSessionCreation, SessionRow};
1730
1731 async fn controller_fixture() -> AppRepositories {
1734 controller_fixture_with_pool().await.0
1735 }
1736
1737 async fn controller_fixture_with_pool() -> (AppRepositories, SqlitePool) {
1738 let (database, pool) = AppRepositories::in_memory_with_pool()
1739 .await
1740 .expect("db should open");
1741 let project_id = database
1742 .projects()
1743 .upsert_project("/tmp/project", None)
1744 .await
1745 .expect("failed to upsert project");
1746 database
1747 .sessions()
1748 .insert_session_with_agent(PersistedSessionCreation {
1749 agent: "codex",
1750 base_branch: "main",
1751 id: "controller",
1752 is_draft: false,
1753 model: AgentKind::Codex.default_model().as_str(),
1754 orchestration_task_id: None,
1755 parent_session_id: None,
1756 permission_mode: ag_agent::PermissionMode::AutoEdit,
1757 personality_id: None,
1758 project_id,
1759 reasoning_level: ReasoningLevel::default(),
1760 role: Some("Orchestrator"),
1761 speed_mode: SpeedMode::Normal,
1762 status: "Review",
1763 })
1764 .await
1765 .expect("failed to insert controller session");
1766
1767 (database, pool)
1768 }
1769
1770 #[tokio::test]
1771 async fn hydration_rejects_unknown_orchestration_status() {
1772 let (database, pool) = AppRepositories::in_memory_with_pool()
1774 .await
1775 .expect("db should open");
1776 let project_id = database
1777 .projects()
1778 .upsert_project("/tmp/invalid-orchestration", None)
1779 .await
1780 .expect("failed to upsert project");
1781 database
1782 .sessions()
1783 .insert_session("controller", "gpt-5.6-sol", "main", "Draft", project_id)
1784 .await
1785 .expect("failed to insert controller session");
1786 let orchestration_id = database
1787 .orchestrations()
1788 .insert_orchestration("controller", "Running", 1)
1789 .await
1790 .expect("failed to insert orchestration");
1791 sqlx::query("UPDATE session_orchestration SET status = 'Unknown' WHERE id = ?")
1792 .bind(orchestration_id)
1793 .execute(&pool)
1794 .await
1795 .expect("failed to corrupt orchestration status");
1796
1797 let error = database
1799 .orchestrations()
1800 .load_orchestration_for_controller("controller")
1801 .await
1802 .expect_err("invalid status should fail hydration");
1803
1804 assert!(matches!(
1806 error,
1807 DbError::InvalidStatus {
1808 entity: "orchestration",
1809 value,
1810 } if value == "Unknown"
1811 ));
1812 }
1813
1814 #[tokio::test]
1815 async fn rollup_recovery_failures_report_semantic_operation_context() {
1816 let (database, pool) = AppRepositories::in_memory_with_pool()
1818 .await
1819 .expect("db should open");
1820 sqlx::query("DROP TABLE session_orchestration")
1821 .execute(&pool)
1822 .await
1823 .expect("failed to drop orchestration table");
1824
1825 let claim_error = database
1827 .orchestrations()
1828 .claim_orchestration_rollup(1)
1829 .await
1830 .expect_err("claim should fail");
1831 let completion_error = database
1832 .orchestrations()
1833 .complete_orchestration_rollup(1)
1834 .await
1835 .expect_err("completion should fail");
1836
1837 assert!(matches!(
1839 claim_error,
1840 DbError::QueryContext {
1841 operation: "claim orchestration rollup",
1842 ..
1843 }
1844 ));
1845 assert!(matches!(
1846 completion_error,
1847 DbError::QueryContext {
1848 operation: "complete orchestration rollup",
1849 ..
1850 }
1851 ));
1852 }
1853
1854 fn planned_task(session_orchestration_id: i64, task_key: &str) -> PersistedOrchestrationTask {
1856 PersistedOrchestrationTask {
1857 acceptance_criteria: format!(r#"["Complete {task_key}"]"#),
1858 kind: "Implementation".to_string(),
1859 merge_position: 0,
1860 prompt: format!("Complete {task_key}"),
1861 session_orchestration_id,
1862 task_key: task_key.to_string(),
1863 title: format!("Task {task_key}"),
1864 touched_areas: format!(r#"["crates/{task_key}/"]"#),
1865 }
1866 }
1867
1868 async fn insert_orchestration_child(
1870 database: &AppRepositories,
1871 project_id: i64,
1872 session_id: &str,
1873 task_id: i64,
1874 ) {
1875 database
1876 .sessions()
1877 .insert_session_with_agent(PersistedSessionCreation {
1878 agent: "codex",
1879 base_branch: "main",
1880 id: session_id,
1881 is_draft: false,
1882 model: AgentKind::Codex.default_model().as_str(),
1883 orchestration_task_id: Some(task_id),
1884 parent_session_id: None,
1885 permission_mode: ag_agent::PermissionMode::AutoEdit,
1886 personality_id: None,
1887 project_id,
1888 reasoning_level: ReasoningLevel::default(),
1889 role: Some("OrchestrationWorker"),
1890 speed_mode: SpeedMode::Normal,
1891 status: "Review",
1892 })
1893 .await
1894 .expect("failed to insert orchestration child");
1895 }
1896
1897 async fn insert_waiting_orchestration_task(
1898 database: &AppRepositories,
1899 project_id: i64,
1900 session_orchestration_id: i64,
1901 task_key: &str,
1902 child_session_id: &str,
1903 ) -> i64 {
1904 let task_id = database
1905 .orchestrations()
1906 .upsert_orchestration_task(planned_task(session_orchestration_id, task_key))
1907 .await
1908 .expect("failed to insert waiting task");
1909 assert!(
1910 database
1911 .orchestrations()
1912 .claim_orchestration_task(task_id)
1913 .await
1914 .expect("failed to claim waiting task")
1915 );
1916 insert_orchestration_child(database, project_id, child_session_id, task_id).await;
1917 assert!(
1918 database
1919 .orchestrations()
1920 .link_orchestration_task_child(task_id, child_session_id)
1921 .await
1922 .expect("failed to link waiting child")
1923 );
1924 database
1925 .orchestrations()
1926 .update_orchestration_task_status(
1927 task_id,
1928 &OrchestrationTaskStatus::WaitingForInput.to_string(),
1929 None,
1930 )
1931 .await
1932 .expect("failed to wait for child question");
1933
1934 task_id
1935 }
1936
1937 async fn surface_and_clear_orchestration_questions(
1938 database: &AppRepositories,
1939 session_orchestration_id: i64,
1940 task_id: i64,
1941 questions: &str,
1942 ) -> bool {
1943 let surfaced = database
1944 .orchestrations()
1945 .surface_orchestration_questions(session_orchestration_id, task_id, questions)
1946 .await
1947 .expect("failed to surface child question");
1948 database
1949 .orchestrations()
1950 .clear_orchestration_questions(session_orchestration_id)
1951 .await
1952 .expect("failed to clear child question");
1953
1954 surfaced
1955 }
1956
1957 async fn load_detached_campaign_state(
1958 database: &AppRepositories,
1959 orchestration_id: i64,
1960 ) -> (SessionOrchestrationTaskRow, SessionRow, SessionRow) {
1961 let task = database
1962 .orchestrations()
1963 .load_orchestration_tasks(orchestration_id)
1964 .await
1965 .expect("failed to load detached task")
1966 .remove(0);
1967 let child = database
1968 .sessions()
1969 .load_session("child-alpha")
1970 .await
1971 .expect("failed to load detached child")
1972 .expect("detached child should exist");
1973 let controller = database
1974 .sessions()
1975 .load_session("controller")
1976 .await
1977 .expect("failed to load controller")
1978 .expect("controller should exist");
1979
1980 (task, child, controller)
1981 }
1982
1983 #[tokio::test]
1984 async fn test_planned_orchestration_round_trips_before_any_child_exists() {
1987 let database = controller_fixture().await;
1989
1990 let orchestration_id = database
1992 .orchestrations()
1993 .insert_orchestration(
1994 "controller",
1995 &OrchestrationStatus::AwaitingApproval.to_string(),
1996 3,
1997 )
1998 .await
1999 .expect("failed to insert orchestration");
2000 database
2001 .orchestrations()
2002 .upsert_orchestration_task(planned_task(orchestration_id, "alpha"))
2003 .await
2004 .expect("failed to insert task");
2005 let orchestration = database
2006 .orchestrations()
2007 .load_orchestration_for_controller("controller")
2008 .await
2009 .expect("failed to load orchestration")
2010 .expect("orchestration should exist");
2011 let tasks = database
2012 .orchestrations()
2013 .load_orchestration_tasks(orchestration_id)
2014 .await
2015 .expect("failed to load tasks");
2016
2017 assert_eq!(orchestration.controller_project_id, 1);
2019 assert_eq!(orchestration.controller_session_id, "controller");
2020 assert_eq!(orchestration.max_parallelism, 3);
2021 assert_eq!(
2022 orchestration.status,
2023 OrchestrationStatus::AwaitingApproval.to_string()
2024 );
2025 assert_eq!(tasks.len(), 1);
2026 assert_eq!(tasks[0].task_key, "alpha");
2027 assert_eq!(tasks[0].child_session_id, None);
2028 assert_eq!(tasks[0].attempt_count, 0);
2029 assert_eq!(
2030 tasks[0].kind,
2031 OrchestrationTaskKind::Implementation.to_string()
2032 );
2033 assert_eq!(tasks[0].touched_areas, r#"["crates/alpha/"]"#);
2034 assert!(tasks[0].child_answer.is_none());
2035 assert!(tasks[0].research_report.is_none());
2036 }
2037
2038 #[tokio::test]
2039 async fn research_task_round_trips_report_and_latest_child_answer_without_scope() {
2040 let database = controller_fixture().await;
2042 let orchestration_id = database
2043 .orchestrations()
2044 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
2045 .await
2046 .expect("failed to insert orchestration");
2047 let task_id = database
2048 .orchestrations()
2049 .upsert_orchestration_task(PersistedOrchestrationTask {
2050 kind: OrchestrationTaskKind::Research.to_string(),
2051 touched_areas: "[]".to_string(),
2052 ..planned_task(orchestration_id, "architecture")
2053 })
2054 .await
2055 .expect("failed to insert research task");
2056 assert!(
2057 database
2058 .orchestrations()
2059 .claim_orchestration_task(task_id)
2060 .await
2061 .expect("failed to claim research task")
2062 );
2063 database
2064 .sessions()
2065 .insert_session_with_agent(PersistedSessionCreation {
2066 agent: "codex",
2067 base_branch: "main",
2068 id: "research-child",
2069 is_draft: false,
2070 model: AgentKind::Codex.default_model().as_str(),
2071 orchestration_task_id: Some(task_id),
2072 parent_session_id: None,
2073 permission_mode: ag_agent::PermissionMode::AutoEdit,
2074 personality_id: None,
2075 project_id: 1,
2076 reasoning_level: ReasoningLevel::default(),
2077 role: Some("OrchestrationResearcher"),
2078 speed_mode: SpeedMode::Normal,
2079 status: "Review",
2080 })
2081 .await
2082 .expect("failed to insert research child");
2083 assert!(
2084 database
2085 .orchestrations()
2086 .link_orchestration_task_child(task_id, "research-child")
2087 .await
2088 .expect("failed to link research child")
2089 );
2090 database
2091 .sessions()
2092 .append_session_message(
2093 "research-child",
2094 SessionMessageKind::AssistantAnswer,
2095 "Full architecture report",
2096 )
2097 .await
2098 .expect("failed to persist research answer");
2099
2100 database
2102 .orchestrations()
2103 .update_orchestration_task_research_report(task_id, "Full architecture report")
2104 .await
2105 .expect("failed to persist research report");
2106 let task = database
2107 .orchestrations()
2108 .load_orchestration_tasks(orchestration_id)
2109 .await
2110 .expect("failed to load research task")
2111 .remove(0);
2112 let scope = database
2113 .orchestrations()
2114 .load_orchestration_task_scope_for_child("research-child")
2115 .await
2116 .expect("failed to inspect research child scope");
2117
2118 assert_eq!(task.kind, OrchestrationTaskKind::Research.to_string());
2120 assert_eq!(
2121 task.child_answer.as_deref(),
2122 Some("Full architecture report")
2123 );
2124 assert_eq!(
2125 task.research_report.as_deref(),
2126 Some("Full architecture report")
2127 );
2128 assert!(scope.is_none());
2129 }
2130
2131 #[tokio::test]
2132 async fn orchestration_task_kind_validation_rejects_unknown_writes_and_hydration() {
2133 let (database, pool) = controller_fixture_with_pool().await;
2135 let orchestration_id = database
2136 .orchestrations()
2137 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
2138 .await
2139 .expect("failed to insert orchestration");
2140 let invalid_task = PersistedOrchestrationTask {
2141 kind: "Unknown".to_string(),
2142 ..planned_task(orchestration_id, "invalid-write")
2143 };
2144
2145 let write_error = database
2147 .orchestrations()
2148 .upsert_orchestration_task(invalid_task)
2149 .await
2150 .expect_err("unknown task kind should be rejected before persistence");
2151 sqlx::query(
2152 "INSERT INTO session_orchestration_task (session_orchestration_id, task_key, title, \
2153 prompt, status, kind) VALUES (?, 'invalid-read', 'Invalid', 'Inspect', 'Planned', \
2154 'Unknown')",
2155 )
2156 .bind(orchestration_id)
2157 .execute(&pool)
2158 .await
2159 .expect("failed to seed invalid persisted kind");
2160 let read_error = database
2161 .orchestrations()
2162 .load_orchestration_tasks(orchestration_id)
2163 .await
2164 .expect_err("unknown persisted task kind should fail hydration");
2165
2166 assert!(matches!(
2168 write_error,
2169 DbError::InvalidData {
2170 entity: "orchestration task kind",
2171 reason,
2172 } if reason == "unknown persisted kind `Unknown`"
2173 ));
2174 assert!(matches!(
2175 read_error,
2176 DbError::InvalidData {
2177 entity: "orchestration task kind",
2178 reason,
2179 } if reason == "unknown persisted kind `Unknown`"
2180 ));
2181 }
2182
2183 #[tokio::test]
2184 async fn test_retry_with_same_task_key_updates_the_existing_row() {
2187 let database = controller_fixture().await;
2189 let project_id = database
2190 .projects()
2191 .upsert_project("/tmp/project", None)
2192 .await
2193 .expect("failed to load project");
2194 let orchestration_id = database
2195 .orchestrations()
2196 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
2197 .await
2198 .expect("failed to insert orchestration");
2199 let first_id = database
2200 .orchestrations()
2201 .upsert_orchestration_task(planned_task(orchestration_id, "alpha"))
2202 .await
2203 .expect("failed to insert task");
2204 let first_claim = database
2205 .orchestrations()
2206 .claim_orchestration_task(first_id)
2207 .await
2208 .expect("failed to claim first attempt");
2209 insert_orchestration_child(&database, project_id, "child-old", first_id).await;
2210 let first_link = database
2211 .orchestrations()
2212 .link_orchestration_task_child(first_id, "child-old")
2213 .await
2214 .expect("failed to link old child");
2215 database
2216 .orchestrations()
2217 .update_orchestration_task_status(
2218 first_id,
2219 &OrchestrationTaskStatus::Failed.to_string(),
2220 Some("agent crashed".to_string()),
2221 )
2222 .await
2223 .expect("failed to fail task");
2224
2225 let retried_id = database
2227 .orchestrations()
2228 .upsert_orchestration_task(PersistedOrchestrationTask {
2229 title: "Task alpha, retried".to_string(),
2230 ..planned_task(orchestration_id, "alpha")
2231 })
2232 .await
2233 .expect("failed to retry task");
2234 let detached_child = database
2235 .orchestrations()
2236 .load_child_session_id_for_task(first_id)
2237 .await
2238 .expect("failed to check detached child");
2239 let retry_claim = database
2240 .orchestrations()
2241 .claim_orchestration_task(retried_id)
2242 .await
2243 .expect("failed to claim replacement attempt");
2244 insert_orchestration_child(&database, project_id, "child-replacement", retried_id).await;
2245 let replacement_link = database
2246 .orchestrations()
2247 .link_orchestration_task_child(retried_id, "child-replacement")
2248 .await
2249 .expect("failed to link replacement child");
2250 let linked_child = database
2251 .orchestrations()
2252 .load_child_session_id_for_task(first_id)
2253 .await
2254 .expect("failed to check replacement child");
2255 let tasks = database
2256 .orchestrations()
2257 .load_orchestration_tasks(orchestration_id)
2258 .await
2259 .expect("failed to load tasks");
2260
2261 assert!(first_claim);
2263 assert!(first_link);
2264 assert!(retry_claim);
2265 assert!(replacement_link);
2266 assert_eq!(retried_id, first_id);
2267 assert_eq!(tasks.len(), 1);
2268 assert_eq!(tasks[0].title, "Task alpha, retried");
2269 assert_eq!(
2270 tasks[0].status,
2271 OrchestrationTaskStatus::Running.to_string()
2272 );
2273 assert_eq!(tasks[0].attempt_count, 2);
2274 assert_eq!(
2275 tasks[0].child_session_id.as_deref(),
2276 Some("child-replacement")
2277 );
2278 assert_eq!(tasks[0].last_error, None);
2279 assert_eq!(detached_child, None);
2280 assert_eq!(linked_child.as_deref(), Some("child-replacement"));
2281 }
2282
2283 #[tokio::test]
2284 async fn test_child_linkage_counts_attempts_and_loads_observed_state() {
2287 let database = controller_fixture().await;
2289 let project_id = database
2290 .projects()
2291 .upsert_project("/tmp/project", None)
2292 .await
2293 .expect("failed to upsert project");
2294 database
2295 .sessions()
2296 .insert_session("child-a", "gpt-5.6-sol", "main", "Draft", project_id)
2297 .await
2298 .expect("failed to insert child session");
2299 let orchestration_id = database
2300 .orchestrations()
2301 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
2302 .await
2303 .expect("failed to insert orchestration");
2304 let task_id = database
2305 .orchestrations()
2306 .upsert_orchestration_task(planned_task(orchestration_id, "alpha"))
2307 .await
2308 .expect("failed to insert task");
2309 let claimed = database
2310 .orchestrations()
2311 .claim_orchestration_task(task_id)
2312 .await
2313 .expect("failed to claim task");
2314
2315 let linked = database
2317 .orchestrations()
2318 .link_orchestration_task_child(task_id, "child-a")
2319 .await
2320 .expect("failed to link child");
2321 database
2322 .orchestrations()
2323 .update_orchestration_task_result_summary(task_id, "Added the parser")
2324 .await
2325 .expect("failed to record summary");
2326 let task = database
2327 .orchestrations()
2328 .load_orchestration_tasks(orchestration_id)
2329 .await
2330 .expect("failed to load task")
2331 .remove(0);
2332
2333 assert!(claimed);
2335 assert!(linked);
2336 assert_eq!(task.id, task_id);
2337 assert_eq!(task.attempt_count, 1);
2338 assert_eq!(task.child_session_id.as_deref(), Some("child-a"));
2339 assert_eq!(task.result_summary.as_deref(), Some("Added the parser"));
2340 assert_eq!(task.status, OrchestrationTaskStatus::Running.to_string());
2341 assert_eq!(task.child_status.as_deref(), Some("Draft"));
2342 assert_eq!(task.child_summary, None);
2343 assert_eq!(task.child_input_tokens, 0);
2344 assert_eq!(task.child_output_tokens, 0);
2345 }
2346
2347 #[tokio::test]
2348 async fn test_active_orchestration_load_includes_recoverable_states() {
2351 let database = controller_fixture().await;
2353 let running_id = database
2354 .orchestrations()
2355 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
2356 .await
2357 .expect("failed to insert running orchestration");
2358 let submitting_id = database
2359 .orchestrations()
2360 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
2361 .await
2362 .expect("failed to insert submitting orchestration");
2363 let canceling_id = database
2364 .orchestrations()
2365 .insert_orchestration("controller", &OrchestrationStatus::Canceling.to_string(), 2)
2366 .await
2367 .expect("failed to insert canceling orchestration");
2368 let settled_id = database
2369 .orchestrations()
2370 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
2371 .await
2372 .expect("failed to insert settled orchestration");
2373
2374 database
2376 .orchestrations()
2377 .update_orchestration_status(settled_id, &OrchestrationStatus::Done.to_string())
2378 .await
2379 .expect("failed to settle orchestration");
2380 let first_claim = database
2381 .orchestrations()
2382 .claim_orchestration_rollup(submitting_id)
2383 .await
2384 .expect("failed to claim roll-up");
2385 let duplicate_claim = database
2386 .orchestrations()
2387 .claim_orchestration_rollup(submitting_id)
2388 .await
2389 .expect("failed to repeat roll-up claim");
2390 let active = database
2391 .orchestrations()
2392 .load_active_orchestrations()
2393 .await
2394 .expect("failed to load active orchestrations");
2395
2396 assert!(first_claim);
2398 assert!(!duplicate_claim);
2399 assert_eq!(
2400 active.iter().map(|row| row.id).collect::<Vec<_>>(),
2401 vec![running_id, submitting_id, canceling_id]
2402 );
2403 assert_eq!(active[1].status, OrchestrationStatus::Verifying.to_string());
2404 assert_eq!(active[1].verification_generation, 1);
2405 assert_eq!(active[2].status, OrchestrationStatus::Canceling.to_string());
2406 }
2407
2408 #[tokio::test]
2409 async fn test_cancellation_barrier_blocks_fan_out_claims_and_links() {
2412 let database = controller_fixture().await;
2414 let project_id = database
2415 .projects()
2416 .upsert_project("/tmp/project", None)
2417 .await
2418 .expect("failed to reload project");
2419 let orchestration_id = database
2420 .orchestrations()
2421 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
2422 .await
2423 .expect("failed to insert orchestration");
2424 let late_task_id = database
2425 .orchestrations()
2426 .upsert_orchestration_task(planned_task(orchestration_id, "late"))
2427 .await
2428 .expect("failed to insert late task");
2429 let claimed_task_id = database
2430 .orchestrations()
2431 .upsert_orchestration_task(planned_task(orchestration_id, "claimed"))
2432 .await
2433 .expect("failed to insert claimed task");
2434 let initial_claim = database
2435 .orchestrations()
2436 .claim_orchestration_task(claimed_task_id)
2437 .await
2438 .expect("failed to claim task before cancellation");
2439 insert_orchestration_child(&database, project_id, "child-claimed", claimed_task_id).await;
2440
2441 let cancellation_started = database
2443 .orchestrations()
2444 .begin_orchestration_cancellation(orchestration_id)
2445 .await
2446 .expect("failed to begin cancellation");
2447 let cancellation_retried = database
2448 .orchestrations()
2449 .begin_orchestration_cancellation(orchestration_id)
2450 .await
2451 .expect("failed to retry cancellation");
2452 let late_claim = database
2453 .orchestrations()
2454 .claim_orchestration_task(late_task_id)
2455 .await
2456 .expect("failed to inspect late claim");
2457 let late_link = database
2458 .orchestrations()
2459 .link_orchestration_task_child(claimed_task_id, "child-claimed")
2460 .await
2461 .expect("failed to inspect late link");
2462 let rollup_completion = database
2463 .orchestrations()
2464 .complete_orchestration_rollup(orchestration_id)
2465 .await
2466 .expect("failed to inspect late roll-up completion");
2467 let orchestration = database
2468 .orchestrations()
2469 .load_orchestration_for_controller("controller")
2470 .await
2471 .expect("failed to load orchestration")
2472 .expect("orchestration should exist");
2473 assert!(initial_claim);
2475 assert!(cancellation_started);
2476 assert!(cancellation_retried);
2477 assert!(!late_claim);
2478 assert!(!late_link);
2479 assert!(!rollup_completion);
2480 assert_eq!(
2481 orchestration.status,
2482 OrchestrationStatus::Canceling.to_string()
2483 );
2484 }
2485
2486 #[tokio::test]
2487 async fn test_rollup_operation_status_controls_orchestration_completion() {
2490 let database = controller_fixture().await;
2492 let orchestration_id = database
2493 .orchestrations()
2494 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
2495 .await
2496 .expect("failed to insert orchestration");
2497 let claimed = database
2498 .orchestrations()
2499 .claim_orchestration_rollup(orchestration_id)
2500 .await
2501 .expect("failed to claim roll-up");
2502 assert!(claimed);
2503 let operation_id = format!("orchestration-rollup-{orchestration_id}-1");
2504
2505 let missing_status = database
2507 .orchestrations()
2508 .load_rollup_operation_status(&operation_id)
2509 .await
2510 .expect("failed to load missing operation");
2511 database
2512 .operations()
2513 .claim_session_operation(&operation_id, "controller", "reply")
2514 .await
2515 .expect("failed to claim operation");
2516 let queued_status = database
2517 .orchestrations()
2518 .load_rollup_operation_status(&operation_id)
2519 .await
2520 .expect("failed to load queued operation");
2521 database
2522 .operations()
2523 .mark_session_operation_running(&operation_id)
2524 .await
2525 .expect("failed to run operation");
2526 let running_status = database
2527 .orchestrations()
2528 .load_rollup_operation_status(&operation_id)
2529 .await
2530 .expect("failed to load running operation");
2531 database
2532 .operations()
2533 .mark_session_operation_failed(&operation_id, "turn failed")
2534 .await
2535 .expect("failed to fail operation");
2536 let failed_status = database
2537 .orchestrations()
2538 .load_rollup_operation_status(&operation_id)
2539 .await
2540 .expect("failed to load failed operation");
2541 database
2542 .operations()
2543 .claim_session_operation(&operation_id, "controller", "reply")
2544 .await
2545 .expect("failed to reclaim operation");
2546 database
2547 .operations()
2548 .mark_session_operation_done(&operation_id)
2549 .await
2550 .expect("failed to complete operation");
2551 let done_status = database
2552 .orchestrations()
2553 .load_rollup_operation_status(&operation_id)
2554 .await
2555 .expect("failed to load completed operation");
2556 let completed = database
2557 .orchestrations()
2558 .complete_orchestration_rollup(orchestration_id)
2559 .await
2560 .expect("failed to complete orchestration");
2561 let duplicate_completion = database
2562 .orchestrations()
2563 .complete_orchestration_rollup(orchestration_id)
2564 .await
2565 .expect("failed to inspect duplicate completion");
2566 let orchestration = database
2567 .orchestrations()
2568 .load_orchestration_for_controller("controller")
2569 .await
2570 .expect("failed to load orchestration")
2571 .expect("orchestration should exist");
2572
2573 assert_eq!(missing_status, None);
2575 assert_eq!(queued_status.as_deref(), Some("queued"));
2576 assert_eq!(running_status.as_deref(), Some("running"));
2577 assert_eq!(failed_status.as_deref(), Some("failed"));
2578 assert_eq!(done_status.as_deref(), Some("done"));
2579 assert!(completed);
2580 assert!(!duplicate_completion);
2581 assert_eq!(
2582 orchestration.status,
2583 OrchestrationStatus::AwaitingIntegration.to_string()
2584 );
2585 }
2586
2587 #[tokio::test]
2588 async fn test_session_metadata_for_project_loads_controller_and_children_together() {
2591 let database = controller_fixture().await;
2593 let project_id = database
2594 .projects()
2595 .upsert_project("/tmp/project", None)
2596 .await
2597 .expect("failed to reload project");
2598 let orchestration_id = database
2599 .orchestrations()
2600 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
2601 .await
2602 .expect("failed to insert orchestration");
2603 let running_task_id = database
2604 .orchestrations()
2605 .upsert_orchestration_task(planned_task(orchestration_id, "running"))
2606 .await
2607 .expect("failed to insert running task");
2608 let waiting_task_id = database
2609 .orchestrations()
2610 .upsert_orchestration_task(planned_task(orchestration_id, "waiting"))
2611 .await
2612 .expect("failed to insert waiting task");
2613 insert_orchestration_child(&database, project_id, "child-running", running_task_id).await;
2614 insert_orchestration_child(&database, project_id, "child-waiting", waiting_task_id).await;
2615 let running_claim = database
2616 .orchestrations()
2617 .claim_orchestration_task(running_task_id)
2618 .await
2619 .expect("failed to claim running task");
2620 let waiting_claim = database
2621 .orchestrations()
2622 .claim_orchestration_task(waiting_task_id)
2623 .await
2624 .expect("failed to claim waiting task");
2625 let running_link = database
2626 .orchestrations()
2627 .link_orchestration_task_child(running_task_id, "child-running")
2628 .await
2629 .expect("failed to link running child");
2630 let waiting_link = database
2631 .orchestrations()
2632 .link_orchestration_task_child(waiting_task_id, "child-waiting")
2633 .await
2634 .expect("failed to link waiting child");
2635 database
2636 .orchestrations()
2637 .update_orchestration_task_status(
2638 waiting_task_id,
2639 &OrchestrationTaskStatus::WaitingForInput.to_string(),
2640 None,
2641 )
2642 .await
2643 .expect("failed to mark waiting child");
2644
2645 let metadata = database
2647 .orchestrations()
2648 .load_session_metadata_for_project(project_id)
2649 .await
2650 .expect("failed to load bulk orchestration metadata");
2651
2652 assert!(running_claim);
2654 assert!(waiting_claim);
2655 assert!(running_link);
2656 assert!(waiting_link);
2657 assert_eq!(
2658 metadata,
2659 vec![
2660 SessionOrchestrationMetadataRow {
2661 controller_session_id: Some("controller".to_string()),
2662 orchestration_status: None,
2663 running_task_count: 0,
2664 session_id: "child-running".to_string(),
2665 waiting_task_count: 0,
2666 },
2667 SessionOrchestrationMetadataRow {
2668 controller_session_id: Some("controller".to_string()),
2669 orchestration_status: None,
2670 running_task_count: 0,
2671 session_id: "child-waiting".to_string(),
2672 waiting_task_count: 0,
2673 },
2674 SessionOrchestrationMetadataRow {
2675 controller_session_id: None,
2676 orchestration_status: Some(OrchestrationStatus::Running.to_string()),
2677 running_task_count: 1,
2678 session_id: "controller".to_string(),
2679 waiting_task_count: 1,
2680 },
2681 ]
2682 );
2683 }
2684
2685 #[tokio::test]
2686 async fn approval_persists_plan_and_releases_proposed_tasks() {
2687 let database = controller_fixture().await;
2689 let orchestration_id = database
2690 .orchestrations()
2691 .insert_orchestration(
2692 "controller",
2693 &OrchestrationStatus::AwaitingApproval.to_string(),
2694 2,
2695 )
2696 .await
2697 .expect("failed to insert orchestration");
2698 let task_id = database
2699 .orchestrations()
2700 .upsert_orchestration_task(planned_task(orchestration_id, "follow-up"))
2701 .await
2702 .expect("failed to insert proposed task");
2703 database
2704 .orchestrations()
2705 .update_orchestration_task_status(
2706 task_id,
2707 &OrchestrationTaskStatus::Proposed.to_string(),
2708 None,
2709 )
2710 .await
2711 .expect("failed to propose task");
2712
2713 database
2715 .orchestrations()
2716 .update_orchestration_plan(orchestration_id, "Ship the campaign", 4)
2717 .await
2718 .expect("failed to update campaign plan");
2719 let approved = database
2720 .orchestrations()
2721 .approve_orchestration_plan(orchestration_id)
2722 .await
2723 .expect("failed to approve campaign");
2724 let duplicate_approval = database
2725 .orchestrations()
2726 .approve_orchestration_plan(orchestration_id)
2727 .await
2728 .expect("failed to inspect duplicate approval");
2729 let orchestration = database
2730 .orchestrations()
2731 .load_orchestration_for_controller("controller")
2732 .await
2733 .expect("failed to load campaign")
2734 .expect("campaign should exist");
2735 let task = database
2736 .orchestrations()
2737 .load_orchestration_tasks(orchestration_id)
2738 .await
2739 .expect("failed to load campaign tasks")
2740 .remove(0);
2741
2742 assert!(approved);
2744 assert!(!duplicate_approval);
2745 assert_eq!(orchestration.goal_statement, "Ship the campaign");
2746 assert_eq!(orchestration.max_parallelism, 4);
2747 assert_eq!(
2748 orchestration.status,
2749 OrchestrationStatus::Running.to_string()
2750 );
2751 assert_eq!(task.status, OrchestrationTaskStatus::Planned.to_string());
2752 }
2753
2754 #[tokio::test]
2755 async fn integration_approval_persists_selected_approach_atomically() {
2756 let database = controller_fixture().await;
2758 let orchestration_id = database
2759 .orchestrations()
2760 .insert_orchestration(
2761 "controller",
2762 &OrchestrationStatus::AwaitingIntegration.to_string(),
2763 2,
2764 )
2765 .await
2766 .expect("failed to insert orchestration");
2767
2768 let approved = database
2770 .orchestrations()
2771 .approve_orchestration_integration(orchestration_id, IntegrationApproach::ReviewRequest)
2772 .await
2773 .expect("failed to approve integration");
2774 let duplicate_approval = database
2775 .orchestrations()
2776 .approve_orchestration_integration(orchestration_id, IntegrationApproach::LocalMerge)
2777 .await
2778 .expect("failed to inspect duplicate approval");
2779 let approach = database
2780 .orchestrations()
2781 .load_orchestration_integration_approach(orchestration_id)
2782 .await
2783 .expect("failed to load integration approach");
2784 let orchestration = database
2785 .orchestrations()
2786 .load_orchestration_for_controller("controller")
2787 .await
2788 .expect("failed to load orchestration")
2789 .expect("orchestration should exist");
2790
2791 assert!(approved);
2793 assert!(!duplicate_approval);
2794 assert_eq!(approach, IntegrationApproach::ReviewRequest.to_string());
2795 assert_eq!(
2796 orchestration.status,
2797 OrchestrationStatus::Integrating.to_string()
2798 );
2799 }
2800
2801 #[tokio::test]
2802 async fn recoverable_focused_reviews_exclude_outstanding_continuations() {
2803 let database = controller_fixture().await;
2805 let project_id = database
2806 .projects()
2807 .upsert_project("/tmp/project", None)
2808 .await
2809 .expect("failed to reload project");
2810 let orchestration_id = database
2811 .orchestrations()
2812 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
2813 .await
2814 .expect("failed to insert orchestration");
2815 let task_id = database
2816 .orchestrations()
2817 .upsert_orchestration_task(planned_task(orchestration_id, "recoverable"))
2818 .await
2819 .expect("failed to insert recoverable task");
2820 assert!(
2821 database
2822 .orchestrations()
2823 .claim_orchestration_task(task_id)
2824 .await
2825 .expect("failed to claim recoverable task")
2826 );
2827 insert_orchestration_child(&database, project_id, "child-recoverable", task_id).await;
2828 assert!(
2829 database
2830 .orchestrations()
2831 .link_orchestration_task_child(task_id, "child-recoverable")
2832 .await
2833 .expect("failed to link recoverable child")
2834 );
2835 database
2836 .orchestrations()
2837 .update_orchestration_task_status(
2838 task_id,
2839 &OrchestrationTaskStatus::Reviewing.to_string(),
2840 None,
2841 )
2842 .await
2843 .expect("failed to mark child reviewing");
2844
2845 let incomplete = database
2847 .orchestrations()
2848 .load_recoverable_focused_review_session_ids(project_id)
2849 .await
2850 .expect("failed to load incomplete review");
2851 database
2852 .orchestrations()
2853 .update_orchestration_task_status(
2854 task_id,
2855 &OrchestrationTaskStatus::ContinuationPending.to_string(),
2856 None,
2857 )
2858 .await
2859 .expect("failed to mark continuation pending");
2860 let pending = database
2861 .orchestrations()
2862 .load_recoverable_focused_review_session_ids(project_id)
2863 .await
2864 .expect("failed to inspect pending continuation recovery");
2865 database
2866 .orchestrations()
2867 .update_orchestration_task_status(
2868 task_id,
2869 &OrchestrationTaskStatus::ReviewApplying.to_string(),
2870 None,
2871 )
2872 .await
2873 .expect("failed to mark review application pending");
2874 let applying = database
2875 .orchestrations()
2876 .load_recoverable_focused_review_session_ids(project_id)
2877 .await
2878 .expect("failed to inspect review application recovery");
2879 database
2880 .orchestrations()
2881 .update_orchestration_task_status(
2882 task_id,
2883 &OrchestrationTaskStatus::Reviewing.to_string(),
2884 None,
2885 )
2886 .await
2887 .expect("failed to restore reviewing state");
2888 database
2889 .sessions()
2890 .update_session_focused_review(
2891 "child-recoverable",
2892 Some(FocusedReviewStatus::Ready),
2893 Some("42".to_string()),
2894 Some("### Suggestions\n\n- None".to_string()),
2895 )
2896 .await
2897 .expect("failed to complete focused review");
2898 let completed = database
2899 .orchestrations()
2900 .load_recoverable_focused_review_session_ids(project_id)
2901 .await
2902 .expect("failed to reload completed review");
2903
2904 assert_eq!(incomplete, ["child-recoverable"]);
2906 assert!(pending.is_empty() && applying.is_empty() && completed.is_empty());
2907 }
2908
2909 #[tokio::test]
2910 async fn review_application_claim_is_bounded_and_clears_consumed_review() {
2911 let database = controller_fixture().await;
2913 let project_id = database
2914 .projects()
2915 .upsert_project("/tmp/project", None)
2916 .await
2917 .expect("failed to reload project");
2918 let orchestration_id = database
2919 .orchestrations()
2920 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
2921 .await
2922 .expect("failed to insert orchestration");
2923 let task_id = database
2924 .orchestrations()
2925 .upsert_orchestration_task(planned_task(orchestration_id, "reviewed"))
2926 .await
2927 .expect("failed to insert reviewed task");
2928 assert!(
2929 database
2930 .orchestrations()
2931 .claim_orchestration_task(task_id)
2932 .await
2933 .expect("failed to claim reviewed task")
2934 );
2935 insert_orchestration_child(&database, project_id, "child-reviewed", task_id).await;
2936 assert!(
2937 database
2938 .orchestrations()
2939 .link_orchestration_task_child(task_id, "child-reviewed")
2940 .await
2941 .expect("failed to link reviewed child")
2942 );
2943 database
2944 .orchestrations()
2945 .update_orchestration_task_status(
2946 task_id,
2947 &OrchestrationTaskStatus::Reviewing.to_string(),
2948 None,
2949 )
2950 .await
2951 .expect("failed to mark child reviewing");
2952 database
2953 .sessions()
2954 .update_session_focused_review(
2955 "child-reviewed",
2956 Some(FocusedReviewStatus::Ready),
2957 Some("42".to_string()),
2958 Some("### Suggestions\n\n- Fix it".to_string()),
2959 )
2960 .await
2961 .expect("failed to seed focused review");
2962
2963 let claimed = database
2965 .orchestrations()
2966 .claim_orchestration_review_application(task_id, "Verify then apply", 3)
2967 .await
2968 .expect("failed to claim review application");
2969 let duplicate_claim = database
2970 .orchestrations()
2971 .claim_orchestration_review_application(task_id, "Duplicate", 3)
2972 .await
2973 .expect("failed to inspect duplicate review application");
2974 let task = database
2975 .orchestrations()
2976 .load_orchestration_tasks(orchestration_id)
2977 .await
2978 .expect("failed to load reviewed task")
2979 .remove(0);
2980 let review_cache = database
2981 .sessions()
2982 .load_session_focused_reviews_for_project(project_id)
2983 .await
2984 .expect("failed to load consumed review cache");
2985
2986 assert!(claimed);
2988 assert!(!duplicate_claim);
2989 assert_eq!(
2990 task.status,
2991 OrchestrationTaskStatus::ReviewApplying.to_string()
2992 );
2993 assert_eq!(task.continuation_generation, 1);
2994 assert_eq!(
2995 task.continuation_prompt.as_deref(),
2996 Some("Verify then apply")
2997 );
2998 assert_eq!(task.review_iteration, 1);
2999 assert_eq!(review_cache, [] as [crate::SessionFocusedReviewRow; 0]);
3000 assert_eq!(task.child_focused_review_status, None);
3001 assert_eq!(task.child_focused_review_text, None);
3002 }
3003
3004 #[tokio::test]
3005 async fn managed_child_continuation_questions_and_detach_are_durable() {
3006 let database = controller_fixture().await;
3008 let project_id = database
3009 .projects()
3010 .upsert_project("/tmp/project", None)
3011 .await
3012 .expect("failed to reload project");
3013 let orchestration_id = database
3014 .orchestrations()
3015 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
3016 .await
3017 .expect("failed to insert orchestration");
3018 let task_id = database
3019 .orchestrations()
3020 .upsert_orchestration_task(planned_task(orchestration_id, "alpha"))
3021 .await
3022 .expect("failed to insert task");
3023 assert!(
3024 database
3025 .orchestrations()
3026 .claim_orchestration_task(task_id)
3027 .await
3028 .expect("failed to claim task")
3029 );
3030 insert_orchestration_child(&database, project_id, "child-alpha", task_id).await;
3031 assert!(
3032 database
3033 .orchestrations()
3034 .link_orchestration_task_child(task_id, "child-alpha")
3035 .await
3036 .expect("failed to link child")
3037 );
3038 database
3039 .orchestrations()
3040 .update_orchestration_task_status(
3041 task_id,
3042 &OrchestrationTaskStatus::WaitingForInput.to_string(),
3043 None,
3044 )
3045 .await
3046 .expect("failed to wait for child question");
3047
3048 let surfaced = surface_and_clear_orchestration_questions(
3050 &database,
3051 orchestration_id,
3052 task_id,
3053 r#"[{"text":"Choose one"}]"#,
3054 )
3055 .await;
3056 database
3057 .orchestrations()
3058 .update_orchestration_task_status(
3059 task_id,
3060 &OrchestrationTaskStatus::Ready.to_string(),
3061 None,
3062 )
3063 .await
3064 .expect("failed to settle child after its question");
3065 let queued = database
3066 .orchestrations()
3067 .queue_orchestration_continuation(
3068 task_id,
3069 "Add the missing edge case",
3070 r#"["The edge case is tested"]"#,
3071 r#"["docs/"]"#,
3072 )
3073 .await
3074 .expect("failed to queue continuation");
3075 let duplicate_queue = database
3076 .orchestrations()
3077 .queue_orchestration_continuation(
3078 task_id,
3079 "Duplicate",
3080 r#"["Duplicate"]"#,
3081 r#"["ignored/"]"#,
3082 )
3083 .await
3084 .expect("failed to inspect duplicate continuation");
3085 let detached = database
3086 .orchestrations()
3087 .detach_orchestration_child("child-alpha")
3088 .await
3089 .expect("failed to detach child");
3090 let duplicate_detach = database
3091 .orchestrations()
3092 .detach_orchestration_child("child-alpha")
3093 .await
3094 .expect("failed to inspect duplicate detach");
3095 let (task, child, controller) =
3096 load_detached_campaign_state(&database, orchestration_id).await;
3097 let continuation_prompt = task.continuation_prompt.as_deref();
3098
3099 assert!(queued && surfaced && detached);
3101 assert!(!duplicate_queue && !duplicate_detach);
3102 assert_eq!(task.status, OrchestrationTaskStatus::Detached.to_string());
3103 assert_eq!(task.child_session_id, None);
3104 assert_eq!(task.continuation_generation, 1);
3105 assert_eq!(continuation_prompt, Some("Add the missing edge case"));
3106 assert_eq!(task.touched_areas, r#"["docs/"]"#);
3107 assert_eq!(child.role.as_deref(), Some("Worker"));
3108 assert_eq!(controller.status, "Review");
3109 assert_eq!(controller.questions.as_deref(), Some(""));
3110 }
3111
3112 #[tokio::test]
3113 async fn question_relay_preserves_controller_questions() {
3114 let database = controller_fixture().await;
3116 let project_id = database
3117 .projects()
3118 .upsert_project("/tmp/project", None)
3119 .await
3120 .expect("failed to reload project");
3121 let orchestration_id = database
3122 .orchestrations()
3123 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
3124 .await
3125 .expect("failed to insert orchestration");
3126 let task_id = insert_waiting_orchestration_task(
3127 &database,
3128 project_id,
3129 orchestration_id,
3130 "worker",
3131 "child-worker",
3132 )
3133 .await;
3134 let controller_question = r#"[{"text":"Controller question"}]"#;
3135 database
3136 .sessions()
3137 .update_session_questions("controller", controller_question)
3138 .await
3139 .expect("failed to seed controller question");
3140
3141 let blocked_by_controller = database
3143 .orchestrations()
3144 .surface_orchestration_questions(
3145 orchestration_id,
3146 task_id,
3147 r#"[{"text":"Child question"}]"#,
3148 )
3149 .await
3150 .expect("failed to inspect controller question");
3151 let orchestration = database
3152 .orchestrations()
3153 .load_orchestration_for_controller("controller")
3154 .await
3155 .expect("failed to load orchestration")
3156 .expect("orchestration should exist");
3157 let controller = database
3158 .sessions()
3159 .load_session("controller")
3160 .await
3161 .expect("failed to load controller")
3162 .expect("controller should exist");
3163
3164 assert!(!blocked_by_controller);
3166 assert_eq!(orchestration.relayed_question_task_id, None);
3167 assert_eq!(controller.questions.as_deref(), Some(controller_question));
3168 }
3169
3170 #[tokio::test]
3171 async fn question_relay_claims_one_exact_task_at_a_time() {
3172 let database = controller_fixture().await;
3174 let project_id = database
3175 .projects()
3176 .upsert_project("/tmp/project", None)
3177 .await
3178 .expect("failed to reload project");
3179 let orchestration_id = database
3180 .orchestrations()
3181 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
3182 .await
3183 .expect("failed to insert orchestration");
3184 let first_task_id = insert_waiting_orchestration_task(
3185 &database,
3186 project_id,
3187 orchestration_id,
3188 "first",
3189 "child-first",
3190 )
3191 .await;
3192 let second_task_id = insert_waiting_orchestration_task(
3193 &database,
3194 project_id,
3195 orchestration_id,
3196 "second",
3197 "child-second",
3198 )
3199 .await;
3200
3201 let first_surfaced = database
3203 .orchestrations()
3204 .surface_orchestration_questions(
3205 orchestration_id,
3206 first_task_id,
3207 r#"[{"text":"First child question"}]"#,
3208 )
3209 .await
3210 .expect("failed to surface first child question");
3211 let second_blocked = database
3212 .orchestrations()
3213 .surface_orchestration_questions(
3214 orchestration_id,
3215 second_task_id,
3216 r#"[{"text":"Second child question"}]"#,
3217 )
3218 .await
3219 .expect("failed to inspect occupied relay");
3220 let claimed_orchestration = database
3221 .orchestrations()
3222 .load_orchestration_for_controller("controller")
3223 .await
3224 .expect("failed to load claimed orchestration")
3225 .expect("orchestration should exist");
3226 database
3227 .orchestrations()
3228 .clear_orchestration_questions(orchestration_id)
3229 .await
3230 .expect("failed to release first relay");
3231 let second_surfaced = database
3232 .orchestrations()
3233 .surface_orchestration_questions(
3234 orchestration_id,
3235 second_task_id,
3236 r#"[{"text":"Second child question"}]"#,
3237 )
3238 .await
3239 .expect("failed to surface second child question");
3240 let second_claimed_orchestration = database
3241 .orchestrations()
3242 .load_orchestration_for_controller("controller")
3243 .await
3244 .expect("failed to load second claimed orchestration")
3245 .expect("orchestration should exist");
3246
3247 assert!(first_surfaced);
3249 assert!(!second_blocked);
3250 assert_eq!(
3251 claimed_orchestration.relayed_question_task_id,
3252 Some(first_task_id)
3253 );
3254 assert!(second_surfaced);
3255 assert_eq!(
3256 second_claimed_orchestration.relayed_question_task_id,
3257 Some(second_task_id)
3258 );
3259 }
3260
3261 #[tokio::test]
3262 async fn infrastructure_retries_are_bounded_and_completion_is_idempotent() {
3263 let database = controller_fixture().await;
3265 let orchestration_id = database
3266 .orchestrations()
3267 .insert_orchestration("controller", &OrchestrationStatus::Running.to_string(), 2)
3268 .await
3269 .expect("failed to insert orchestration");
3270 let task_id = database
3271 .orchestrations()
3272 .upsert_orchestration_task(planned_task(orchestration_id, "alpha"))
3273 .await
3274 .expect("failed to insert task");
3275
3276 let first_status = database
3278 .orchestrations()
3279 .record_orchestration_spawn_failure(task_id, "provider unavailable", 2)
3280 .await
3281 .expect("failed to record first retry");
3282 let second_status = database
3283 .orchestrations()
3284 .record_orchestration_spawn_failure(task_id, "provider unavailable", 2)
3285 .await
3286 .expect("failed to record second retry");
3287 let final_status = database
3288 .orchestrations()
3289 .record_orchestration_spawn_failure(task_id, "provider unavailable", 2)
3290 .await
3291 .expect("failed to exhaust retries");
3292 database
3293 .orchestrations()
3294 .update_orchestration_status(
3295 orchestration_id,
3296 &OrchestrationStatus::Integrating.to_string(),
3297 )
3298 .await
3299 .expect("failed to begin integration completion");
3300 let completed = database
3301 .orchestrations()
3302 .complete_orchestration_campaign(orchestration_id)
3303 .await
3304 .expect("failed to complete campaign");
3305 let duplicate_completion = database
3306 .orchestrations()
3307 .complete_orchestration_campaign(orchestration_id)
3308 .await
3309 .expect("failed to inspect duplicate completion");
3310 let orchestration = database
3311 .orchestrations()
3312 .load_orchestration_for_controller("controller")
3313 .await
3314 .expect("failed to load completed campaign")
3315 .expect("completed campaign should exist");
3316 let task = database
3317 .orchestrations()
3318 .load_orchestration_tasks(orchestration_id)
3319 .await
3320 .expect("failed to load exhausted task")
3321 .remove(0);
3322 let controller = database
3323 .sessions()
3324 .load_session("controller")
3325 .await
3326 .expect("failed to load completed controller")
3327 .expect("controller should exist");
3328
3329 assert_eq!(first_status, OrchestrationTaskStatus::Planned.to_string());
3331 assert_eq!(second_status, OrchestrationTaskStatus::Planned.to_string());
3332 assert_eq!(final_status, OrchestrationTaskStatus::Failed.to_string());
3333 assert!(completed);
3334 assert!(!duplicate_completion);
3335 assert_eq!(task.infrastructure_retry_count, 3);
3336 assert_eq!(task.status, OrchestrationTaskStatus::Failed.to_string());
3337 assert_eq!(orchestration.status, OrchestrationStatus::Done.to_string());
3338 assert_eq!(controller.status, "Done");
3339 }
3340
3341 #[tokio::test]
3342 async fn test_missing_orchestration_returns_none() {
3344 let database = controller_fixture().await;
3346
3347 let orchestration = database
3349 .orchestrations()
3350 .load_orchestration_for_controller("controller")
3351 .await
3352 .expect("failed to load orchestration");
3353 assert_eq!(orchestration, None);
3355 }
3356}