1use crate::backends::{ExternalId, ExternalTask, Status};
4use crate::engine;
5use crate::engine::auto_merge;
6use crate::engine::review_gate;
7use crate::engine::tasks::TaskManager;
8use crate::github::http::{GhHttp, RulesetCreateKind, classify_ruleset_create_error};
9use crate::github::types::{GitHubCheckRun, RequiredStatusCheck};
10use crate::plan;
11use crate::store;
12use crate::store::TaskStore;
13use std::sync::Arc;
14
15const KV_PENDING: &str = "pending";
16const KV_APPLIED: &str = "applied";
17const KV_REFUSED: &str = "refused";
18
19#[derive(Debug, PartialEq)]
20pub(crate) enum ChecksReady {
21 Ready(Vec<RequiredStatusCheck>),
22 Pending,
24 Failed(Vec<FailedCheck>),
26 Empty,
28}
29
30#[derive(Debug, PartialEq)]
31pub(crate) struct FailedCheck {
32 name: String,
33 conclusion: String,
34}
35
36pub(crate) fn is_ci_bootstrap_task(body: &str) -> bool {
37 plan::is_ci_bootstrap_body(body)
38 || plan::marker_id(body).is_some_and(|id| plan::is_ci_bootstrap_id(&id))
39}
40
41pub(crate) fn kv_key(repo: &str, task_id: &str) -> String {
42 format!("ci-bootstrap:{repo}:{task_id}")
43}
44
45pub(crate) async fn hold_after_merge(
47 task: &ExternalTask,
48 store: &Arc<TaskStore>,
49 repo: &str,
50) -> bool {
51 if !is_ci_bootstrap_task(&task.body) {
52 return false;
53 }
54 let key = kv_key(repo, &task.id.0);
55 match store.kv_get(&key).await {
56 Ok(Some(state)) if state == KV_APPLIED => false,
57 Ok(Some(_)) => true,
58 Ok(None) => {
59 if let Err(e) = store.kv_set(&key, KV_PENDING).await {
60 tracing::warn!(
61 task_id = %task.id.0,
62 err = %e,
63 "failed to record CI bootstrap pending"
64 );
65 }
66 true
67 }
68 Err(e) => {
69 tracing::warn!(
70 task_id = %task.id.0,
71 err = %e,
72 "failed to read CI bootstrap state"
73 );
74 true
75 }
76 }
77}
78
79pub(crate) async fn apply_pending(
80 store: &Arc<TaskStore>,
81 task_manager: &Arc<TaskManager>,
82 repo: &str,
83) -> anyhow::Result<()> {
84 let gh = GhHttp::owner()?;
85 apply_pending_with(&gh, store, task_manager, repo).await
86}
87
88pub(crate) async fn apply_pending_with(
89 gh: &GhHttp,
90 store: &Arc<TaskStore>,
91 task_manager: &Arc<TaskManager>,
92 repo: &str,
93) -> anyhow::Result<()> {
94 let prefix = format!("ci-bootstrap:{repo}:");
95 let entries = store.kv_list_prefix(&prefix).await?;
96 for (key, state) in entries {
97 if state == KV_APPLIED {
98 continue;
99 }
100 let Some(task_id) = key.strip_prefix(&prefix) else {
101 continue;
102 };
103 if let Err(e) = apply_one(gh, store, task_manager, repo, task_id, &state).await {
104 tracing::warn!(task_id, err = %e, "CI bootstrap ruleset apply failed");
105 }
106 }
107 Ok(())
108}
109
110pub(crate) fn checks_ready_for_ruleset(
111 runs: &[GitHubCheckRun],
112 statuses: &[(String, String)],
113) -> ChecksReady {
114 let mut ready: Vec<RequiredStatusCheck> = Vec::new();
115 let mut pending = false;
116 let mut failed: Vec<FailedCheck> = Vec::new();
117
118 for run in runs {
119 if review_gate::is_own_check(&run.name) {
120 continue;
121 }
122 match run.status.as_str() {
123 "completed" => match run.conclusion.as_deref() {
124 Some("success") => {
125 let integration_id = run.app.as_ref().map(|app| app.id).filter(|&id| id > 0);
126 if !ready.iter().any(|check| check.context == run.name) {
127 ready.push(RequiredStatusCheck {
128 context: run.name.clone(),
129 integration_id,
130 });
131 }
132 }
133 Some("skipped" | "neutral") => {}
134 Some(conclusion) => failed.push(FailedCheck {
135 name: run.name.clone(),
136 conclusion: conclusion.to_string(),
137 }),
138 None => pending = true,
139 },
140 _ => pending = true,
141 }
142 }
143
144 for (context, state) in statuses {
145 if review_gate::is_own_check(context) {
146 continue;
147 }
148 match state.as_str() {
149 "pending" => pending = true,
150 "success" => {
151 if !ready.iter().any(|check| &check.context == context) {
152 ready.push(RequiredStatusCheck {
153 context: context.clone(),
154 integration_id: None,
155 });
156 }
157 }
158 "failure" | "error" => failed.push(FailedCheck {
159 name: context.clone(),
160 conclusion: state.clone(),
161 }),
162 _ => {}
163 }
164 }
165
166 if !failed.is_empty() {
167 ChecksReady::Failed(failed)
168 } else if pending {
169 ChecksReady::Pending
170 } else if ready.is_empty() {
171 ChecksReady::Empty
172 } else {
173 ChecksReady::Ready(ready)
174 }
175}
176
177async fn apply_one(
178 gh: &GhHttp,
179 store: &Arc<TaskStore>,
180 task_manager: &Arc<TaskManager>,
181 repo: &str,
182 task_id: &str,
183 state: &str,
184) -> anyhow::Result<()> {
185 if store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
186 .await
187 .is_none()
188 {
189 return Ok(());
190 }
191
192 let branch = gh.get_default_branch(repo).await?;
193 let required = gh.get_required_status_check_contexts(repo, &branch).await?;
194 if !required.is_empty() {
195 return finish_applied(store, task_manager, task_id, repo).await;
196 }
197
198 reopen_if_closed(gh, repo, task_id).await;
200
201 if state == KV_REFUSED {
202 return Ok(());
203 }
204
205 let sha = gh.get_branch_sha(repo, &branch).await?;
206 let runs = gh.get_check_runs(repo, &sha).await?;
207 let statuses = gh.get_commit_status_contexts(repo, &sha).await?;
208 let checks = match checks_ready_for_ruleset(&runs, &statuses) {
209 ChecksReady::Pending => {
210 tracing::info!(
211 task_id,
212 sha = %sha,
213 "CI bootstrap waiting for default-branch checks"
214 );
215 clear_wait(store, repo, task_id).await;
216 return Ok(());
217 }
218 ChecksReady::Empty => {
219 return wait_for_checks(gh, store, repo, task_id, &sha).await;
220 }
221 ChecksReady::Failed(failed) => {
222 clear_wait(store, repo, task_id).await;
223 return handle_failed_checks(gh, store, task_manager, repo, task_id, &sha, &failed)
224 .await;
225 }
226 ChecksReady::Ready(checks) => {
227 clear_wait(store, repo, task_id).await;
228 checks
229 }
230 };
231
232 match gh.create_required_check_ruleset(repo, &checks).await {
233 Ok(()) => finish_applied(store, task_manager, task_id, repo).await,
234 Err(e) => {
235 let err = e.to_string();
236 match classify_ruleset_create_error(&err) {
237 RulesetCreateKind::NameTaken => {
238 finish_applied(store, task_manager, task_id, repo).await
239 }
240 RulesetCreateKind::Retryable => {
241 tracing::info!(
242 task_id,
243 err = %err,
244 "CI bootstrap ruleset create retrying"
245 );
246 Ok(())
247 }
248 RulesetCreateKind::CheckNeverRun => {
249 tracing::info!(
250 task_id,
251 err = %err,
252 "CI bootstrap ruleset create waiting, check has not run"
253 );
254 comment_never_run_once(gh, store, repo, task_id, &err).await;
255 Ok(())
256 }
257 RulesetCreateKind::Refused | RulesetCreateKind::Other => {
258 comment_refused(gh, repo, task_id, &err).await;
259 store.kv_set(&kv_key(repo, task_id), KV_REFUSED).await?;
260 Ok(())
261 }
262 }
263 }
264 }
265}
266
267async fn finish_applied(
268 store: &Arc<TaskStore>,
269 task_manager: &Arc<TaskManager>,
270 task_id: &str,
271 repo: &str,
272) -> anyhow::Result<()> {
273 store.kv_set(&kv_key(repo, task_id), KV_APPLIED).await?;
274 task_manager
275 .update_task_status(&ExternalId(task_id.to_string()), Status::Done)
276 .await?;
277 Ok(())
278}
279
280async fn reopen_if_closed(gh: &GhHttp, repo: &str, task_id: &str) {
281 match gh.get_issue(repo, task_id).await {
282 Ok(issue) if issue.state.eq_ignore_ascii_case("closed") => {
283 if let Err(e) = gh.reopen_issue(repo, task_id).await {
284 tracing::warn!(
285 task_id,
286 err = %e,
287 "failed to reopen CI bootstrap issue"
288 );
289 }
290 }
291 Ok(_) => {}
292 Err(e) => {
293 tracing::warn!(
294 task_id,
295 err = %e,
296 "failed to read CI bootstrap issue"
297 );
298 }
299 }
300}
301
302fn wait_prefix(repo: &str, task_id: &str) -> String {
303 format!("ci_bootstrap_wait_since:{repo}#{task_id}#")
304}
305
306fn no_checks_comment_key(repo: &str, task_id: &str, sha: &str) -> String {
307 format!("ci_bootstrap_no_checks_comment:{repo}#{task_id}#{sha}")
308}
309
310fn never_run_comment_key(repo: &str, task_id: &str) -> String {
311 format!("ci_bootstrap_never_run_comment:{repo}#{task_id}")
312}
313
314fn format_failed(failed: &[FailedCheck]) -> String {
315 failed
316 .iter()
317 .map(|check| format!("{}: {}", check.name, check.conclusion))
318 .collect::<Vec<_>>()
319 .join(", ")
320}
321
322async fn clear_wait(store: &Arc<TaskStore>, repo: &str, task_id: &str) {
323 let prefix = wait_prefix(repo, task_id);
324 let keys = match store.kv_list_prefix(&prefix).await {
325 Ok(entries) => entries,
326 Err(e) => {
327 tracing::warn!(task_id, err = %e, "failed to list CI bootstrap wait keys");
328 return;
329 }
330 };
331 for (key, _) in keys {
332 if let Err(e) = store.kv_delete(&key).await {
333 tracing::warn!(task_id, err = %e, "failed to clear CI bootstrap wait key");
334 }
335 }
336}
337
338async fn wait_for_checks(
339 gh: &GhHttp,
340 store: &Arc<TaskStore>,
341 repo: &str,
342 task_id: &str,
343 sha: &str,
344) -> anyhow::Result<()> {
345 let prefix = wait_prefix(repo, task_id);
346 let key = format!("{prefix}{sha}");
347 let now = chrono::Utc::now().timestamp();
348 let since = review_gate::wait_since(store, &prefix, &key, now).await?;
349 if now - since < review_gate::NO_CHECKS_WAIT_SECS {
350 tracing::info!(
351 task_id,
352 sha = %sha,
353 "CI bootstrap waiting for a default-branch check to start"
354 );
355 return Ok(());
356 }
357 let wait_mins = review_gate::NO_CHECKS_WAIT_SECS / 60;
358 tracing::warn!(
359 task_id,
360 sha = %sha,
361 wait_mins,
362 "CI bootstrap has no default-branch check after the wait"
363 );
364 let comment_key = no_checks_comment_key(repo, task_id, sha);
365 if store.kv_get(&comment_key).await?.is_none() {
366 comment_issue(
367 gh,
368 repo,
369 task_id,
370 &format!(
371 "No default-branch check has started {wait_mins} minutes after the CI bootstrap merge. Later tasks stay waiting until a check runs."
372 ),
373 )
374 .await;
375 store.kv_set(&comment_key, "1").await?;
376 }
377 Ok(())
378}
379
380async fn handle_failed_checks(
381 gh: &GhHttp,
382 store: &Arc<TaskStore>,
383 task_manager: &Arc<TaskManager>,
384 repo: &str,
385 task_id: &str,
386 sha: &str,
387 failed: &[FailedCheck],
388) -> anyhow::Result<()> {
389 let details = format_failed(failed);
390 let failures = match auto_merge::count_ci_failure_for_head(store, repo, task_id, sha).await {
391 Ok(Some(v)) => v,
392 Ok(None) => {
393 tracing::info!(
394 task_id,
395 sha = %sha,
396 details,
397 "CI bootstrap default-branch checks still failed on the same head"
398 );
399 return Ok(());
400 }
401 Err(e) => {
402 tracing::warn!(
403 task_id,
404 err = %e,
405 "failed to increment ci_merge_failures for CI bootstrap"
406 );
407 return Ok(());
408 }
409 };
410
411 comment_issue(
412 gh,
413 repo,
414 task_id,
415 &format!(
416 "Default-branch checks failed after the CI bootstrap merge ({details}). Later tasks stay waiting until those checks pass."
417 ),
418 )
419 .await;
420
421 let max = auto_merge::MAX_CI_MERGE_FAILURES;
422 let id = ExternalId(task_id.to_string());
423 if failures >= max {
424 tracing::error!(
425 task_id,
426 failures,
427 details,
428 "CI bootstrap default-branch checks failed, CI failure limit reached"
429 );
430 let fields = [(
431 "block_reason",
432 serde_json::json!(format!(
433 "CI failure limit ({max}) reached after the CI bootstrap merge"
434 )),
435 )];
436 if let Err(e) = task_manager
437 .update_task_status_and_result(&id, Status::Blocked, &fields)
438 .await
439 {
440 tracing::error!(
441 task_id,
442 err = %e,
443 "failed to block CI bootstrap after failed default-branch checks"
444 );
445 }
446 } else {
447 tracing::warn!(
448 task_id,
449 failures,
450 details,
451 "CI bootstrap default-branch checks failed, routing back to the task agent"
452 );
453 let fields = [
454 (
455 "last_error",
456 serde_json::json!(format!(
457 "CI failed on the default branch after the CI bootstrap merge: {details}. Fix the failing checks"
458 )),
459 ),
460 (
461 "route_reason",
462 serde_json::json!("re-dispatch after CI bootstrap default-branch checks failed"),
463 ),
464 ];
465 if let Err(e) = task_manager
466 .update_task_status_and_result(&id, Status::Routed, &fields)
467 .await
468 {
469 tracing::error!(
470 task_id,
471 err = %e,
472 "failed to re-route CI bootstrap after failed default-branch checks"
473 );
474 }
475 }
476 Ok(())
477}
478
479async fn comment_never_run_once(
480 gh: &GhHttp,
481 store: &Arc<TaskStore>,
482 repo: &str,
483 task_id: &str,
484 err: &str,
485) {
486 let key = never_run_comment_key(repo, task_id);
487 match store.kv_get(&key).await {
488 Ok(Some(_)) => return,
489 Ok(None) => {}
490 Err(e) => {
491 tracing::warn!(
492 task_id,
493 err = %e,
494 "failed to read CI bootstrap never-run comment key"
495 );
496 return;
497 }
498 }
499 comment_issue(
500 gh,
501 repo,
502 task_id,
503 &format!(
504 "GitHub does not treat the default-branch checks as valid required checks ({err}). Later tasks stay waiting until a required-checks ruleset can be created."
505 ),
506 )
507 .await;
508 if let Err(e) = store.kv_set(&key, "1").await {
509 tracing::warn!(
510 task_id,
511 err = %e,
512 "failed to record CI bootstrap never-run comment"
513 );
514 }
515}
516
517async fn comment_refused(gh: &GhHttp, repo: &str, task_id: &str, err: &str) {
518 comment_issue(
519 gh,
520 repo,
521 task_id,
522 &format!(
523 "GitHub refused a required-checks ruleset on the default branch ({err}). Turn branch protection on for this repository. Later tasks stay waiting until that happens."
524 ),
525 )
526 .await;
527}
528
529async fn comment_issue(gh: &GhHttp, repo: &str, task_id: &str, message: &str) {
530 let body = format!("{message}{}", engine::orch_footer());
531 if let Err(e) = gh.add_comment(repo, task_id, &body).await {
532 tracing::warn!(
533 task_id,
534 err = %e,
535 "failed to comment on CI bootstrap issue"
536 );
537 }
538}
539
540#[cfg(test)]
541mod tests {
542 use super::*;
543 use crate::backends::ExternalBackend;
544 use crate::backends::test_helpers::NoopBackend;
545 use crate::github::types::GitHubApp;
546 use crate::plan::{CI_BOOTSTRAP_MARKER, ci_bootstrap_task, issue_body};
547 use crate::store::TaskStatus;
548 use crate::store::UpsertExternal;
549 use std::env;
550 use std::ffi::OsString;
551 use wiremock::matchers::{method, path};
552 use wiremock::{Mock, MockServer, ResponseTemplate};
553
554 struct EnvVarGuard {
555 key: &'static str,
556 previous: Option<OsString>,
557 }
558
559 impl EnvVarGuard {
560 fn set(key: &'static str, value: &str) -> Self {
561 let previous = env::var_os(key);
562 unsafe { env::set_var(key, value) };
564 Self { key, previous }
565 }
566 }
567
568 impl Drop for EnvVarGuard {
569 fn drop(&mut self) {
570 match self.previous.take() {
571 Some(value) => unsafe { env::set_var(self.key, value) },
573 None => unsafe { env::remove_var(self.key) },
575 }
576 }
577 }
578
579 fn run(
580 name: &str,
581 status: &str,
582 conclusion: Option<&str>,
583 app_id: Option<i64>,
584 ) -> GitHubCheckRun {
585 GitHubCheckRun {
586 id: 1,
587 name: name.to_string(),
588 status: status.to_string(),
589 conclusion: conclusion.map(String::from),
590 app: app_id.map(|id| GitHubApp { id }),
591 }
592 }
593
594 fn bootstrap_body() -> String {
595 issue_body(
596 &ci_bootstrap_task(false),
597 "acme/app",
598 "main",
599 "docs/ROADMAP.md",
600 )
601 }
602
603 async fn pending_store() -> (Arc<TaskStore>, Arc<TaskManager>, i64) {
604 let store = Arc::new(TaskStore::open_memory().await.unwrap());
605 let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
606 let task_manager = Arc::new(TaskManager::with_store(
607 backend,
608 Arc::clone(&store),
609 "o/r".to_string(),
610 ));
611 let id = store
612 .upsert_external(&UpsertExternal {
613 repo: "o/r",
614 ext_id: "12",
615 title: "CI-01 Set up CI and tests",
616 body: &bootstrap_body(),
617 author: "u",
618 url: "",
619 labels: &[],
620 origin: "github",
621 })
622 .await
623 .unwrap();
624 store.update_status(id, TaskStatus::Approved).await.unwrap();
625 store
626 .kv_set(&kv_key("o/r", "12"), KV_PENDING)
627 .await
628 .unwrap();
629 (store, task_manager, id)
630 }
631
632 async fn mock_unprotected_main(server: &MockServer, sha: &str) {
633 Mock::given(method("GET"))
634 .and(path("/repos/o/r"))
635 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
636 "default_branch": "main"
637 })))
638 .mount(server)
639 .await;
640 Mock::given(method("GET"))
641 .and(path("/repos/o/r/branches/main/protection"))
642 .respond_with(
643 ResponseTemplate::new(404)
644 .set_body_string("{\"message\":\"Branch not protected\"}"),
645 )
646 .mount(server)
647 .await;
648 Mock::given(method("GET"))
649 .and(path("/repos/o/r/rules/branches/main"))
650 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
651 .mount(server)
652 .await;
653 Mock::given(method("GET"))
654 .and(path("/repos/o/r/branches/main"))
655 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
656 "commit": { "sha": sha }
657 })))
658 .mount(server)
659 .await;
660 }
661
662 async fn mock_runs_and_empty_status(
663 server: &MockServer,
664 sha: &str,
665 check_runs: serde_json::Value,
666 ) {
667 Mock::given(method("GET"))
668 .and(path(format!("/repos/o/r/commits/{sha}/check-runs")))
669 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
670 "total_count": 1,
671 "check_runs": check_runs
672 })))
673 .mount(server)
674 .await;
675 Mock::given(method("GET"))
676 .and(path(format!("/repos/o/r/commits/{sha}/status")))
677 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
678 "state": "pending",
679 "total_count": 0,
680 "statuses": []
681 })))
682 .mount(server)
683 .await;
684 }
685
686 async fn mock_issue_comment(server: &MockServer, expected: u64) {
687 Mock::given(method("POST"))
688 .and(path("/repos/o/r/issues/12/comments"))
689 .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
690 "id": 1
691 })))
692 .expect(expected)
693 .mount(server)
694 .await;
695 }
696
697 #[test]
698 fn is_ci_bootstrap_task_reads_the_body_marker() {
699 assert!(is_ci_bootstrap_task(&bootstrap_body()));
700 assert!(is_ci_bootstrap_task(&format!("{CI_BOOTSTRAP_MARKER}\n")));
701 assert!(is_ci_bootstrap_task("goal\n<!-- orch-plan: CI-01 -->"));
702 assert!(!is_ci_bootstrap_task(""));
703 assert!(!is_ci_bootstrap_task("goal"));
704 }
705
706 #[test]
707 fn checks_ready_for_ruleset_splits_pending_failed_and_empty() {
708 assert_eq!(checks_ready_for_ruleset(&[], &[]), ChecksReady::Empty);
709 assert_eq!(
710 checks_ready_for_ruleset(&[run("test", "in_progress", None, None)], &[]),
711 ChecksReady::Pending
712 );
713 assert_eq!(
714 checks_ready_for_ruleset(&[run("test", "completed", Some("failure"), None)], &[]),
715 ChecksReady::Failed(vec![FailedCheck {
716 name: "test".to_string(),
717 conclusion: "failure".to_string(),
718 }])
719 );
720 assert_eq!(
721 checks_ready_for_ruleset(&[run("test", "completed", Some("cancelled"), None)], &[]),
722 ChecksReady::Failed(vec![FailedCheck {
723 name: "test".to_string(),
724 conclusion: "cancelled".to_string(),
725 }])
726 );
727 assert_eq!(
728 checks_ready_for_ruleset(&[], &[("ci/test".to_string(), "failure".to_string())]),
729 ChecksReady::Failed(vec![FailedCheck {
730 name: "ci/test".to_string(),
731 conclusion: "failure".to_string(),
732 }])
733 );
734 assert_eq!(
735 checks_ready_for_ruleset(&[run("test", "completed", Some("skipped"), None)], &[]),
736 ChecksReady::Empty
737 );
738 assert_eq!(
739 checks_ready_for_ruleset(
740 &[run("orch/review", "completed", Some("success"), None)],
741 &[]
742 ),
743 ChecksReady::Empty
744 );
745 assert_eq!(
746 checks_ready_for_ruleset(
747 &[
748 run("test", "completed", Some("failure"), None),
749 run("lint", "in_progress", None, None),
750 ],
751 &[]
752 ),
753 ChecksReady::Failed(vec![FailedCheck {
754 name: "test".to_string(),
755 conclusion: "failure".to_string(),
756 }])
757 );
758 let ChecksReady::Ready(ready) = checks_ready_for_ruleset(
759 &[
760 run("test", "completed", Some("success"), Some(15368)),
761 run("review-gate", "completed", Some("success"), None),
762 ],
763 &[],
764 ) else {
765 panic!("expected ready checks");
766 };
767 assert_eq!(
768 ready,
769 vec![RequiredStatusCheck {
770 context: "test".to_string(),
771 integration_id: Some(15368),
772 }]
773 );
774 }
775
776 #[tokio::test]
777 async fn hold_after_merge_records_pending_for_a_bootstrap_task() {
778 let store = Arc::new(TaskStore::open_memory().await.unwrap());
779 let task = ExternalTask {
780 id: ExternalId("12".to_string()),
781 title: "CI-01 Set up CI and tests".to_string(),
782 body: bootstrap_body(),
783 state: "open".to_string(),
784 labels: vec![],
785 author: String::new(),
786 created_at: String::new(),
787 updated_at: String::new(),
788 url: String::new(),
789 };
790 assert!(hold_after_merge(&task, &store, "o/r").await);
791 assert_eq!(
792 store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
793 Some(KV_PENDING)
794 );
795 store
796 .kv_set(&kv_key("o/r", "12"), KV_APPLIED)
797 .await
798 .unwrap();
799 assert!(!hold_after_merge(&task, &store, "o/r").await);
800 }
801
802 #[tokio::test]
803 async fn hold_after_merge_skips_a_title_without_a_plan_marker() {
804 let store = Arc::new(TaskStore::open_memory().await.unwrap());
805 let task = ExternalTask {
806 id: ExternalId("12".to_string()),
807 title: "CI-01 Set up CI and tests".to_string(),
808 body: "ordinary issue".to_string(),
809 state: "open".to_string(),
810 labels: vec![],
811 author: String::new(),
812 created_at: String::new(),
813 updated_at: String::new(),
814 url: String::new(),
815 };
816 assert!(!hold_after_merge(&task, &store, "o/r").await);
817 assert!(store.kv_get(&kv_key("o/r", "12")).await.unwrap().is_none());
818 }
819
820 #[tokio::test]
821 async fn apply_pending_dones_when_required_checks_already_exist() {
822 let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
823 let server = MockServer::start().await;
824 Mock::given(method("GET"))
825 .and(path("/repos/o/r"))
826 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
827 "default_branch": "main"
828 })))
829 .mount(&server)
830 .await;
831 Mock::given(method("GET"))
832 .and(path("/repos/o/r/branches/main/protection"))
833 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
834 "required_status_checks": {
835 "checks": [{ "context": "test", "app_id": 15368 }]
836 }
837 })))
838 .mount(&server)
839 .await;
840 Mock::given(method("GET"))
841 .and(path("/repos/o/r/rules/branches/main"))
842 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
843 .mount(&server)
844 .await;
845
846 let store = Arc::new(TaskStore::open_memory().await.unwrap());
847 let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
848 let task_manager = Arc::new(TaskManager::with_store(
849 backend,
850 Arc::clone(&store),
851 "o/r".to_string(),
852 ));
853 let id = store
854 .upsert_external(&UpsertExternal {
855 repo: "o/r",
856 ext_id: "12",
857 title: "CI-01 Set up CI and tests",
858 body: &bootstrap_body(),
859 author: "u",
860 url: "",
861 labels: &[],
862 origin: "github",
863 })
864 .await
865 .unwrap();
866 store.update_status(id, TaskStatus::Approved).await.unwrap();
867 store
868 .kv_set(&kv_key("o/r", "12"), KV_PENDING)
869 .await
870 .unwrap();
871
872 let gh = GhHttp::for_test(server.uri(), None);
873 apply_pending_with(&gh, &store, &task_manager, "o/r")
874 .await
875 .unwrap();
876
877 let stored = store.get(id).await.unwrap();
878 assert_eq!(stored.status, TaskStatus::Done);
879 assert_eq!(
880 store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
881 Some(KV_APPLIED)
882 );
883 }
884
885 #[tokio::test]
886 async fn apply_pending_creates_a_ruleset_when_default_branch_checks_pass() {
887 let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
888 let server = MockServer::start().await;
889 Mock::given(method("GET"))
890 .and(path("/repos/o/r"))
891 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
892 "default_branch": "main"
893 })))
894 .mount(&server)
895 .await;
896 Mock::given(method("GET"))
897 .and(path("/repos/o/r/branches/main/protection"))
898 .respond_with(
899 ResponseTemplate::new(404)
900 .set_body_string("{\"message\":\"Branch not protected\"}"),
901 )
902 .mount(&server)
903 .await;
904 Mock::given(method("GET"))
905 .and(path("/repos/o/r/rules/branches/main"))
906 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
907 .mount(&server)
908 .await;
909 Mock::given(method("GET"))
910 .and(path("/repos/o/r/branches/main"))
911 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
912 "commit": { "sha": "deadbeef" }
913 })))
914 .mount(&server)
915 .await;
916 Mock::given(method("GET"))
917 .and(path("/repos/o/r/commits/deadbeef/check-runs"))
918 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
919 "total_count": 1,
920 "check_runs": [{
921 "id": 1,
922 "name": "test",
923 "status": "completed",
924 "conclusion": "success",
925 "app": { "id": 15368 }
926 }]
927 })))
928 .mount(&server)
929 .await;
930 Mock::given(method("GET"))
931 .and(path("/repos/o/r/commits/deadbeef/status"))
932 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
933 "state": "success",
934 "total_count": 0,
935 "statuses": []
936 })))
937 .mount(&server)
938 .await;
939 Mock::given(method("POST"))
940 .and(path("/repos/o/r/rulesets"))
941 .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
942 "id": 9,
943 "name": "orch required checks"
944 })))
945 .expect(1)
946 .mount(&server)
947 .await;
948
949 let store = Arc::new(TaskStore::open_memory().await.unwrap());
950 let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
951 let task_manager = Arc::new(TaskManager::with_store(
952 backend,
953 Arc::clone(&store),
954 "o/r".to_string(),
955 ));
956 let id = store
957 .upsert_external(&UpsertExternal {
958 repo: "o/r",
959 ext_id: "12",
960 title: "CI-01 Set up CI and tests",
961 body: &bootstrap_body(),
962 author: "u",
963 url: "",
964 labels: &[],
965 origin: "github",
966 })
967 .await
968 .unwrap();
969 store.update_status(id, TaskStatus::Approved).await.unwrap();
970 store
971 .kv_set(&kv_key("o/r", "12"), KV_PENDING)
972 .await
973 .unwrap();
974
975 let gh = GhHttp::for_test(server.uri(), None);
976 apply_pending_with(&gh, &store, &task_manager, "o/r")
977 .await
978 .unwrap();
979
980 let stored = store.get(id).await.unwrap();
981 assert_eq!(stored.status, TaskStatus::Done);
982 assert_eq!(
983 store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
984 Some(KV_APPLIED)
985 );
986 }
987
988 #[tokio::test]
989 async fn apply_pending_comments_and_holds_when_github_refuses_the_ruleset() {
990 let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
991 let server = MockServer::start().await;
992 Mock::given(method("GET"))
993 .and(path("/repos/o/r"))
994 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
995 "default_branch": "main"
996 })))
997 .mount(&server)
998 .await;
999 Mock::given(method("GET"))
1000 .and(path("/repos/o/r/branches/main/protection"))
1001 .respond_with(
1002 ResponseTemplate::new(404)
1003 .set_body_string("{\"message\":\"Branch not protected\"}"),
1004 )
1005 .mount(&server)
1006 .await;
1007 Mock::given(method("GET"))
1008 .and(path("/repos/o/r/rules/branches/main"))
1009 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
1010 .mount(&server)
1011 .await;
1012 Mock::given(method("GET"))
1013 .and(path("/repos/o/r/branches/main"))
1014 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1015 "commit": { "sha": "deadbeef" }
1016 })))
1017 .mount(&server)
1018 .await;
1019 Mock::given(method("GET"))
1020 .and(path("/repos/o/r/commits/deadbeef/check-runs"))
1021 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1022 "total_count": 1,
1023 "check_runs": [{
1024 "id": 1,
1025 "name": "test",
1026 "status": "completed",
1027 "conclusion": "success",
1028 "app": { "id": 15368 }
1029 }]
1030 })))
1031 .mount(&server)
1032 .await;
1033 Mock::given(method("GET"))
1034 .and(path("/repos/o/r/commits/deadbeef/status"))
1035 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1036 "state": "success",
1037 "total_count": 0,
1038 "statuses": []
1039 })))
1040 .mount(&server)
1041 .await;
1042 Mock::given(method("POST"))
1043 .and(path("/repos/o/r/rulesets"))
1044 .respond_with(ResponseTemplate::new(403).set_body_json(serde_json::json!({
1045 "message": "Upgrade to GitHub Pro or make this repository public to enable this feature."
1046 })))
1047 .mount(&server)
1048 .await;
1049 Mock::given(method("POST"))
1050 .and(path("/repos/o/r/issues/12/comments"))
1051 .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1052 "id": 1
1053 })))
1054 .expect(1)
1055 .mount(&server)
1056 .await;
1057
1058 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1059 let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
1060 let task_manager = Arc::new(TaskManager::with_store(
1061 backend,
1062 Arc::clone(&store),
1063 "o/r".to_string(),
1064 ));
1065 let id = store
1066 .upsert_external(&UpsertExternal {
1067 repo: "o/r",
1068 ext_id: "12",
1069 title: "CI-01 Set up CI and tests",
1070 body: &bootstrap_body(),
1071 author: "u",
1072 url: "",
1073 labels: &[],
1074 origin: "github",
1075 })
1076 .await
1077 .unwrap();
1078 store.update_status(id, TaskStatus::Approved).await.unwrap();
1079 store
1080 .kv_set(&kv_key("o/r", "12"), KV_PENDING)
1081 .await
1082 .unwrap();
1083
1084 let gh = GhHttp::for_test(server.uri(), None);
1085 apply_pending_with(&gh, &store, &task_manager, "o/r")
1086 .await
1087 .unwrap();
1088
1089 let stored = store.get(id).await.unwrap();
1090 assert_eq!(stored.status, TaskStatus::Approved);
1091 assert_eq!(
1092 store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1093 Some(KV_REFUSED)
1094 );
1095 }
1096
1097 #[tokio::test]
1098 async fn apply_pending_reopens_a_closed_issue_and_creates_the_ruleset() {
1099 let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1100 let server = MockServer::start().await;
1101 Mock::given(method("GET"))
1102 .and(path("/repos/o/r"))
1103 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1104 "default_branch": "main"
1105 })))
1106 .mount(&server)
1107 .await;
1108 Mock::given(method("GET"))
1109 .and(path("/repos/o/r/branches/main/protection"))
1110 .respond_with(
1111 ResponseTemplate::new(404)
1112 .set_body_string("{\"message\":\"Branch not protected\"}"),
1113 )
1114 .mount(&server)
1115 .await;
1116 Mock::given(method("GET"))
1117 .and(path("/repos/o/r/rules/branches/main"))
1118 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
1119 .mount(&server)
1120 .await;
1121 Mock::given(method("GET"))
1122 .and(path("/repos/o/r/issues/12"))
1123 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1124 "number": 12,
1125 "id": 120,
1126 "title": "CI-01 Set up CI and tests",
1127 "body": "",
1128 "state": "closed",
1129 "labels": [],
1130 "user": { "login": "u" },
1131 "created_at": "",
1132 "updated_at": "",
1133 "html_url": ""
1134 })))
1135 .mount(&server)
1136 .await;
1137 Mock::given(method("PATCH"))
1138 .and(path("/repos/o/r/issues/12"))
1139 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1140 "number": 12,
1141 "id": 120,
1142 "title": "CI-01 Set up CI and tests",
1143 "body": "",
1144 "state": "open",
1145 "labels": [],
1146 "user": { "login": "u" },
1147 "created_at": "",
1148 "updated_at": "",
1149 "html_url": ""
1150 })))
1151 .expect(1)
1152 .mount(&server)
1153 .await;
1154 Mock::given(method("GET"))
1155 .and(path("/repos/o/r/branches/main"))
1156 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1157 "commit": { "sha": "deadbeef" }
1158 })))
1159 .mount(&server)
1160 .await;
1161 Mock::given(method("GET"))
1162 .and(path("/repos/o/r/commits/deadbeef/check-runs"))
1163 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1164 "total_count": 1,
1165 "check_runs": [{
1166 "id": 1,
1167 "name": "test",
1168 "status": "completed",
1169 "conclusion": "success",
1170 "app": { "id": 15368 }
1171 }]
1172 })))
1173 .mount(&server)
1174 .await;
1175 Mock::given(method("GET"))
1176 .and(path("/repos/o/r/commits/deadbeef/status"))
1177 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1178 "state": "success",
1179 "total_count": 0,
1180 "statuses": []
1181 })))
1182 .mount(&server)
1183 .await;
1184 Mock::given(method("POST"))
1185 .and(path("/repos/o/r/rulesets"))
1186 .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1187 "id": 9,
1188 "name": "orch required checks"
1189 })))
1190 .expect(1)
1191 .mount(&server)
1192 .await;
1193
1194 let store = Arc::new(TaskStore::open_memory().await.unwrap());
1195 let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
1196 let task_manager = Arc::new(TaskManager::with_store(
1197 backend,
1198 Arc::clone(&store),
1199 "o/r".to_string(),
1200 ));
1201 let id = store
1202 .upsert_external(&UpsertExternal {
1203 repo: "o/r",
1204 ext_id: "12",
1205 title: "CI-01 Set up CI and tests",
1206 body: &bootstrap_body(),
1207 author: "u",
1208 url: "",
1209 labels: &[],
1210 origin: "github",
1211 })
1212 .await
1213 .unwrap();
1214 store.update_status(id, TaskStatus::Done).await.unwrap();
1215 store
1216 .kv_set(&kv_key("o/r", "12"), KV_PENDING)
1217 .await
1218 .unwrap();
1219
1220 let gh = GhHttp::for_test(server.uri(), None);
1221 apply_pending_with(&gh, &store, &task_manager, "o/r")
1222 .await
1223 .unwrap();
1224
1225 let stored = store.get(id).await.unwrap();
1226 assert_eq!(stored.status, TaskStatus::Done);
1227 assert_eq!(
1228 store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1229 Some(KV_APPLIED)
1230 );
1231 }
1232
1233 #[tokio::test]
1234 async fn apply_pending_logs_and_holds_while_default_branch_checks_run() {
1235 let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1236 let server = MockServer::start().await;
1237 mock_unprotected_main(&server, "deadbeef").await;
1238 mock_runs_and_empty_status(
1239 &server,
1240 "deadbeef",
1241 serde_json::json!([{
1242 "id": 1,
1243 "name": "test",
1244 "status": "in_progress",
1245 "conclusion": null,
1246 "app": { "id": 15368 }
1247 }]),
1248 )
1249 .await;
1250 mock_issue_comment(&server, 0).await;
1251 Mock::given(method("POST"))
1252 .and(path("/repos/o/r/rulesets"))
1253 .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1254 "id": 9
1255 })))
1256 .expect(0)
1257 .mount(&server)
1258 .await;
1259
1260 let (store, task_manager, id) = pending_store().await;
1261 let gh = GhHttp::for_test(server.uri(), None);
1262 apply_pending_with(&gh, &store, &task_manager, "o/r")
1263 .await
1264 .unwrap();
1265
1266 let stored = store.get(id).await.unwrap();
1267 assert_eq!(stored.status, TaskStatus::Approved);
1268 assert_eq!(
1269 store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1270 Some(KV_PENDING)
1271 );
1272 }
1273
1274 #[tokio::test]
1275 async fn apply_pending_comments_and_routes_when_default_branch_checks_fail() {
1276 let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1277 let server = MockServer::start().await;
1278 mock_unprotected_main(&server, "deadbeef").await;
1279 mock_runs_and_empty_status(
1280 &server,
1281 "deadbeef",
1282 serde_json::json!([{
1283 "id": 1,
1284 "name": "test",
1285 "status": "completed",
1286 "conclusion": "failure",
1287 "app": { "id": 15368 }
1288 }]),
1289 )
1290 .await;
1291 mock_issue_comment(&server, 1).await;
1292
1293 let (store, task_manager, id) = pending_store().await;
1294 let gh = GhHttp::for_test(server.uri(), None);
1295 apply_pending_with(&gh, &store, &task_manager, "o/r")
1296 .await
1297 .unwrap();
1298
1299 let stored = store.get(id).await.unwrap();
1300 assert_eq!(stored.status, TaskStatus::Routed);
1301 assert_eq!(stored.ci_merge_failures, 1);
1302 assert!(stored.last_error.contains("test: failure"));
1303 assert_eq!(
1304 store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1305 Some(KV_PENDING)
1306 );
1307
1308 apply_pending_with(&gh, &store, &task_manager, "o/r")
1309 .await
1310 .unwrap();
1311 let stored = store.get(id).await.unwrap();
1312 assert_eq!(stored.status, TaskStatus::Routed);
1313 assert_eq!(stored.ci_merge_failures, 1);
1314 }
1315
1316 #[tokio::test]
1317 async fn apply_pending_blocks_after_max_failed_default_branch_heads() {
1318 let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1319 let server = MockServer::start().await;
1320 mock_unprotected_main(&server, "deadbeef").await;
1321 mock_runs_and_empty_status(
1322 &server,
1323 "deadbeef",
1324 serde_json::json!([{
1325 "id": 1,
1326 "name": "test",
1327 "status": "completed",
1328 "conclusion": "cancelled",
1329 "app": { "id": 15368 }
1330 }]),
1331 )
1332 .await;
1333 mock_issue_comment(&server, 1).await;
1334
1335 let (store, task_manager, id) = pending_store().await;
1336 store.increment(id, "ci_merge_failures").await.unwrap();
1337 store.increment(id, "ci_merge_failures").await.unwrap();
1338 let gh = GhHttp::for_test(server.uri(), None);
1339 apply_pending_with(&gh, &store, &task_manager, "o/r")
1340 .await
1341 .unwrap();
1342
1343 let stored = store.get(id).await.unwrap();
1344 assert_eq!(stored.status, TaskStatus::Blocked);
1345 assert_eq!(
1346 stored.ci_merge_failures,
1347 i32::try_from(auto_merge::MAX_CI_MERGE_FAILURES).unwrap()
1348 );
1349 assert!(
1350 stored
1351 .block_reason
1352 .as_deref()
1353 .unwrap_or("")
1354 .contains("CI failure limit")
1355 );
1356 }
1357
1358 #[tokio::test]
1359 async fn apply_pending_waits_before_commenting_when_no_check_starts() {
1360 let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1361 let server = MockServer::start().await;
1362 mock_unprotected_main(&server, "deadbeef").await;
1363 mock_runs_and_empty_status(&server, "deadbeef", serde_json::json!([])).await;
1364 mock_issue_comment(&server, 0).await;
1365
1366 let (store, task_manager, id) = pending_store().await;
1367 let gh = GhHttp::for_test(server.uri(), None);
1368 apply_pending_with(&gh, &store, &task_manager, "o/r")
1369 .await
1370 .unwrap();
1371
1372 let stored = store.get(id).await.unwrap();
1373 assert_eq!(stored.status, TaskStatus::Approved);
1374 assert!(
1375 store
1376 .kv_get("ci_bootstrap_wait_since:o/r#12#deadbeef")
1377 .await
1378 .unwrap()
1379 .is_some()
1380 );
1381 assert!(
1382 store
1383 .kv_get(&no_checks_comment_key("o/r", "12", "deadbeef"))
1384 .await
1385 .unwrap()
1386 .is_none()
1387 );
1388 }
1389
1390 #[tokio::test]
1391 async fn apply_pending_comments_once_when_no_check_starts_after_the_wait() {
1392 let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1393 let server = MockServer::start().await;
1394 mock_unprotected_main(&server, "deadbeef").await;
1395 mock_runs_and_empty_status(&server, "deadbeef", serde_json::json!([])).await;
1396 mock_issue_comment(&server, 1).await;
1397
1398 let (store, task_manager, id) = pending_store().await;
1399 let started = chrono::Utc::now().timestamp() - review_gate::NO_CHECKS_WAIT_SECS - 1;
1400 store
1401 .kv_set(
1402 "ci_bootstrap_wait_since:o/r#12#deadbeef",
1403 &started.to_string(),
1404 )
1405 .await
1406 .unwrap();
1407 let gh = GhHttp::for_test(server.uri(), None);
1408 apply_pending_with(&gh, &store, &task_manager, "o/r")
1409 .await
1410 .unwrap();
1411
1412 let stored = store.get(id).await.unwrap();
1413 assert_eq!(stored.status, TaskStatus::Approved);
1414 assert_eq!(
1415 store
1416 .kv_get(&no_checks_comment_key("o/r", "12", "deadbeef"))
1417 .await
1418 .unwrap()
1419 .as_deref(),
1420 Some("1")
1421 );
1422
1423 apply_pending_with(&gh, &store, &task_manager, "o/r")
1424 .await
1425 .unwrap();
1426 assert_eq!(store.get(id).await.unwrap().status, TaskStatus::Approved);
1427 }
1428
1429 #[tokio::test]
1430 async fn apply_pending_comments_once_when_github_says_the_check_never_ran() {
1431 let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1432 let server = MockServer::start().await;
1433 mock_unprotected_main(&server, "deadbeef").await;
1434 mock_runs_and_empty_status(
1435 &server,
1436 "deadbeef",
1437 serde_json::json!([{
1438 "id": 1,
1439 "name": "test",
1440 "status": "completed",
1441 "conclusion": "success",
1442 "app": { "id": 15368 }
1443 }]),
1444 )
1445 .await;
1446 Mock::given(method("POST"))
1447 .and(path("/repos/o/r/rulesets"))
1448 .respond_with(ResponseTemplate::new(422).set_body_json(serde_json::json!({
1449 "message": "test was not a valid check"
1450 })))
1451 .mount(&server)
1452 .await;
1453 mock_issue_comment(&server, 1).await;
1454
1455 let (store, task_manager, id) = pending_store().await;
1456 let gh = GhHttp::for_test(server.uri(), None);
1457 apply_pending_with(&gh, &store, &task_manager, "o/r")
1458 .await
1459 .unwrap();
1460 apply_pending_with(&gh, &store, &task_manager, "o/r")
1461 .await
1462 .unwrap();
1463
1464 let stored = store.get(id).await.unwrap();
1465 assert_eq!(stored.status, TaskStatus::Approved);
1466 assert_eq!(
1467 store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1468 Some(KV_PENDING)
1469 );
1470 assert_eq!(
1471 store
1472 .kv_get(&never_run_comment_key("o/r", "12"))
1473 .await
1474 .unwrap()
1475 .as_deref(),
1476 Some("1")
1477 );
1478 }
1479}