Skip to main content

orch/engine/
ci_bootstrap.rs

1//! After a CI bootstrap task merges, require its default-branch checks
2
3use crate::backends::{ExternalId, ExternalTask, Status};
4use crate::engine;
5use crate::engine::auto_merge;
6use crate::engine::review_gate;
7use crate::engine::tasks::TaskManager;
8use crate::github::http::{GhHttp, RulesetCreateKind, classify_ruleset_create_error};
9use crate::github::types::{GitHubCheckRun, RequiredStatusCheck};
10use crate::plan;
11use crate::store;
12use crate::store::TaskStore;
13use std::sync::Arc;
14
15const KV_PENDING: &str = "pending";
16const KV_APPLIED: &str = "applied";
17const KV_REFUSED: &str = "refused";
18
19#[derive(Debug, PartialEq)]
20pub(crate) enum ChecksReady {
21    Ready(Vec<RequiredStatusCheck>),
22    /// A real check is still queued or in progress
23    Pending,
24    /// A real check finished as failure, error, cancelled, or timed out
25    Failed(Vec<FailedCheck>),
26    /// No real check has started
27    Empty,
28}
29
30#[derive(Debug, PartialEq)]
31pub(crate) struct FailedCheck {
32    name: String,
33    conclusion: String,
34}
35
36pub(crate) fn is_ci_bootstrap_task(body: &str) -> bool {
37    plan::is_ci_bootstrap_body(body)
38        || plan::marker_id(body).is_some_and(|id| plan::is_ci_bootstrap_id(&id))
39}
40
41pub(crate) fn kv_key(repo: &str, task_id: &str) -> String {
42    format!("ci-bootstrap:{repo}:{task_id}")
43}
44
45/// True when this merged CI bootstrap task must stay approved until a ruleset exists
46pub(crate) async fn hold_after_merge(
47    task: &ExternalTask,
48    store: &Arc<TaskStore>,
49    repo: &str,
50) -> bool {
51    if !is_ci_bootstrap_task(&task.body) {
52        return false;
53    }
54    let key = kv_key(repo, &task.id.0);
55    match store.kv_get(&key).await {
56        Ok(Some(state)) if state == KV_APPLIED => false,
57        Ok(Some(_)) => true,
58        Ok(None) => {
59            if let Err(e) = store.kv_set(&key, KV_PENDING).await {
60                tracing::warn!(
61                    task_id = %task.id.0,
62                    err = %e,
63                    "failed to record CI bootstrap pending"
64                );
65            }
66            true
67        }
68        Err(e) => {
69            tracing::warn!(
70                task_id = %task.id.0,
71                err = %e,
72                "failed to read CI bootstrap state"
73            );
74            true
75        }
76    }
77}
78
79pub(crate) async fn apply_pending(
80    store: &Arc<TaskStore>,
81    task_manager: &Arc<TaskManager>,
82    repo: &str,
83) -> anyhow::Result<()> {
84    let gh = GhHttp::owner()?;
85    apply_pending_with(&gh, store, task_manager, repo).await
86}
87
88pub(crate) async fn apply_pending_with(
89    gh: &GhHttp,
90    store: &Arc<TaskStore>,
91    task_manager: &Arc<TaskManager>,
92    repo: &str,
93) -> anyhow::Result<()> {
94    let prefix = format!("ci-bootstrap:{repo}:");
95    let entries = store.kv_list_prefix(&prefix).await?;
96    for (key, state) in entries {
97        if state == KV_APPLIED {
98            continue;
99        }
100        let Some(task_id) = key.strip_prefix(&prefix) else {
101            continue;
102        };
103        if let Err(e) = apply_one(gh, store, task_manager, repo, task_id, &state).await {
104            tracing::warn!(task_id, err = %e, "CI bootstrap ruleset apply failed");
105        }
106    }
107    Ok(())
108}
109
110pub(crate) fn checks_ready_for_ruleset(
111    runs: &[GitHubCheckRun],
112    statuses: &[(String, String)],
113) -> ChecksReady {
114    let mut ready: Vec<RequiredStatusCheck> = Vec::new();
115    let mut pending = false;
116    let mut failed: Vec<FailedCheck> = Vec::new();
117
118    for run in runs {
119        if review_gate::is_own_check(&run.name) {
120            continue;
121        }
122        match run.status.as_str() {
123            "completed" => match run.conclusion.as_deref() {
124                Some("success") => {
125                    let integration_id = run.app.as_ref().map(|app| app.id).filter(|&id| id > 0);
126                    if !ready.iter().any(|check| check.context == run.name) {
127                        ready.push(RequiredStatusCheck {
128                            context: run.name.clone(),
129                            integration_id,
130                        });
131                    }
132                }
133                Some("skipped" | "neutral") => {}
134                Some(conclusion) => failed.push(FailedCheck {
135                    name: run.name.clone(),
136                    conclusion: conclusion.to_string(),
137                }),
138                None => pending = true,
139            },
140            _ => pending = true,
141        }
142    }
143
144    for (context, state) in statuses {
145        if review_gate::is_own_check(context) {
146            continue;
147        }
148        match state.as_str() {
149            "pending" => pending = true,
150            "success" => {
151                if !ready.iter().any(|check| &check.context == context) {
152                    ready.push(RequiredStatusCheck {
153                        context: context.clone(),
154                        integration_id: None,
155                    });
156                }
157            }
158            "failure" | "error" => failed.push(FailedCheck {
159                name: context.clone(),
160                conclusion: state.clone(),
161            }),
162            _ => {}
163        }
164    }
165
166    if !failed.is_empty() {
167        ChecksReady::Failed(failed)
168    } else if pending {
169        ChecksReady::Pending
170    } else if ready.is_empty() {
171        ChecksReady::Empty
172    } else {
173        ChecksReady::Ready(ready)
174    }
175}
176
177async fn apply_one(
178    gh: &GhHttp,
179    store: &Arc<TaskStore>,
180    task_manager: &Arc<TaskManager>,
181    repo: &str,
182    task_id: &str,
183    state: &str,
184) -> anyhow::Result<()> {
185    if store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
186        .await
187        .is_none()
188    {
189        return Ok(());
190    }
191
192    let branch = gh.get_default_branch(repo).await?;
193    let required = gh.get_required_status_check_contexts(repo, &branch).await?;
194    if !required.is_empty() {
195        return finish_applied(store, task_manager, task_id, repo).await;
196    }
197
198    // GitHub may have closed the issue via Closes on the PR, keep it open until the ruleset exists
199    reopen_if_closed(gh, repo, task_id).await;
200
201    if state == KV_REFUSED {
202        return Ok(());
203    }
204
205    let sha = gh.get_branch_sha(repo, &branch).await?;
206    let runs = gh.get_check_runs(repo, &sha).await?;
207    let statuses = gh.get_commit_status_contexts(repo, &sha).await?;
208    let checks = match checks_ready_for_ruleset(&runs, &statuses) {
209        ChecksReady::Pending => {
210            tracing::info!(
211                task_id,
212                sha = %sha,
213                "CI bootstrap waiting for default-branch checks"
214            );
215            clear_wait(store, repo, task_id).await;
216            return Ok(());
217        }
218        ChecksReady::Empty => {
219            return wait_for_checks(gh, store, repo, task_id, &sha).await;
220        }
221        ChecksReady::Failed(failed) => {
222            clear_wait(store, repo, task_id).await;
223            return handle_failed_checks(gh, store, task_manager, repo, task_id, &sha, &failed)
224                .await;
225        }
226        ChecksReady::Ready(checks) => {
227            clear_wait(store, repo, task_id).await;
228            checks
229        }
230    };
231
232    match gh.create_required_check_ruleset(repo, &checks).await {
233        Ok(()) => finish_applied(store, task_manager, task_id, repo).await,
234        Err(e) => {
235            let err = e.to_string();
236            match classify_ruleset_create_error(&err) {
237                RulesetCreateKind::NameTaken => {
238                    finish_applied(store, task_manager, task_id, repo).await
239                }
240                RulesetCreateKind::Retryable => {
241                    tracing::info!(
242                        task_id,
243                        err = %err,
244                        "CI bootstrap ruleset create retrying"
245                    );
246                    Ok(())
247                }
248                RulesetCreateKind::CheckNeverRun => {
249                    tracing::info!(
250                        task_id,
251                        err = %err,
252                        "CI bootstrap ruleset create waiting, check has not run"
253                    );
254                    comment_never_run_once(gh, store, repo, task_id, &err).await;
255                    Ok(())
256                }
257                RulesetCreateKind::Refused | RulesetCreateKind::Other => {
258                    comment_refused(gh, repo, task_id, &err).await;
259                    store.kv_set(&kv_key(repo, task_id), KV_REFUSED).await?;
260                    Ok(())
261                }
262            }
263        }
264    }
265}
266
267async fn finish_applied(
268    store: &Arc<TaskStore>,
269    task_manager: &Arc<TaskManager>,
270    task_id: &str,
271    repo: &str,
272) -> anyhow::Result<()> {
273    store.kv_set(&kv_key(repo, task_id), KV_APPLIED).await?;
274    task_manager
275        .update_task_status(&ExternalId(task_id.to_string()), Status::Done)
276        .await?;
277    Ok(())
278}
279
280async fn reopen_if_closed(gh: &GhHttp, repo: &str, task_id: &str) {
281    match gh.get_issue(repo, task_id).await {
282        Ok(issue) if issue.state.eq_ignore_ascii_case("closed") => {
283            if let Err(e) = gh.reopen_issue(repo, task_id).await {
284                tracing::warn!(
285                    task_id,
286                    err = %e,
287                    "failed to reopen CI bootstrap issue"
288                );
289            }
290        }
291        Ok(_) => {}
292        Err(e) => {
293            tracing::warn!(
294                task_id,
295                err = %e,
296                "failed to read CI bootstrap issue"
297            );
298        }
299    }
300}
301
302fn wait_prefix(repo: &str, task_id: &str) -> String {
303    format!("ci_bootstrap_wait_since:{repo}#{task_id}#")
304}
305
306fn no_checks_comment_key(repo: &str, task_id: &str, sha: &str) -> String {
307    format!("ci_bootstrap_no_checks_comment:{repo}#{task_id}#{sha}")
308}
309
310fn never_run_comment_key(repo: &str, task_id: &str) -> String {
311    format!("ci_bootstrap_never_run_comment:{repo}#{task_id}")
312}
313
314fn format_failed(failed: &[FailedCheck]) -> String {
315    failed
316        .iter()
317        .map(|check| format!("{}: {}", check.name, check.conclusion))
318        .collect::<Vec<_>>()
319        .join(", ")
320}
321
322async fn clear_wait(store: &Arc<TaskStore>, repo: &str, task_id: &str) {
323    let prefix = wait_prefix(repo, task_id);
324    let keys = match store.kv_list_prefix(&prefix).await {
325        Ok(entries) => entries,
326        Err(e) => {
327            tracing::warn!(task_id, err = %e, "failed to list CI bootstrap wait keys");
328            return;
329        }
330    };
331    for (key, _) in keys {
332        if let Err(e) = store.kv_delete(&key).await {
333            tracing::warn!(task_id, err = %e, "failed to clear CI bootstrap wait key");
334        }
335    }
336}
337
338async fn wait_for_checks(
339    gh: &GhHttp,
340    store: &Arc<TaskStore>,
341    repo: &str,
342    task_id: &str,
343    sha: &str,
344) -> anyhow::Result<()> {
345    let prefix = wait_prefix(repo, task_id);
346    let key = format!("{prefix}{sha}");
347    let now = chrono::Utc::now().timestamp();
348    let since = review_gate::wait_since(store, &prefix, &key, now).await?;
349    if now - since < review_gate::NO_CHECKS_WAIT_SECS {
350        tracing::info!(
351            task_id,
352            sha = %sha,
353            "CI bootstrap waiting for a default-branch check to start"
354        );
355        return Ok(());
356    }
357    let wait_mins = review_gate::NO_CHECKS_WAIT_SECS / 60;
358    tracing::warn!(
359        task_id,
360        sha = %sha,
361        wait_mins,
362        "CI bootstrap has no default-branch check after the wait"
363    );
364    let comment_key = no_checks_comment_key(repo, task_id, sha);
365    if store.kv_get(&comment_key).await?.is_none() {
366        comment_issue(
367            gh,
368            repo,
369            task_id,
370            &format!(
371                "No default-branch check has started {wait_mins} minutes after the CI bootstrap merge. Later tasks stay waiting until a check runs."
372            ),
373        )
374        .await;
375        store.kv_set(&comment_key, "1").await?;
376    }
377    Ok(())
378}
379
380async fn handle_failed_checks(
381    gh: &GhHttp,
382    store: &Arc<TaskStore>,
383    task_manager: &Arc<TaskManager>,
384    repo: &str,
385    task_id: &str,
386    sha: &str,
387    failed: &[FailedCheck],
388) -> anyhow::Result<()> {
389    let details = format_failed(failed);
390    let failures = match auto_merge::count_ci_failure_for_head(store, repo, task_id, sha).await {
391        Ok(Some(v)) => v,
392        Ok(None) => {
393            tracing::info!(
394                task_id,
395                sha = %sha,
396                details,
397                "CI bootstrap default-branch checks still failed on the same head"
398            );
399            return Ok(());
400        }
401        Err(e) => {
402            tracing::warn!(
403                task_id,
404                err = %e,
405                "failed to increment ci_merge_failures for CI bootstrap"
406            );
407            return Ok(());
408        }
409    };
410
411    comment_issue(
412        gh,
413        repo,
414        task_id,
415        &format!(
416            "Default-branch checks failed after the CI bootstrap merge ({details}). Later tasks stay waiting until those checks pass."
417        ),
418    )
419    .await;
420
421    let max = auto_merge::MAX_CI_MERGE_FAILURES;
422    let id = ExternalId(task_id.to_string());
423    if failures >= max {
424        tracing::error!(
425            task_id,
426            failures,
427            details,
428            "CI bootstrap default-branch checks failed, CI failure limit reached"
429        );
430        let fields = [(
431            "block_reason",
432            serde_json::json!(format!(
433                "CI failure limit ({max}) reached after the CI bootstrap merge"
434            )),
435        )];
436        if let Err(e) = task_manager
437            .update_task_status_and_result(&id, Status::Blocked, &fields)
438            .await
439        {
440            tracing::error!(
441                task_id,
442                err = %e,
443                "failed to block CI bootstrap after failed default-branch checks"
444            );
445        }
446    } else {
447        tracing::warn!(
448            task_id,
449            failures,
450            details,
451            "CI bootstrap default-branch checks failed, routing back to the task agent"
452        );
453        let fields = [
454            (
455                "last_error",
456                serde_json::json!(format!(
457                    "CI failed on the default branch after the CI bootstrap merge: {details}. Fix the failing checks"
458                )),
459            ),
460            (
461                "route_reason",
462                serde_json::json!("re-dispatch after CI bootstrap default-branch checks failed"),
463            ),
464        ];
465        if let Err(e) = task_manager
466            .update_task_status_and_result(&id, Status::Routed, &fields)
467            .await
468        {
469            tracing::error!(
470                task_id,
471                err = %e,
472                "failed to re-route CI bootstrap after failed default-branch checks"
473            );
474        }
475    }
476    Ok(())
477}
478
479async fn comment_never_run_once(
480    gh: &GhHttp,
481    store: &Arc<TaskStore>,
482    repo: &str,
483    task_id: &str,
484    err: &str,
485) {
486    let key = never_run_comment_key(repo, task_id);
487    match store.kv_get(&key).await {
488        Ok(Some(_)) => return,
489        Ok(None) => {}
490        Err(e) => {
491            tracing::warn!(
492                task_id,
493                err = %e,
494                "failed to read CI bootstrap never-run comment key"
495            );
496            return;
497        }
498    }
499    comment_issue(
500        gh,
501        repo,
502        task_id,
503        &format!(
504            "GitHub does not treat the default-branch checks as valid required checks ({err}). Later tasks stay waiting until a required-checks ruleset can be created."
505        ),
506    )
507    .await;
508    if let Err(e) = store.kv_set(&key, "1").await {
509        tracing::warn!(
510            task_id,
511            err = %e,
512            "failed to record CI bootstrap never-run comment"
513        );
514    }
515}
516
517async fn comment_refused(gh: &GhHttp, repo: &str, task_id: &str, err: &str) {
518    comment_issue(
519        gh,
520        repo,
521        task_id,
522        &format!(
523            "GitHub refused a required-checks ruleset on the default branch ({err}). Turn branch protection on for this repository. Later tasks stay waiting until that happens."
524        ),
525    )
526    .await;
527}
528
529async fn comment_issue(gh: &GhHttp, repo: &str, task_id: &str, message: &str) {
530    let body = format!("{message}{}", engine::orch_footer());
531    if let Err(e) = gh.add_comment(repo, task_id, &body).await {
532        tracing::warn!(
533            task_id,
534            err = %e,
535            "failed to comment on CI bootstrap issue"
536        );
537    }
538}
539
540#[cfg(test)]
541mod tests {
542    use super::*;
543    use crate::backends::ExternalBackend;
544    use crate::backends::test_helpers::NoopBackend;
545    use crate::github::types::GitHubApp;
546    use crate::plan::{CI_BOOTSTRAP_MARKER, ci_bootstrap_task, issue_body};
547    use crate::store::TaskStatus;
548    use crate::store::UpsertExternal;
549    use std::env;
550    use std::ffi::OsString;
551    use wiremock::matchers::{method, path};
552    use wiremock::{Mock, MockServer, ResponseTemplate};
553
554    struct EnvVarGuard {
555        key: &'static str,
556        previous: Option<OsString>,
557    }
558
559    impl EnvVarGuard {
560        fn set(key: &'static str, value: &str) -> Self {
561            let previous = env::var_os(key);
562            // SAFETY: nextest runs each test in its own process
563            unsafe { env::set_var(key, value) };
564            Self { key, previous }
565        }
566    }
567
568    impl Drop for EnvVarGuard {
569        fn drop(&mut self) {
570            match self.previous.take() {
571                // SAFETY: nextest runs each test in its own process
572                Some(value) => unsafe { env::set_var(self.key, value) },
573                // SAFETY: nextest runs each test in its own process
574                None => unsafe { env::remove_var(self.key) },
575            }
576        }
577    }
578
579    fn run(
580        name: &str,
581        status: &str,
582        conclusion: Option<&str>,
583        app_id: Option<i64>,
584    ) -> GitHubCheckRun {
585        GitHubCheckRun {
586            id: 1,
587            name: name.to_string(),
588            status: status.to_string(),
589            conclusion: conclusion.map(String::from),
590            app: app_id.map(|id| GitHubApp { id }),
591        }
592    }
593
594    fn bootstrap_body() -> String {
595        issue_body(
596            &ci_bootstrap_task(false),
597            "acme/app",
598            "main",
599            "docs/ROADMAP.md",
600        )
601    }
602
603    async fn pending_store() -> (Arc<TaskStore>, Arc<TaskManager>, i64) {
604        let store = Arc::new(TaskStore::open_memory().await.unwrap());
605        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
606        let task_manager = Arc::new(TaskManager::with_store(
607            backend,
608            Arc::clone(&store),
609            "o/r".to_string(),
610        ));
611        let id = store
612            .upsert_external(&UpsertExternal {
613                repo: "o/r",
614                ext_id: "12",
615                title: "CI-01 Set up CI and tests",
616                body: &bootstrap_body(),
617                author: "u",
618                url: "",
619                labels: &[],
620                origin: "github",
621            })
622            .await
623            .unwrap();
624        store.update_status(id, TaskStatus::Approved).await.unwrap();
625        store
626            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
627            .await
628            .unwrap();
629        (store, task_manager, id)
630    }
631
632    async fn mock_unprotected_main(server: &MockServer, sha: &str) {
633        Mock::given(method("GET"))
634            .and(path("/repos/o/r"))
635            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
636                "default_branch": "main"
637            })))
638            .mount(server)
639            .await;
640        Mock::given(method("GET"))
641            .and(path("/repos/o/r/branches/main/protection"))
642            .respond_with(
643                ResponseTemplate::new(404)
644                    .set_body_string("{\"message\":\"Branch not protected\"}"),
645            )
646            .mount(server)
647            .await;
648        Mock::given(method("GET"))
649            .and(path("/repos/o/r/rules/branches/main"))
650            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
651            .mount(server)
652            .await;
653        Mock::given(method("GET"))
654            .and(path("/repos/o/r/branches/main"))
655            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
656                "commit": { "sha": sha }
657            })))
658            .mount(server)
659            .await;
660    }
661
662    async fn mock_runs_and_empty_status(
663        server: &MockServer,
664        sha: &str,
665        check_runs: serde_json::Value,
666    ) {
667        Mock::given(method("GET"))
668            .and(path(format!("/repos/o/r/commits/{sha}/check-runs")))
669            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
670                "total_count": 1,
671                "check_runs": check_runs
672            })))
673            .mount(server)
674            .await;
675        Mock::given(method("GET"))
676            .and(path(format!("/repos/o/r/commits/{sha}/status")))
677            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
678                "state": "pending",
679                "total_count": 0,
680                "statuses": []
681            })))
682            .mount(server)
683            .await;
684    }
685
686    async fn mock_issue_comment(server: &MockServer, expected: u64) {
687        Mock::given(method("POST"))
688            .and(path("/repos/o/r/issues/12/comments"))
689            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
690                "id": 1
691            })))
692            .expect(expected)
693            .mount(server)
694            .await;
695    }
696
697    #[test]
698    fn is_ci_bootstrap_task_reads_the_body_marker() {
699        assert!(is_ci_bootstrap_task(&bootstrap_body()));
700        assert!(is_ci_bootstrap_task(&format!("{CI_BOOTSTRAP_MARKER}\n")));
701        assert!(is_ci_bootstrap_task("goal\n<!-- orch-plan: CI-01 -->"));
702        assert!(!is_ci_bootstrap_task(""));
703        assert!(!is_ci_bootstrap_task("goal"));
704    }
705
706    #[test]
707    fn checks_ready_for_ruleset_splits_pending_failed_and_empty() {
708        assert_eq!(checks_ready_for_ruleset(&[], &[]), ChecksReady::Empty);
709        assert_eq!(
710            checks_ready_for_ruleset(&[run("test", "in_progress", None, None)], &[]),
711            ChecksReady::Pending
712        );
713        assert_eq!(
714            checks_ready_for_ruleset(&[run("test", "completed", Some("failure"), None)], &[]),
715            ChecksReady::Failed(vec![FailedCheck {
716                name: "test".to_string(),
717                conclusion: "failure".to_string(),
718            }])
719        );
720        assert_eq!(
721            checks_ready_for_ruleset(&[run("test", "completed", Some("cancelled"), None)], &[]),
722            ChecksReady::Failed(vec![FailedCheck {
723                name: "test".to_string(),
724                conclusion: "cancelled".to_string(),
725            }])
726        );
727        assert_eq!(
728            checks_ready_for_ruleset(&[], &[("ci/test".to_string(), "failure".to_string())]),
729            ChecksReady::Failed(vec![FailedCheck {
730                name: "ci/test".to_string(),
731                conclusion: "failure".to_string(),
732            }])
733        );
734        assert_eq!(
735            checks_ready_for_ruleset(&[run("test", "completed", Some("skipped"), None)], &[]),
736            ChecksReady::Empty
737        );
738        assert_eq!(
739            checks_ready_for_ruleset(
740                &[run("orch/review", "completed", Some("success"), None)],
741                &[]
742            ),
743            ChecksReady::Empty
744        );
745        assert_eq!(
746            checks_ready_for_ruleset(
747                &[
748                    run("test", "completed", Some("failure"), None),
749                    run("lint", "in_progress", None, None),
750                ],
751                &[]
752            ),
753            ChecksReady::Failed(vec![FailedCheck {
754                name: "test".to_string(),
755                conclusion: "failure".to_string(),
756            }])
757        );
758        let ChecksReady::Ready(ready) = checks_ready_for_ruleset(
759            &[
760                run("test", "completed", Some("success"), Some(15368)),
761                run("review-gate", "completed", Some("success"), None),
762            ],
763            &[],
764        ) else {
765            panic!("expected ready checks");
766        };
767        assert_eq!(
768            ready,
769            vec![RequiredStatusCheck {
770                context: "test".to_string(),
771                integration_id: Some(15368),
772            }]
773        );
774    }
775
776    #[tokio::test]
777    async fn hold_after_merge_records_pending_for_a_bootstrap_task() {
778        let store = Arc::new(TaskStore::open_memory().await.unwrap());
779        let task = ExternalTask {
780            id: ExternalId("12".to_string()),
781            title: "CI-01 Set up CI and tests".to_string(),
782            body: bootstrap_body(),
783            state: "open".to_string(),
784            labels: vec![],
785            author: String::new(),
786            created_at: String::new(),
787            updated_at: String::new(),
788            url: String::new(),
789        };
790        assert!(hold_after_merge(&task, &store, "o/r").await);
791        assert_eq!(
792            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
793            Some(KV_PENDING)
794        );
795        store
796            .kv_set(&kv_key("o/r", "12"), KV_APPLIED)
797            .await
798            .unwrap();
799        assert!(!hold_after_merge(&task, &store, "o/r").await);
800    }
801
802    #[tokio::test]
803    async fn hold_after_merge_skips_a_title_without_a_plan_marker() {
804        let store = Arc::new(TaskStore::open_memory().await.unwrap());
805        let task = ExternalTask {
806            id: ExternalId("12".to_string()),
807            title: "CI-01 Set up CI and tests".to_string(),
808            body: "ordinary issue".to_string(),
809            state: "open".to_string(),
810            labels: vec![],
811            author: String::new(),
812            created_at: String::new(),
813            updated_at: String::new(),
814            url: String::new(),
815        };
816        assert!(!hold_after_merge(&task, &store, "o/r").await);
817        assert!(store.kv_get(&kv_key("o/r", "12")).await.unwrap().is_none());
818    }
819
820    #[tokio::test]
821    async fn apply_pending_dones_when_required_checks_already_exist() {
822        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
823        let server = MockServer::start().await;
824        Mock::given(method("GET"))
825            .and(path("/repos/o/r"))
826            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
827                "default_branch": "main"
828            })))
829            .mount(&server)
830            .await;
831        Mock::given(method("GET"))
832            .and(path("/repos/o/r/branches/main/protection"))
833            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
834                "required_status_checks": {
835                    "checks": [{ "context": "test", "app_id": 15368 }]
836                }
837            })))
838            .mount(&server)
839            .await;
840        Mock::given(method("GET"))
841            .and(path("/repos/o/r/rules/branches/main"))
842            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
843            .mount(&server)
844            .await;
845
846        let store = Arc::new(TaskStore::open_memory().await.unwrap());
847        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
848        let task_manager = Arc::new(TaskManager::with_store(
849            backend,
850            Arc::clone(&store),
851            "o/r".to_string(),
852        ));
853        let id = store
854            .upsert_external(&UpsertExternal {
855                repo: "o/r",
856                ext_id: "12",
857                title: "CI-01 Set up CI and tests",
858                body: &bootstrap_body(),
859                author: "u",
860                url: "",
861                labels: &[],
862                origin: "github",
863            })
864            .await
865            .unwrap();
866        store.update_status(id, TaskStatus::Approved).await.unwrap();
867        store
868            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
869            .await
870            .unwrap();
871
872        let gh = GhHttp::for_test(server.uri(), None);
873        apply_pending_with(&gh, &store, &task_manager, "o/r")
874            .await
875            .unwrap();
876
877        let stored = store.get(id).await.unwrap();
878        assert_eq!(stored.status, TaskStatus::Done);
879        assert_eq!(
880            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
881            Some(KV_APPLIED)
882        );
883    }
884
885    #[tokio::test]
886    async fn apply_pending_creates_a_ruleset_when_default_branch_checks_pass() {
887        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
888        let server = MockServer::start().await;
889        Mock::given(method("GET"))
890            .and(path("/repos/o/r"))
891            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
892                "default_branch": "main"
893            })))
894            .mount(&server)
895            .await;
896        Mock::given(method("GET"))
897            .and(path("/repos/o/r/branches/main/protection"))
898            .respond_with(
899                ResponseTemplate::new(404)
900                    .set_body_string("{\"message\":\"Branch not protected\"}"),
901            )
902            .mount(&server)
903            .await;
904        Mock::given(method("GET"))
905            .and(path("/repos/o/r/rules/branches/main"))
906            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
907            .mount(&server)
908            .await;
909        Mock::given(method("GET"))
910            .and(path("/repos/o/r/branches/main"))
911            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
912                "commit": { "sha": "deadbeef" }
913            })))
914            .mount(&server)
915            .await;
916        Mock::given(method("GET"))
917            .and(path("/repos/o/r/commits/deadbeef/check-runs"))
918            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
919                "total_count": 1,
920                "check_runs": [{
921                    "id": 1,
922                    "name": "test",
923                    "status": "completed",
924                    "conclusion": "success",
925                    "app": { "id": 15368 }
926                }]
927            })))
928            .mount(&server)
929            .await;
930        Mock::given(method("GET"))
931            .and(path("/repos/o/r/commits/deadbeef/status"))
932            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
933                "state": "success",
934                "total_count": 0,
935                "statuses": []
936            })))
937            .mount(&server)
938            .await;
939        Mock::given(method("POST"))
940            .and(path("/repos/o/r/rulesets"))
941            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
942                "id": 9,
943                "name": "orch required checks"
944            })))
945            .expect(1)
946            .mount(&server)
947            .await;
948
949        let store = Arc::new(TaskStore::open_memory().await.unwrap());
950        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
951        let task_manager = Arc::new(TaskManager::with_store(
952            backend,
953            Arc::clone(&store),
954            "o/r".to_string(),
955        ));
956        let id = store
957            .upsert_external(&UpsertExternal {
958                repo: "o/r",
959                ext_id: "12",
960                title: "CI-01 Set up CI and tests",
961                body: &bootstrap_body(),
962                author: "u",
963                url: "",
964                labels: &[],
965                origin: "github",
966            })
967            .await
968            .unwrap();
969        store.update_status(id, TaskStatus::Approved).await.unwrap();
970        store
971            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
972            .await
973            .unwrap();
974
975        let gh = GhHttp::for_test(server.uri(), None);
976        apply_pending_with(&gh, &store, &task_manager, "o/r")
977            .await
978            .unwrap();
979
980        let stored = store.get(id).await.unwrap();
981        assert_eq!(stored.status, TaskStatus::Done);
982        assert_eq!(
983            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
984            Some(KV_APPLIED)
985        );
986    }
987
988    #[tokio::test]
989    async fn apply_pending_comments_and_holds_when_github_refuses_the_ruleset() {
990        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
991        let server = MockServer::start().await;
992        Mock::given(method("GET"))
993            .and(path("/repos/o/r"))
994            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
995                "default_branch": "main"
996            })))
997            .mount(&server)
998            .await;
999        Mock::given(method("GET"))
1000            .and(path("/repos/o/r/branches/main/protection"))
1001            .respond_with(
1002                ResponseTemplate::new(404)
1003                    .set_body_string("{\"message\":\"Branch not protected\"}"),
1004            )
1005            .mount(&server)
1006            .await;
1007        Mock::given(method("GET"))
1008            .and(path("/repos/o/r/rules/branches/main"))
1009            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
1010            .mount(&server)
1011            .await;
1012        Mock::given(method("GET"))
1013            .and(path("/repos/o/r/branches/main"))
1014            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1015                "commit": { "sha": "deadbeef" }
1016            })))
1017            .mount(&server)
1018            .await;
1019        Mock::given(method("GET"))
1020            .and(path("/repos/o/r/commits/deadbeef/check-runs"))
1021            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1022                "total_count": 1,
1023                "check_runs": [{
1024                    "id": 1,
1025                    "name": "test",
1026                    "status": "completed",
1027                    "conclusion": "success",
1028                    "app": { "id": 15368 }
1029                }]
1030            })))
1031            .mount(&server)
1032            .await;
1033        Mock::given(method("GET"))
1034            .and(path("/repos/o/r/commits/deadbeef/status"))
1035            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1036                "state": "success",
1037                "total_count": 0,
1038                "statuses": []
1039            })))
1040            .mount(&server)
1041            .await;
1042        Mock::given(method("POST"))
1043            .and(path("/repos/o/r/rulesets"))
1044            .respond_with(ResponseTemplate::new(403).set_body_json(serde_json::json!({
1045                "message": "Upgrade to GitHub Pro or make this repository public to enable this feature."
1046            })))
1047            .mount(&server)
1048            .await;
1049        Mock::given(method("POST"))
1050            .and(path("/repos/o/r/issues/12/comments"))
1051            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1052                "id": 1
1053            })))
1054            .expect(1)
1055            .mount(&server)
1056            .await;
1057
1058        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1059        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
1060        let task_manager = Arc::new(TaskManager::with_store(
1061            backend,
1062            Arc::clone(&store),
1063            "o/r".to_string(),
1064        ));
1065        let id = store
1066            .upsert_external(&UpsertExternal {
1067                repo: "o/r",
1068                ext_id: "12",
1069                title: "CI-01 Set up CI and tests",
1070                body: &bootstrap_body(),
1071                author: "u",
1072                url: "",
1073                labels: &[],
1074                origin: "github",
1075            })
1076            .await
1077            .unwrap();
1078        store.update_status(id, TaskStatus::Approved).await.unwrap();
1079        store
1080            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
1081            .await
1082            .unwrap();
1083
1084        let gh = GhHttp::for_test(server.uri(), None);
1085        apply_pending_with(&gh, &store, &task_manager, "o/r")
1086            .await
1087            .unwrap();
1088
1089        let stored = store.get(id).await.unwrap();
1090        assert_eq!(stored.status, TaskStatus::Approved);
1091        assert_eq!(
1092            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1093            Some(KV_REFUSED)
1094        );
1095    }
1096
1097    #[tokio::test]
1098    async fn apply_pending_reopens_a_closed_issue_and_creates_the_ruleset() {
1099        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1100        let server = MockServer::start().await;
1101        Mock::given(method("GET"))
1102            .and(path("/repos/o/r"))
1103            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1104                "default_branch": "main"
1105            })))
1106            .mount(&server)
1107            .await;
1108        Mock::given(method("GET"))
1109            .and(path("/repos/o/r/branches/main/protection"))
1110            .respond_with(
1111                ResponseTemplate::new(404)
1112                    .set_body_string("{\"message\":\"Branch not protected\"}"),
1113            )
1114            .mount(&server)
1115            .await;
1116        Mock::given(method("GET"))
1117            .and(path("/repos/o/r/rules/branches/main"))
1118            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
1119            .mount(&server)
1120            .await;
1121        Mock::given(method("GET"))
1122            .and(path("/repos/o/r/issues/12"))
1123            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1124                "number": 12,
1125                "id": 120,
1126                "title": "CI-01 Set up CI and tests",
1127                "body": "",
1128                "state": "closed",
1129                "labels": [],
1130                "user": { "login": "u" },
1131                "created_at": "",
1132                "updated_at": "",
1133                "html_url": ""
1134            })))
1135            .mount(&server)
1136            .await;
1137        Mock::given(method("PATCH"))
1138            .and(path("/repos/o/r/issues/12"))
1139            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1140                "number": 12,
1141                "id": 120,
1142                "title": "CI-01 Set up CI and tests",
1143                "body": "",
1144                "state": "open",
1145                "labels": [],
1146                "user": { "login": "u" },
1147                "created_at": "",
1148                "updated_at": "",
1149                "html_url": ""
1150            })))
1151            .expect(1)
1152            .mount(&server)
1153            .await;
1154        Mock::given(method("GET"))
1155            .and(path("/repos/o/r/branches/main"))
1156            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1157                "commit": { "sha": "deadbeef" }
1158            })))
1159            .mount(&server)
1160            .await;
1161        Mock::given(method("GET"))
1162            .and(path("/repos/o/r/commits/deadbeef/check-runs"))
1163            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1164                "total_count": 1,
1165                "check_runs": [{
1166                    "id": 1,
1167                    "name": "test",
1168                    "status": "completed",
1169                    "conclusion": "success",
1170                    "app": { "id": 15368 }
1171                }]
1172            })))
1173            .mount(&server)
1174            .await;
1175        Mock::given(method("GET"))
1176            .and(path("/repos/o/r/commits/deadbeef/status"))
1177            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1178                "state": "success",
1179                "total_count": 0,
1180                "statuses": []
1181            })))
1182            .mount(&server)
1183            .await;
1184        Mock::given(method("POST"))
1185            .and(path("/repos/o/r/rulesets"))
1186            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1187                "id": 9,
1188                "name": "orch required checks"
1189            })))
1190            .expect(1)
1191            .mount(&server)
1192            .await;
1193
1194        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1195        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
1196        let task_manager = Arc::new(TaskManager::with_store(
1197            backend,
1198            Arc::clone(&store),
1199            "o/r".to_string(),
1200        ));
1201        let id = store
1202            .upsert_external(&UpsertExternal {
1203                repo: "o/r",
1204                ext_id: "12",
1205                title: "CI-01 Set up CI and tests",
1206                body: &bootstrap_body(),
1207                author: "u",
1208                url: "",
1209                labels: &[],
1210                origin: "github",
1211            })
1212            .await
1213            .unwrap();
1214        store.update_status(id, TaskStatus::Done).await.unwrap();
1215        store
1216            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
1217            .await
1218            .unwrap();
1219
1220        let gh = GhHttp::for_test(server.uri(), None);
1221        apply_pending_with(&gh, &store, &task_manager, "o/r")
1222            .await
1223            .unwrap();
1224
1225        let stored = store.get(id).await.unwrap();
1226        assert_eq!(stored.status, TaskStatus::Done);
1227        assert_eq!(
1228            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1229            Some(KV_APPLIED)
1230        );
1231    }
1232
1233    #[tokio::test]
1234    async fn apply_pending_logs_and_holds_while_default_branch_checks_run() {
1235        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1236        let server = MockServer::start().await;
1237        mock_unprotected_main(&server, "deadbeef").await;
1238        mock_runs_and_empty_status(
1239            &server,
1240            "deadbeef",
1241            serde_json::json!([{
1242                "id": 1,
1243                "name": "test",
1244                "status": "in_progress",
1245                "conclusion": null,
1246                "app": { "id": 15368 }
1247            }]),
1248        )
1249        .await;
1250        mock_issue_comment(&server, 0).await;
1251        Mock::given(method("POST"))
1252            .and(path("/repos/o/r/rulesets"))
1253            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1254                "id": 9
1255            })))
1256            .expect(0)
1257            .mount(&server)
1258            .await;
1259
1260        let (store, task_manager, id) = pending_store().await;
1261        let gh = GhHttp::for_test(server.uri(), None);
1262        apply_pending_with(&gh, &store, &task_manager, "o/r")
1263            .await
1264            .unwrap();
1265
1266        let stored = store.get(id).await.unwrap();
1267        assert_eq!(stored.status, TaskStatus::Approved);
1268        assert_eq!(
1269            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1270            Some(KV_PENDING)
1271        );
1272    }
1273
1274    #[tokio::test]
1275    async fn apply_pending_comments_and_routes_when_default_branch_checks_fail() {
1276        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1277        let server = MockServer::start().await;
1278        mock_unprotected_main(&server, "deadbeef").await;
1279        mock_runs_and_empty_status(
1280            &server,
1281            "deadbeef",
1282            serde_json::json!([{
1283                "id": 1,
1284                "name": "test",
1285                "status": "completed",
1286                "conclusion": "failure",
1287                "app": { "id": 15368 }
1288            }]),
1289        )
1290        .await;
1291        mock_issue_comment(&server, 1).await;
1292
1293        let (store, task_manager, id) = pending_store().await;
1294        let gh = GhHttp::for_test(server.uri(), None);
1295        apply_pending_with(&gh, &store, &task_manager, "o/r")
1296            .await
1297            .unwrap();
1298
1299        let stored = store.get(id).await.unwrap();
1300        assert_eq!(stored.status, TaskStatus::Routed);
1301        assert_eq!(stored.ci_merge_failures, 1);
1302        assert!(stored.last_error.contains("test: failure"));
1303        assert_eq!(
1304            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1305            Some(KV_PENDING)
1306        );
1307
1308        apply_pending_with(&gh, &store, &task_manager, "o/r")
1309            .await
1310            .unwrap();
1311        let stored = store.get(id).await.unwrap();
1312        assert_eq!(stored.status, TaskStatus::Routed);
1313        assert_eq!(stored.ci_merge_failures, 1);
1314    }
1315
1316    #[tokio::test]
1317    async fn apply_pending_blocks_after_max_failed_default_branch_heads() {
1318        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1319        let server = MockServer::start().await;
1320        mock_unprotected_main(&server, "deadbeef").await;
1321        mock_runs_and_empty_status(
1322            &server,
1323            "deadbeef",
1324            serde_json::json!([{
1325                "id": 1,
1326                "name": "test",
1327                "status": "completed",
1328                "conclusion": "cancelled",
1329                "app": { "id": 15368 }
1330            }]),
1331        )
1332        .await;
1333        mock_issue_comment(&server, 1).await;
1334
1335        let (store, task_manager, id) = pending_store().await;
1336        store.increment(id, "ci_merge_failures").await.unwrap();
1337        store.increment(id, "ci_merge_failures").await.unwrap();
1338        let gh = GhHttp::for_test(server.uri(), None);
1339        apply_pending_with(&gh, &store, &task_manager, "o/r")
1340            .await
1341            .unwrap();
1342
1343        let stored = store.get(id).await.unwrap();
1344        assert_eq!(stored.status, TaskStatus::Blocked);
1345        assert_eq!(
1346            stored.ci_merge_failures,
1347            i32::try_from(auto_merge::MAX_CI_MERGE_FAILURES).unwrap()
1348        );
1349        assert!(
1350            stored
1351                .block_reason
1352                .as_deref()
1353                .unwrap_or("")
1354                .contains("CI failure limit")
1355        );
1356    }
1357
1358    #[tokio::test]
1359    async fn apply_pending_waits_before_commenting_when_no_check_starts() {
1360        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1361        let server = MockServer::start().await;
1362        mock_unprotected_main(&server, "deadbeef").await;
1363        mock_runs_and_empty_status(&server, "deadbeef", serde_json::json!([])).await;
1364        mock_issue_comment(&server, 0).await;
1365
1366        let (store, task_manager, id) = pending_store().await;
1367        let gh = GhHttp::for_test(server.uri(), None);
1368        apply_pending_with(&gh, &store, &task_manager, "o/r")
1369            .await
1370            .unwrap();
1371
1372        let stored = store.get(id).await.unwrap();
1373        assert_eq!(stored.status, TaskStatus::Approved);
1374        assert!(
1375            store
1376                .kv_get("ci_bootstrap_wait_since:o/r#12#deadbeef")
1377                .await
1378                .unwrap()
1379                .is_some()
1380        );
1381        assert!(
1382            store
1383                .kv_get(&no_checks_comment_key("o/r", "12", "deadbeef"))
1384                .await
1385                .unwrap()
1386                .is_none()
1387        );
1388    }
1389
1390    #[tokio::test]
1391    async fn apply_pending_comments_once_when_no_check_starts_after_the_wait() {
1392        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1393        let server = MockServer::start().await;
1394        mock_unprotected_main(&server, "deadbeef").await;
1395        mock_runs_and_empty_status(&server, "deadbeef", serde_json::json!([])).await;
1396        mock_issue_comment(&server, 1).await;
1397
1398        let (store, task_manager, id) = pending_store().await;
1399        let started = chrono::Utc::now().timestamp() - review_gate::NO_CHECKS_WAIT_SECS - 1;
1400        store
1401            .kv_set(
1402                "ci_bootstrap_wait_since:o/r#12#deadbeef",
1403                &started.to_string(),
1404            )
1405            .await
1406            .unwrap();
1407        let gh = GhHttp::for_test(server.uri(), None);
1408        apply_pending_with(&gh, &store, &task_manager, "o/r")
1409            .await
1410            .unwrap();
1411
1412        let stored = store.get(id).await.unwrap();
1413        assert_eq!(stored.status, TaskStatus::Approved);
1414        assert_eq!(
1415            store
1416                .kv_get(&no_checks_comment_key("o/r", "12", "deadbeef"))
1417                .await
1418                .unwrap()
1419                .as_deref(),
1420            Some("1")
1421        );
1422
1423        apply_pending_with(&gh, &store, &task_manager, "o/r")
1424            .await
1425            .unwrap();
1426        assert_eq!(store.get(id).await.unwrap().status, TaskStatus::Approved);
1427    }
1428
1429    #[tokio::test]
1430    async fn apply_pending_comments_once_when_github_says_the_check_never_ran() {
1431        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1432        let server = MockServer::start().await;
1433        mock_unprotected_main(&server, "deadbeef").await;
1434        mock_runs_and_empty_status(
1435            &server,
1436            "deadbeef",
1437            serde_json::json!([{
1438                "id": 1,
1439                "name": "test",
1440                "status": "completed",
1441                "conclusion": "success",
1442                "app": { "id": 15368 }
1443            }]),
1444        )
1445        .await;
1446        Mock::given(method("POST"))
1447            .and(path("/repos/o/r/rulesets"))
1448            .respond_with(ResponseTemplate::new(422).set_body_json(serde_json::json!({
1449                "message": "test was not a valid check"
1450            })))
1451            .mount(&server)
1452            .await;
1453        mock_issue_comment(&server, 1).await;
1454
1455        let (store, task_manager, id) = pending_store().await;
1456        let gh = GhHttp::for_test(server.uri(), None);
1457        apply_pending_with(&gh, &store, &task_manager, "o/r")
1458            .await
1459            .unwrap();
1460        apply_pending_with(&gh, &store, &task_manager, "o/r")
1461            .await
1462            .unwrap();
1463
1464        let stored = store.get(id).await.unwrap();
1465        assert_eq!(stored.status, TaskStatus::Approved);
1466        assert_eq!(
1467            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1468            Some(KV_PENDING)
1469        );
1470        assert_eq!(
1471            store
1472                .kv_get(&never_run_comment_key("o/r", "12"))
1473                .await
1474                .unwrap()
1475                .as_deref(),
1476            Some("1")
1477        );
1478    }
1479}