Skip to main content

orch/engine/
ci_bootstrap.rs

1//! After a CI bootstrap task merges, require its default-branch checks
2
3use crate::backends::{ExternalId, ExternalTask, Status};
4use crate::engine;
5use crate::engine::auto_merge;
6use crate::engine::review_gate;
7use crate::engine::tasks::TaskManager;
8use crate::github::http::{
9    GhHttp, REQUIRED_CHECK_RULESET_NAME, RulesetCreateKind, classify_ruleset_create_error,
10};
11use crate::github::types::{GitHubCheckRun, RequiredStatusCheck};
12use crate::plan;
13use crate::store;
14use crate::store::TaskStore;
15use std::sync::Arc;
16
17const KV_PENDING: &str = "pending";
18const KV_APPLIED: &str = "applied";
19const KV_REFUSED: &str = "refused";
20
21#[derive(Debug, PartialEq)]
22pub(crate) enum ChecksReady {
23    Ready(Vec<RequiredStatusCheck>),
24    /// A real check is still queued or in progress
25    Pending,
26    /// A real check finished as failure, error, cancelled, or timed out
27    Failed(Vec<FailedCheck>),
28    /// No real check has started
29    Empty,
30}
31
32#[derive(Debug, PartialEq)]
33pub(crate) struct FailedCheck {
34    name: String,
35    conclusion: String,
36}
37
38pub(crate) fn is_ci_bootstrap_task(body: &str) -> bool {
39    plan::is_ci_bootstrap_body(body)
40        || plan::marker_id(body).is_some_and(|id| plan::is_ci_bootstrap_id(&id))
41}
42
43pub(crate) fn kv_key(repo: &str, task_id: &str) -> String {
44    format!("ci-bootstrap:{repo}:{task_id}")
45}
46
47/// True when this merged CI bootstrap task must stay approved until a ruleset exists
48pub(crate) async fn hold_after_merge(
49    task: &ExternalTask,
50    store: &Arc<TaskStore>,
51    repo: &str,
52) -> bool {
53    if !is_ci_bootstrap_task(&task.body) {
54        return false;
55    }
56    let key = kv_key(repo, &task.id.0);
57    match store.kv_get(&key).await {
58        Ok(Some(state)) if state == KV_APPLIED => false,
59        Ok(Some(_)) => true,
60        Ok(None) => {
61            if let Err(e) = store.kv_set(&key, KV_PENDING).await {
62                tracing::warn!(
63                    task_id = %task.id.0,
64                    err = %e,
65                    "failed to record CI bootstrap pending"
66                );
67            }
68            true
69        }
70        Err(e) => {
71            tracing::warn!(
72                task_id = %task.id.0,
73                err = %e,
74                "failed to read CI bootstrap state"
75            );
76            true
77        }
78    }
79}
80
81pub(crate) async fn apply_pending(
82    store: &Arc<TaskStore>,
83    task_manager: &Arc<TaskManager>,
84    repo: &str,
85) -> anyhow::Result<()> {
86    let gh = GhHttp::owner()?;
87    apply_pending_with(&gh, store, task_manager, repo).await
88}
89
90pub(crate) async fn apply_pending_with(
91    gh: &GhHttp,
92    store: &Arc<TaskStore>,
93    task_manager: &Arc<TaskManager>,
94    repo: &str,
95) -> anyhow::Result<()> {
96    let prefix = format!("ci-bootstrap:{repo}:");
97    let entries = store.kv_list_prefix(&prefix).await?;
98    for (key, state) in entries {
99        if state == KV_APPLIED {
100            continue;
101        }
102        let Some(task_id) = key.strip_prefix(&prefix) else {
103            continue;
104        };
105        if let Err(e) = apply_one(gh, store, task_manager, repo, task_id, &state).await {
106            tracing::warn!(task_id, err = %e, "CI bootstrap ruleset apply failed");
107        }
108    }
109    Ok(())
110}
111
112pub(crate) fn checks_ready_for_ruleset(
113    runs: &[GitHubCheckRun],
114    statuses: &[(String, String)],
115) -> ChecksReady {
116    let mut ready: Vec<RequiredStatusCheck> = Vec::new();
117    let mut pending = false;
118    let mut failed: Vec<FailedCheck> = Vec::new();
119
120    for run in runs {
121        if review_gate::is_own_check(&run.name) {
122            continue;
123        }
124        match run.status.as_str() {
125            "completed" => match run.conclusion.as_deref() {
126                Some("success") => {
127                    let integration_id = run.app.as_ref().map(|app| app.id).filter(|&id| id > 0);
128                    if !ready.iter().any(|check| check.context == run.name) {
129                        ready.push(RequiredStatusCheck {
130                            context: run.name.clone(),
131                            integration_id,
132                        });
133                    }
134                }
135                Some("skipped" | "neutral") => {}
136                Some(conclusion) => failed.push(FailedCheck {
137                    name: run.name.clone(),
138                    conclusion: conclusion.to_string(),
139                }),
140                None => pending = true,
141            },
142            _ => pending = true,
143        }
144    }
145
146    for (context, state) in statuses {
147        if review_gate::is_own_check(context) {
148            continue;
149        }
150        match state.as_str() {
151            "pending" => pending = true,
152            "success" => {
153                if !ready.iter().any(|check| &check.context == context) {
154                    ready.push(RequiredStatusCheck {
155                        context: context.clone(),
156                        integration_id: None,
157                    });
158                }
159            }
160            "failure" | "error" => failed.push(FailedCheck {
161                name: context.clone(),
162                conclusion: state.clone(),
163            }),
164            _ => {}
165        }
166    }
167
168    if !failed.is_empty() {
169        ChecksReady::Failed(failed)
170    } else if pending {
171        ChecksReady::Pending
172    } else if ready.is_empty() {
173        ChecksReady::Empty
174    } else {
175        ChecksReady::Ready(ready)
176    }
177}
178
179fn required_contexts_cover(
180    required: &[RequiredStatusCheck],
181    needed: &[RequiredStatusCheck],
182) -> bool {
183    needed.iter().all(|check| {
184        required
185            .iter()
186            .any(|existing| existing.context == check.context)
187    })
188}
189
190fn merge_missing_checks(
191    mut existing: Vec<RequiredStatusCheck>,
192    needed: &[RequiredStatusCheck],
193) -> Vec<RequiredStatusCheck> {
194    for check in needed {
195        if !existing.iter().any(|e| e.context == check.context) {
196            existing.push(check.clone());
197        }
198    }
199    existing
200}
201
202async fn apply_one(
203    gh: &GhHttp,
204    store: &Arc<TaskStore>,
205    task_manager: &Arc<TaskManager>,
206    repo: &str,
207    task_id: &str,
208    state: &str,
209) -> anyhow::Result<()> {
210    if store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
211        .await
212        .is_none()
213    {
214        return Ok(());
215    }
216
217    let branch = gh.get_default_branch(repo).await?;
218    let required = gh.get_required_status_check_contexts(repo, &branch).await?;
219    let sha = gh.get_branch_sha(repo, &branch).await?;
220    let runs = gh.get_check_runs(repo, &sha).await?;
221    let statuses = gh.get_commit_status_contexts(repo, &sha).await?;
222    let checks = match checks_ready_for_ruleset(&runs, &statuses) {
223        ChecksReady::Pending => {
224            tracing::info!(
225                task_id,
226                sha = %sha,
227                "CI bootstrap waiting for default-branch checks"
228            );
229            reopen_if_closed(gh, repo, task_id).await;
230            clear_wait(store, repo, task_id).await;
231            return Ok(());
232        }
233        ChecksReady::Empty => {
234            reopen_if_closed(gh, repo, task_id).await;
235            return wait_for_checks(gh, store, repo, task_id, &sha).await;
236        }
237        ChecksReady::Failed(failed) => {
238            reopen_if_closed(gh, repo, task_id).await;
239            clear_wait(store, repo, task_id).await;
240            return handle_failed_checks(gh, store, task_manager, repo, task_id, &sha, &failed)
241                .await;
242        }
243        ChecksReady::Ready(checks) => {
244            clear_wait(store, repo, task_id).await;
245            checks
246        }
247    };
248
249    if required_contexts_cover(&required, &checks) {
250        return finish_applied(store, task_manager, task_id, repo).await;
251    }
252
253    // GitHub may have closed the issue via Closes on the PR, keep it open until the ruleset exists
254    reopen_if_closed(gh, repo, task_id).await;
255    if state == KV_REFUSED {
256        return Ok(());
257    }
258
259    match gh.create_required_check_ruleset(repo, &checks).await {
260        Ok(()) => finish_applied(store, task_manager, task_id, repo).await,
261        Err(e) => {
262            let err = e.to_string();
263            match classify_ruleset_create_error(&err) {
264                RulesetCreateKind::NameTaken => {
265                    apply_existing_orch_ruleset(gh, store, task_manager, repo, task_id, &checks)
266                        .await
267                }
268                kind => handle_ruleset_write_error(gh, store, repo, task_id, &err, kind).await,
269            }
270        }
271    }
272}
273
274async fn apply_existing_orch_ruleset(
275    gh: &GhHttp,
276    store: &Arc<TaskStore>,
277    task_manager: &Arc<TaskManager>,
278    repo: &str,
279    task_id: &str,
280    needed: &[RequiredStatusCheck],
281) -> anyhow::Result<()> {
282    let Some((id, existing)) = gh
283        .get_named_ruleset_required_checks(repo, REQUIRED_CHECK_RULESET_NAME)
284        .await?
285    else {
286        tracing::info!(
287            task_id,
288            "CI bootstrap ruleset name taken, existing ruleset not found"
289        );
290        return Ok(());
291    };
292    if required_contexts_cover(&existing, needed) {
293        return finish_applied(store, task_manager, task_id, repo).await;
294    }
295    let merged = merge_missing_checks(existing, needed);
296    match gh.update_required_check_ruleset(repo, id, &merged).await {
297        Ok(()) => finish_applied(store, task_manager, task_id, repo).await,
298        Err(e) => {
299            let err = e.to_string();
300            handle_ruleset_write_error(
301                gh,
302                store,
303                repo,
304                task_id,
305                &err,
306                classify_ruleset_create_error(&err),
307            )
308            .await
309        }
310    }
311}
312
313async fn handle_ruleset_write_error(
314    gh: &GhHttp,
315    store: &Arc<TaskStore>,
316    repo: &str,
317    task_id: &str,
318    err: &str,
319    kind: RulesetCreateKind,
320) -> anyhow::Result<()> {
321    match kind {
322        RulesetCreateKind::NameTaken | RulesetCreateKind::Retryable => {
323            tracing::info!(
324                task_id,
325                err = %err,
326                "CI bootstrap ruleset create retrying"
327            );
328            Ok(())
329        }
330        RulesetCreateKind::CheckNeverRun => {
331            tracing::info!(
332                task_id,
333                err = %err,
334                "CI bootstrap ruleset create waiting, check has not run"
335            );
336            comment_never_run_once(gh, store, repo, task_id, err).await;
337            Ok(())
338        }
339        RulesetCreateKind::Refused | RulesetCreateKind::Other => {
340            comment_refused(gh, repo, task_id, err).await;
341            store.kv_set(&kv_key(repo, task_id), KV_REFUSED).await?;
342            Ok(())
343        }
344    }
345}
346
347async fn finish_applied(
348    store: &Arc<TaskStore>,
349    task_manager: &Arc<TaskManager>,
350    task_id: &str,
351    repo: &str,
352) -> anyhow::Result<()> {
353    store.kv_set(&kv_key(repo, task_id), KV_APPLIED).await?;
354    task_manager
355        .update_task_status(&ExternalId(task_id.to_string()), Status::Done)
356        .await?;
357    Ok(())
358}
359
360async fn reopen_if_closed(gh: &GhHttp, repo: &str, task_id: &str) {
361    match gh.get_issue(repo, task_id).await {
362        Ok(issue) if issue.state.eq_ignore_ascii_case("closed") => {
363            if let Err(e) = gh.reopen_issue(repo, task_id).await {
364                tracing::warn!(
365                    task_id,
366                    err = %e,
367                    "failed to reopen CI bootstrap issue"
368                );
369            }
370        }
371        Ok(_) => {}
372        Err(e) => {
373            tracing::warn!(
374                task_id,
375                err = %e,
376                "failed to read CI bootstrap issue"
377            );
378        }
379    }
380}
381
382fn wait_prefix(repo: &str, task_id: &str) -> String {
383    format!("ci_bootstrap_wait_since:{repo}#{task_id}#")
384}
385
386fn no_checks_comment_key(repo: &str, task_id: &str, sha: &str) -> String {
387    format!("ci_bootstrap_no_checks_comment:{repo}#{task_id}#{sha}")
388}
389
390fn never_run_comment_key(repo: &str, task_id: &str) -> String {
391    format!("ci_bootstrap_never_run_comment:{repo}#{task_id}")
392}
393
394fn format_failed(failed: &[FailedCheck]) -> String {
395    failed
396        .iter()
397        .map(|check| format!("{}: {}", check.name, check.conclusion))
398        .collect::<Vec<_>>()
399        .join(", ")
400}
401
402async fn clear_wait(store: &Arc<TaskStore>, repo: &str, task_id: &str) {
403    let prefix = wait_prefix(repo, task_id);
404    let keys = match store.kv_list_prefix(&prefix).await {
405        Ok(entries) => entries,
406        Err(e) => {
407            tracing::warn!(task_id, err = %e, "failed to list CI bootstrap wait keys");
408            return;
409        }
410    };
411    for (key, _) in keys {
412        if let Err(e) = store.kv_delete(&key).await {
413            tracing::warn!(task_id, err = %e, "failed to clear CI bootstrap wait key");
414        }
415    }
416}
417
418async fn wait_for_checks(
419    gh: &GhHttp,
420    store: &Arc<TaskStore>,
421    repo: &str,
422    task_id: &str,
423    sha: &str,
424) -> anyhow::Result<()> {
425    let prefix = wait_prefix(repo, task_id);
426    let key = format!("{prefix}{sha}");
427    let now = chrono::Utc::now().timestamp();
428    let since = review_gate::wait_since(store, &prefix, &key, now).await?;
429    if now - since < review_gate::NO_CHECKS_WAIT_SECS {
430        tracing::info!(
431            task_id,
432            sha = %sha,
433            "CI bootstrap waiting for a default-branch check to start"
434        );
435        return Ok(());
436    }
437    let wait_mins = review_gate::NO_CHECKS_WAIT_SECS / 60;
438    tracing::warn!(
439        task_id,
440        sha = %sha,
441        wait_mins,
442        "CI bootstrap has no default-branch check after the wait"
443    );
444    let comment_key = no_checks_comment_key(repo, task_id, sha);
445    if store.kv_get(&comment_key).await?.is_none() {
446        comment_issue(
447            gh,
448            repo,
449            task_id,
450            &format!(
451                "No default-branch check has started {wait_mins} minutes after the CI bootstrap merge. Later tasks stay waiting until a check runs."
452            ),
453        )
454        .await;
455        store.kv_set(&comment_key, "1").await?;
456    }
457    Ok(())
458}
459
460async fn handle_failed_checks(
461    gh: &GhHttp,
462    store: &Arc<TaskStore>,
463    task_manager: &Arc<TaskManager>,
464    repo: &str,
465    task_id: &str,
466    sha: &str,
467    failed: &[FailedCheck],
468) -> anyhow::Result<()> {
469    let details = format_failed(failed);
470    let failures = match auto_merge::count_ci_failure_for_head(store, repo, task_id, sha).await {
471        Ok(Some(v)) => v,
472        Ok(None) => {
473            tracing::info!(
474                task_id,
475                sha = %sha,
476                details,
477                "CI bootstrap default-branch checks still failed on the same head"
478            );
479            return Ok(());
480        }
481        Err(e) => {
482            tracing::warn!(
483                task_id,
484                err = %e,
485                "failed to increment ci_merge_failures for CI bootstrap"
486            );
487            return Ok(());
488        }
489    };
490
491    comment_issue(
492        gh,
493        repo,
494        task_id,
495        &format!(
496            "Default-branch checks failed after the CI bootstrap merge ({details}). Later tasks stay waiting until those checks pass."
497        ),
498    )
499    .await;
500
501    let max = auto_merge::MAX_CI_MERGE_FAILURES;
502    let id = ExternalId(task_id.to_string());
503    if failures >= max {
504        tracing::error!(
505            task_id,
506            failures,
507            details,
508            "CI bootstrap default-branch checks failed, CI failure limit reached"
509        );
510        let fields = [(
511            "block_reason",
512            serde_json::json!(format!(
513                "CI failure limit ({max}) reached after the CI bootstrap merge"
514            )),
515        )];
516        if let Err(e) = task_manager
517            .update_task_status_and_result(&id, Status::Blocked, &fields)
518            .await
519        {
520            tracing::error!(
521                task_id,
522                err = %e,
523                "failed to block CI bootstrap after failed default-branch checks"
524            );
525        }
526    } else {
527        tracing::warn!(
528            task_id,
529            failures,
530            details,
531            "CI bootstrap default-branch checks failed, routing back to the task agent"
532        );
533        let fields = [
534            (
535                "last_error",
536                serde_json::json!(format!(
537                    "CI failed on the default branch after the CI bootstrap merge: {details}. Fix the failing checks"
538                )),
539            ),
540            (
541                "route_reason",
542                serde_json::json!("re-dispatch after CI bootstrap default-branch checks failed"),
543            ),
544        ];
545        if let Err(e) = task_manager
546            .update_task_status_and_result(&id, Status::Routed, &fields)
547            .await
548        {
549            tracing::error!(
550                task_id,
551                err = %e,
552                "failed to re-route CI bootstrap after failed default-branch checks"
553            );
554        }
555    }
556    Ok(())
557}
558
559async fn comment_never_run_once(
560    gh: &GhHttp,
561    store: &Arc<TaskStore>,
562    repo: &str,
563    task_id: &str,
564    err: &str,
565) {
566    let key = never_run_comment_key(repo, task_id);
567    match store.kv_get(&key).await {
568        Ok(Some(_)) => return,
569        Ok(None) => {}
570        Err(e) => {
571            tracing::warn!(
572                task_id,
573                err = %e,
574                "failed to read CI bootstrap never-run comment key"
575            );
576            return;
577        }
578    }
579    comment_issue(
580        gh,
581        repo,
582        task_id,
583        &format!(
584            "GitHub does not treat the default-branch checks as valid required checks ({err}). Later tasks stay waiting until a required-checks ruleset can be created."
585        ),
586    )
587    .await;
588    if let Err(e) = store.kv_set(&key, "1").await {
589        tracing::warn!(
590            task_id,
591            err = %e,
592            "failed to record CI bootstrap never-run comment"
593        );
594    }
595}
596
597async fn comment_refused(gh: &GhHttp, repo: &str, task_id: &str, err: &str) {
598    comment_issue(
599        gh,
600        repo,
601        task_id,
602        &format!(
603            "GitHub refused a required-checks ruleset on the default branch ({err}). Turn branch protection on for this repository. Later tasks stay waiting until that happens."
604        ),
605    )
606    .await;
607}
608
609async fn comment_issue(gh: &GhHttp, repo: &str, task_id: &str, message: &str) {
610    let body = format!("{message}{}", engine::orch_footer());
611    if let Err(e) = gh.add_comment(repo, task_id, &body).await {
612        tracing::warn!(
613            task_id,
614            err = %e,
615            "failed to comment on CI bootstrap issue"
616        );
617    }
618}
619
620#[cfg(test)]
621mod tests {
622    use super::*;
623    use crate::backends::ExternalBackend;
624    use crate::backends::test_helpers::NoopBackend;
625    use crate::github::types::GitHubApp;
626    use crate::plan::{CI_BOOTSTRAP_MARKER, ci_bootstrap_task, issue_body};
627    use crate::store::TaskStatus;
628    use crate::store::UpsertExternal;
629    use std::env;
630    use std::ffi::OsString;
631    use wiremock::matchers::{body_partial_json, method, path};
632    use wiremock::{Mock, MockServer, ResponseTemplate};
633
634    struct EnvVarGuard {
635        key: &'static str,
636        previous: Option<OsString>,
637    }
638
639    impl EnvVarGuard {
640        fn set(key: &'static str, value: &str) -> Self {
641            let previous = env::var_os(key);
642            // SAFETY: nextest runs each test in its own process
643            unsafe { env::set_var(key, value) };
644            Self { key, previous }
645        }
646    }
647
648    impl Drop for EnvVarGuard {
649        fn drop(&mut self) {
650            match self.previous.take() {
651                // SAFETY: nextest runs each test in its own process
652                Some(value) => unsafe { env::set_var(self.key, value) },
653                // SAFETY: nextest runs each test in its own process
654                None => unsafe { env::remove_var(self.key) },
655            }
656        }
657    }
658
659    fn run(
660        name: &str,
661        status: &str,
662        conclusion: Option<&str>,
663        app_id: Option<i64>,
664    ) -> GitHubCheckRun {
665        GitHubCheckRun {
666            id: 1,
667            name: name.to_string(),
668            status: status.to_string(),
669            conclusion: conclusion.map(String::from),
670            app: app_id.map(|id| GitHubApp { id }),
671        }
672    }
673
674    fn bootstrap_body() -> String {
675        issue_body(
676            &ci_bootstrap_task(false),
677            "acme/app",
678            "main",
679            "docs/ROADMAP.md",
680        )
681    }
682
683    async fn pending_store() -> (Arc<TaskStore>, Arc<TaskManager>, i64) {
684        let store = Arc::new(TaskStore::open_memory().await.unwrap());
685        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
686        let task_manager = Arc::new(TaskManager::with_store(
687            backend,
688            Arc::clone(&store),
689            "o/r".to_string(),
690        ));
691        let id = store
692            .upsert_external(&UpsertExternal {
693                repo: "o/r",
694                ext_id: "12",
695                title: "CI-01 Set up CI and tests",
696                body: &bootstrap_body(),
697                author: "u",
698                url: "",
699                labels: &[],
700                origin: "github",
701            })
702            .await
703            .unwrap();
704        store.update_status(id, TaskStatus::Approved).await.unwrap();
705        store
706            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
707            .await
708            .unwrap();
709        (store, task_manager, id)
710    }
711
712    async fn mock_unprotected_main(server: &MockServer, sha: &str) {
713        Mock::given(method("GET"))
714            .and(path("/repos/o/r"))
715            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
716                "default_branch": "main"
717            })))
718            .mount(server)
719            .await;
720        Mock::given(method("GET"))
721            .and(path("/repos/o/r/branches/main/protection"))
722            .respond_with(
723                ResponseTemplate::new(404)
724                    .set_body_string("{\"message\":\"Branch not protected\"}"),
725            )
726            .mount(server)
727            .await;
728        Mock::given(method("GET"))
729            .and(path("/repos/o/r/rules/branches/main"))
730            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
731            .mount(server)
732            .await;
733        Mock::given(method("GET"))
734            .and(path("/repos/o/r/branches/main"))
735            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
736                "commit": { "sha": sha }
737            })))
738            .mount(server)
739            .await;
740    }
741
742    async fn mock_classic_required(server: &MockServer, context: &str, sha: &str) {
743        Mock::given(method("GET"))
744            .and(path("/repos/o/r"))
745            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
746                "default_branch": "main"
747            })))
748            .mount(server)
749            .await;
750        Mock::given(method("GET"))
751            .and(path("/repos/o/r/branches/main/protection"))
752            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
753                "required_status_checks": {
754                    "checks": [{ "context": context, "app_id": 15368 }]
755                }
756            })))
757            .mount(server)
758            .await;
759        Mock::given(method("GET"))
760            .and(path("/repos/o/r/rules/branches/main"))
761            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
762            .mount(server)
763            .await;
764        Mock::given(method("GET"))
765            .and(path("/repos/o/r/branches/main"))
766            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
767                "commit": { "sha": sha }
768            })))
769            .mount(server)
770            .await;
771    }
772
773    async fn mock_name_taken_ruleset(server: &MockServer, ruleset_checks: serde_json::Value) {
774        Mock::given(method("POST"))
775            .and(path("/repos/o/r/rulesets"))
776            .respond_with(ResponseTemplate::new(422).set_body_json(serde_json::json!({
777                "message": "Name has already been taken"
778            })))
779            .expect(1)
780            .mount(server)
781            .await;
782        Mock::given(method("GET"))
783            .and(path("/repos/o/r/rulesets"))
784            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([
785                { "id": 9, "name": REQUIRED_CHECK_RULESET_NAME }
786            ])))
787            .mount(server)
788            .await;
789        Mock::given(method("GET"))
790            .and(path("/repos/o/r/rulesets/9"))
791            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
792                "id": 9,
793                "name": REQUIRED_CHECK_RULESET_NAME,
794                "rules": [{
795                    "type": "required_status_checks",
796                    "parameters": {
797                        "required_status_checks": ruleset_checks
798                    }
799                }]
800            })))
801            .mount(server)
802            .await;
803    }
804
805    async fn mock_runs_and_empty_status(
806        server: &MockServer,
807        sha: &str,
808        check_runs: serde_json::Value,
809    ) {
810        Mock::given(method("GET"))
811            .and(path(format!("/repos/o/r/commits/{sha}/check-runs")))
812            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
813                "total_count": 1,
814                "check_runs": check_runs
815            })))
816            .mount(server)
817            .await;
818        Mock::given(method("GET"))
819            .and(path(format!("/repos/o/r/commits/{sha}/status")))
820            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
821                "state": "pending",
822                "total_count": 0,
823                "statuses": []
824            })))
825            .mount(server)
826            .await;
827    }
828
829    async fn mock_issue_comment(server: &MockServer, expected: u64) {
830        Mock::given(method("POST"))
831            .and(path("/repos/o/r/issues/12/comments"))
832            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
833                "id": 1
834            })))
835            .expect(expected)
836            .mount(server)
837            .await;
838    }
839
840    #[test]
841    fn is_ci_bootstrap_task_reads_the_body_marker() {
842        assert!(is_ci_bootstrap_task(&bootstrap_body()));
843        assert!(is_ci_bootstrap_task(&format!("{CI_BOOTSTRAP_MARKER}\n")));
844        assert!(is_ci_bootstrap_task("goal\n<!-- orch-plan: CI-01 -->"));
845        assert!(!is_ci_bootstrap_task(""));
846        assert!(!is_ci_bootstrap_task("goal"));
847    }
848
849    #[test]
850    fn checks_ready_for_ruleset_splits_pending_failed_and_empty() {
851        assert_eq!(checks_ready_for_ruleset(&[], &[]), ChecksReady::Empty);
852        assert_eq!(
853            checks_ready_for_ruleset(&[run("test", "in_progress", None, None)], &[]),
854            ChecksReady::Pending
855        );
856        assert_eq!(
857            checks_ready_for_ruleset(&[run("test", "completed", Some("failure"), None)], &[]),
858            ChecksReady::Failed(vec![FailedCheck {
859                name: "test".to_string(),
860                conclusion: "failure".to_string(),
861            }])
862        );
863        assert_eq!(
864            checks_ready_for_ruleset(&[run("test", "completed", Some("cancelled"), None)], &[]),
865            ChecksReady::Failed(vec![FailedCheck {
866                name: "test".to_string(),
867                conclusion: "cancelled".to_string(),
868            }])
869        );
870        assert_eq!(
871            checks_ready_for_ruleset(&[], &[("ci/test".to_string(), "failure".to_string())]),
872            ChecksReady::Failed(vec![FailedCheck {
873                name: "ci/test".to_string(),
874                conclusion: "failure".to_string(),
875            }])
876        );
877        assert_eq!(
878            checks_ready_for_ruleset(&[run("test", "completed", Some("skipped"), None)], &[]),
879            ChecksReady::Empty
880        );
881        assert_eq!(
882            checks_ready_for_ruleset(
883                &[run("orch/review", "completed", Some("success"), None)],
884                &[]
885            ),
886            ChecksReady::Empty
887        );
888        assert_eq!(
889            checks_ready_for_ruleset(
890                &[
891                    run("test", "completed", Some("failure"), None),
892                    run("lint", "in_progress", None, None),
893                ],
894                &[]
895            ),
896            ChecksReady::Failed(vec![FailedCheck {
897                name: "test".to_string(),
898                conclusion: "failure".to_string(),
899            }])
900        );
901        let ChecksReady::Ready(ready) = checks_ready_for_ruleset(
902            &[
903                run("test", "completed", Some("success"), Some(15368)),
904                run("review-gate", "completed", Some("success"), None),
905            ],
906            &[],
907        ) else {
908            panic!("expected ready checks");
909        };
910        assert_eq!(
911            ready,
912            vec![RequiredStatusCheck {
913                context: "test".to_string(),
914                integration_id: Some(15368),
915            }]
916        );
917    }
918
919    #[test]
920    fn required_contexts_cover_matches_by_context_name() {
921        let required = vec![RequiredStatusCheck {
922            context: "cla/check".to_string(),
923            integration_id: None,
924        }];
925        let needed = vec![RequiredStatusCheck {
926            context: "test".to_string(),
927            integration_id: Some(15368),
928        }];
929        assert!(!required_contexts_cover(&required, &needed));
930        let required = vec![
931            RequiredStatusCheck {
932                context: "cla/check".to_string(),
933                integration_id: None,
934            },
935            RequiredStatusCheck {
936                context: "test".to_string(),
937                integration_id: None,
938            },
939        ];
940        assert!(required_contexts_cover(&required, &needed));
941    }
942
943    #[test]
944    fn merge_missing_checks_appends_contexts_the_ruleset_lacks() {
945        let existing = vec![RequiredStatusCheck {
946            context: "cla/check".to_string(),
947            integration_id: None,
948        }];
949        let needed = vec![
950            RequiredStatusCheck {
951                context: "cla/check".to_string(),
952                integration_id: Some(1),
953            },
954            RequiredStatusCheck {
955                context: "test".to_string(),
956                integration_id: Some(15368),
957            },
958        ];
959        assert_eq!(
960            merge_missing_checks(existing, &needed),
961            vec![
962                RequiredStatusCheck {
963                    context: "cla/check".to_string(),
964                    integration_id: None,
965                },
966                RequiredStatusCheck {
967                    context: "test".to_string(),
968                    integration_id: Some(15368),
969                },
970            ]
971        );
972    }
973
974    #[tokio::test]
975    async fn hold_after_merge_records_pending_for_a_bootstrap_task() {
976        let store = Arc::new(TaskStore::open_memory().await.unwrap());
977        let task = ExternalTask {
978            id: ExternalId("12".to_string()),
979            title: "CI-01 Set up CI and tests".to_string(),
980            body: bootstrap_body(),
981            state: "open".to_string(),
982            labels: vec![],
983            author: String::new(),
984            created_at: String::new(),
985            updated_at: String::new(),
986            url: String::new(),
987        };
988        assert!(hold_after_merge(&task, &store, "o/r").await);
989        assert_eq!(
990            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
991            Some(KV_PENDING)
992        );
993        store
994            .kv_set(&kv_key("o/r", "12"), KV_APPLIED)
995            .await
996            .unwrap();
997        assert!(!hold_after_merge(&task, &store, "o/r").await);
998    }
999
1000    #[tokio::test]
1001    async fn hold_after_merge_skips_a_title_without_a_plan_marker() {
1002        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1003        let task = ExternalTask {
1004            id: ExternalId("12".to_string()),
1005            title: "CI-01 Set up CI and tests".to_string(),
1006            body: "ordinary issue".to_string(),
1007            state: "open".to_string(),
1008            labels: vec![],
1009            author: String::new(),
1010            created_at: String::new(),
1011            updated_at: String::new(),
1012            url: String::new(),
1013        };
1014        assert!(!hold_after_merge(&task, &store, "o/r").await);
1015        assert!(store.kv_get(&kv_key("o/r", "12")).await.unwrap().is_none());
1016    }
1017
1018    #[tokio::test]
1019    async fn apply_pending_dones_when_required_checks_already_cover_default_branch_checks() {
1020        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1021        let server = MockServer::start().await;
1022        mock_classic_required(&server, "test", "deadbeef").await;
1023        mock_runs_and_empty_status(
1024            &server,
1025            "deadbeef",
1026            serde_json::json!([{
1027                "id": 1,
1028                "name": "test",
1029                "status": "completed",
1030                "conclusion": "success",
1031                "app": { "id": 15368 }
1032            }]),
1033        )
1034        .await;
1035        Mock::given(method("POST"))
1036            .and(path("/repos/o/r/rulesets"))
1037            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1038                "id": 9
1039            })))
1040            .expect(0)
1041            .mount(&server)
1042            .await;
1043
1044        let (store, task_manager, id) = pending_store().await;
1045        let gh = GhHttp::for_test(server.uri(), None);
1046        apply_pending_with(&gh, &store, &task_manager, "o/r")
1047            .await
1048            .unwrap();
1049
1050        let stored = store.get(id).await.unwrap();
1051        assert_eq!(stored.status, TaskStatus::Done);
1052        assert_eq!(
1053            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1054            Some(KV_APPLIED)
1055        );
1056    }
1057
1058    #[tokio::test]
1059    async fn apply_pending_creates_a_ruleset_when_an_unrelated_required_check_exists() {
1060        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1061        let server = MockServer::start().await;
1062        mock_classic_required(&server, "cla/check", "deadbeef").await;
1063        mock_runs_and_empty_status(
1064            &server,
1065            "deadbeef",
1066            serde_json::json!([{
1067                "id": 1,
1068                "name": "test",
1069                "status": "completed",
1070                "conclusion": "success",
1071                "app": { "id": 15368 }
1072            }]),
1073        )
1074        .await;
1075        Mock::given(method("POST"))
1076            .and(path("/repos/o/r/rulesets"))
1077            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1078                "id": 9,
1079                "name": REQUIRED_CHECK_RULESET_NAME
1080            })))
1081            .expect(1)
1082            .mount(&server)
1083            .await;
1084
1085        let (store, task_manager, id) = pending_store().await;
1086        let gh = GhHttp::for_test(server.uri(), None);
1087        apply_pending_with(&gh, &store, &task_manager, "o/r")
1088            .await
1089            .unwrap();
1090
1091        let stored = store.get(id).await.unwrap();
1092        assert_eq!(stored.status, TaskStatus::Done);
1093        assert_eq!(
1094            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1095            Some(KV_APPLIED)
1096        );
1097    }
1098
1099    #[tokio::test]
1100    async fn apply_pending_holds_when_an_unrelated_required_check_exists_and_checks_are_pending() {
1101        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1102        let server = MockServer::start().await;
1103        mock_classic_required(&server, "cla/check", "deadbeef").await;
1104        mock_runs_and_empty_status(
1105            &server,
1106            "deadbeef",
1107            serde_json::json!([{
1108                "id": 1,
1109                "name": "test",
1110                "status": "in_progress",
1111                "conclusion": null,
1112                "app": { "id": 15368 }
1113            }]),
1114        )
1115        .await;
1116        Mock::given(method("POST"))
1117            .and(path("/repos/o/r/rulesets"))
1118            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1119                "id": 9
1120            })))
1121            .expect(0)
1122            .mount(&server)
1123            .await;
1124
1125        let (store, task_manager, id) = pending_store().await;
1126        let gh = GhHttp::for_test(server.uri(), None);
1127        apply_pending_with(&gh, &store, &task_manager, "o/r")
1128            .await
1129            .unwrap();
1130
1131        let stored = store.get(id).await.unwrap();
1132        assert_eq!(stored.status, TaskStatus::Approved);
1133        assert_eq!(
1134            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1135            Some(KV_PENDING)
1136        );
1137    }
1138
1139    #[tokio::test]
1140    async fn apply_pending_dones_when_name_taken_and_orch_ruleset_lists_the_checks() {
1141        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1142        let server = MockServer::start().await;
1143        mock_unprotected_main(&server, "deadbeef").await;
1144        mock_runs_and_empty_status(
1145            &server,
1146            "deadbeef",
1147            serde_json::json!([{
1148                "id": 1,
1149                "name": "test",
1150                "status": "completed",
1151                "conclusion": "success",
1152                "app": { "id": 15368 }
1153            }]),
1154        )
1155        .await;
1156        mock_name_taken_ruleset(
1157            &server,
1158            serde_json::json!([{ "context": "test", "integration_id": 15368 }]),
1159        )
1160        .await;
1161        Mock::given(method("PUT"))
1162            .and(path("/repos/o/r/rulesets/9"))
1163            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1164                "id": 9
1165            })))
1166            .expect(0)
1167            .mount(&server)
1168            .await;
1169
1170        let (store, task_manager, id) = pending_store().await;
1171        let gh = GhHttp::for_test(server.uri(), None);
1172        apply_pending_with(&gh, &store, &task_manager, "o/r")
1173            .await
1174            .unwrap();
1175
1176        let stored = store.get(id).await.unwrap();
1177        assert_eq!(stored.status, TaskStatus::Done);
1178        assert_eq!(
1179            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1180            Some(KV_APPLIED)
1181        );
1182    }
1183
1184    #[tokio::test]
1185    async fn apply_pending_updates_the_orch_ruleset_when_name_taken_and_checks_are_missing() {
1186        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1187        let server = MockServer::start().await;
1188        mock_unprotected_main(&server, "deadbeef").await;
1189        mock_runs_and_empty_status(
1190            &server,
1191            "deadbeef",
1192            serde_json::json!([{
1193                "id": 1,
1194                "name": "test",
1195                "status": "completed",
1196                "conclusion": "success",
1197                "app": { "id": 15368 }
1198            }]),
1199        )
1200        .await;
1201        mock_name_taken_ruleset(&server, serde_json::json!([{ "context": "cla/check" }])).await;
1202        Mock::given(method("PUT"))
1203            .and(path("/repos/o/r/rulesets/9"))
1204            .and(body_partial_json(serde_json::json!({
1205                "name": REQUIRED_CHECK_RULESET_NAME,
1206                "rules": [{
1207                    "type": "required_status_checks",
1208                    "parameters": {
1209                        "required_status_checks": [
1210                            { "context": "cla/check" },
1211                            { "context": "test", "integration_id": 15368 }
1212                        ]
1213                    }
1214                }]
1215            })))
1216            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1217                "id": 9,
1218                "name": REQUIRED_CHECK_RULESET_NAME
1219            })))
1220            .expect(1)
1221            .mount(&server)
1222            .await;
1223
1224        let (store, task_manager, id) = pending_store().await;
1225        let gh = GhHttp::for_test(server.uri(), None);
1226        apply_pending_with(&gh, &store, &task_manager, "o/r")
1227            .await
1228            .unwrap();
1229
1230        let stored = store.get(id).await.unwrap();
1231        assert_eq!(stored.status, TaskStatus::Done);
1232        assert_eq!(
1233            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1234            Some(KV_APPLIED)
1235        );
1236    }
1237
1238    #[tokio::test]
1239    async fn apply_pending_creates_a_ruleset_when_default_branch_checks_pass() {
1240        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1241        let server = MockServer::start().await;
1242        Mock::given(method("GET"))
1243            .and(path("/repos/o/r"))
1244            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1245                "default_branch": "main"
1246            })))
1247            .mount(&server)
1248            .await;
1249        Mock::given(method("GET"))
1250            .and(path("/repos/o/r/branches/main/protection"))
1251            .respond_with(
1252                ResponseTemplate::new(404)
1253                    .set_body_string("{\"message\":\"Branch not protected\"}"),
1254            )
1255            .mount(&server)
1256            .await;
1257        Mock::given(method("GET"))
1258            .and(path("/repos/o/r/rules/branches/main"))
1259            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
1260            .mount(&server)
1261            .await;
1262        Mock::given(method("GET"))
1263            .and(path("/repos/o/r/branches/main"))
1264            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1265                "commit": { "sha": "deadbeef" }
1266            })))
1267            .mount(&server)
1268            .await;
1269        Mock::given(method("GET"))
1270            .and(path("/repos/o/r/commits/deadbeef/check-runs"))
1271            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1272                "total_count": 1,
1273                "check_runs": [{
1274                    "id": 1,
1275                    "name": "test",
1276                    "status": "completed",
1277                    "conclusion": "success",
1278                    "app": { "id": 15368 }
1279                }]
1280            })))
1281            .mount(&server)
1282            .await;
1283        Mock::given(method("GET"))
1284            .and(path("/repos/o/r/commits/deadbeef/status"))
1285            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1286                "state": "success",
1287                "total_count": 0,
1288                "statuses": []
1289            })))
1290            .mount(&server)
1291            .await;
1292        Mock::given(method("POST"))
1293            .and(path("/repos/o/r/rulesets"))
1294            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1295                "id": 9,
1296                "name": "orch required checks"
1297            })))
1298            .expect(1)
1299            .mount(&server)
1300            .await;
1301
1302        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1303        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
1304        let task_manager = Arc::new(TaskManager::with_store(
1305            backend,
1306            Arc::clone(&store),
1307            "o/r".to_string(),
1308        ));
1309        let id = store
1310            .upsert_external(&UpsertExternal {
1311                repo: "o/r",
1312                ext_id: "12",
1313                title: "CI-01 Set up CI and tests",
1314                body: &bootstrap_body(),
1315                author: "u",
1316                url: "",
1317                labels: &[],
1318                origin: "github",
1319            })
1320            .await
1321            .unwrap();
1322        store.update_status(id, TaskStatus::Approved).await.unwrap();
1323        store
1324            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
1325            .await
1326            .unwrap();
1327
1328        let gh = GhHttp::for_test(server.uri(), None);
1329        apply_pending_with(&gh, &store, &task_manager, "o/r")
1330            .await
1331            .unwrap();
1332
1333        let stored = store.get(id).await.unwrap();
1334        assert_eq!(stored.status, TaskStatus::Done);
1335        assert_eq!(
1336            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1337            Some(KV_APPLIED)
1338        );
1339    }
1340
1341    #[tokio::test]
1342    async fn apply_pending_comments_and_holds_when_github_refuses_the_ruleset() {
1343        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1344        let server = MockServer::start().await;
1345        Mock::given(method("GET"))
1346            .and(path("/repos/o/r"))
1347            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1348                "default_branch": "main"
1349            })))
1350            .mount(&server)
1351            .await;
1352        Mock::given(method("GET"))
1353            .and(path("/repos/o/r/branches/main/protection"))
1354            .respond_with(
1355                ResponseTemplate::new(404)
1356                    .set_body_string("{\"message\":\"Branch not protected\"}"),
1357            )
1358            .mount(&server)
1359            .await;
1360        Mock::given(method("GET"))
1361            .and(path("/repos/o/r/rules/branches/main"))
1362            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
1363            .mount(&server)
1364            .await;
1365        Mock::given(method("GET"))
1366            .and(path("/repos/o/r/branches/main"))
1367            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1368                "commit": { "sha": "deadbeef" }
1369            })))
1370            .mount(&server)
1371            .await;
1372        Mock::given(method("GET"))
1373            .and(path("/repos/o/r/commits/deadbeef/check-runs"))
1374            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1375                "total_count": 1,
1376                "check_runs": [{
1377                    "id": 1,
1378                    "name": "test",
1379                    "status": "completed",
1380                    "conclusion": "success",
1381                    "app": { "id": 15368 }
1382                }]
1383            })))
1384            .mount(&server)
1385            .await;
1386        Mock::given(method("GET"))
1387            .and(path("/repos/o/r/commits/deadbeef/status"))
1388            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1389                "state": "success",
1390                "total_count": 0,
1391                "statuses": []
1392            })))
1393            .mount(&server)
1394            .await;
1395        Mock::given(method("POST"))
1396            .and(path("/repos/o/r/rulesets"))
1397            .respond_with(ResponseTemplate::new(403).set_body_json(serde_json::json!({
1398                "message": "Upgrade to GitHub Pro or make this repository public to enable this feature."
1399            })))
1400            .mount(&server)
1401            .await;
1402        Mock::given(method("POST"))
1403            .and(path("/repos/o/r/issues/12/comments"))
1404            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1405                "id": 1
1406            })))
1407            .expect(1)
1408            .mount(&server)
1409            .await;
1410
1411        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1412        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
1413        let task_manager = Arc::new(TaskManager::with_store(
1414            backend,
1415            Arc::clone(&store),
1416            "o/r".to_string(),
1417        ));
1418        let id = store
1419            .upsert_external(&UpsertExternal {
1420                repo: "o/r",
1421                ext_id: "12",
1422                title: "CI-01 Set up CI and tests",
1423                body: &bootstrap_body(),
1424                author: "u",
1425                url: "",
1426                labels: &[],
1427                origin: "github",
1428            })
1429            .await
1430            .unwrap();
1431        store.update_status(id, TaskStatus::Approved).await.unwrap();
1432        store
1433            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
1434            .await
1435            .unwrap();
1436
1437        let gh = GhHttp::for_test(server.uri(), None);
1438        apply_pending_with(&gh, &store, &task_manager, "o/r")
1439            .await
1440            .unwrap();
1441
1442        let stored = store.get(id).await.unwrap();
1443        assert_eq!(stored.status, TaskStatus::Approved);
1444        assert_eq!(
1445            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1446            Some(KV_REFUSED)
1447        );
1448    }
1449
1450    #[tokio::test]
1451    async fn apply_pending_reopens_a_closed_issue_and_creates_the_ruleset() {
1452        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1453        let server = MockServer::start().await;
1454        Mock::given(method("GET"))
1455            .and(path("/repos/o/r"))
1456            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1457                "default_branch": "main"
1458            })))
1459            .mount(&server)
1460            .await;
1461        Mock::given(method("GET"))
1462            .and(path("/repos/o/r/branches/main/protection"))
1463            .respond_with(
1464                ResponseTemplate::new(404)
1465                    .set_body_string("{\"message\":\"Branch not protected\"}"),
1466            )
1467            .mount(&server)
1468            .await;
1469        Mock::given(method("GET"))
1470            .and(path("/repos/o/r/rules/branches/main"))
1471            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
1472            .mount(&server)
1473            .await;
1474        Mock::given(method("GET"))
1475            .and(path("/repos/o/r/issues/12"))
1476            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1477                "number": 12,
1478                "id": 120,
1479                "title": "CI-01 Set up CI and tests",
1480                "body": "",
1481                "state": "closed",
1482                "labels": [],
1483                "user": { "login": "u" },
1484                "created_at": "",
1485                "updated_at": "",
1486                "html_url": ""
1487            })))
1488            .mount(&server)
1489            .await;
1490        Mock::given(method("PATCH"))
1491            .and(path("/repos/o/r/issues/12"))
1492            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1493                "number": 12,
1494                "id": 120,
1495                "title": "CI-01 Set up CI and tests",
1496                "body": "",
1497                "state": "open",
1498                "labels": [],
1499                "user": { "login": "u" },
1500                "created_at": "",
1501                "updated_at": "",
1502                "html_url": ""
1503            })))
1504            .expect(1)
1505            .mount(&server)
1506            .await;
1507        Mock::given(method("GET"))
1508            .and(path("/repos/o/r/branches/main"))
1509            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1510                "commit": { "sha": "deadbeef" }
1511            })))
1512            .mount(&server)
1513            .await;
1514        Mock::given(method("GET"))
1515            .and(path("/repos/o/r/commits/deadbeef/check-runs"))
1516            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1517                "total_count": 1,
1518                "check_runs": [{
1519                    "id": 1,
1520                    "name": "test",
1521                    "status": "completed",
1522                    "conclusion": "success",
1523                    "app": { "id": 15368 }
1524                }]
1525            })))
1526            .mount(&server)
1527            .await;
1528        Mock::given(method("GET"))
1529            .and(path("/repos/o/r/commits/deadbeef/status"))
1530            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1531                "state": "success",
1532                "total_count": 0,
1533                "statuses": []
1534            })))
1535            .mount(&server)
1536            .await;
1537        Mock::given(method("POST"))
1538            .and(path("/repos/o/r/rulesets"))
1539            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1540                "id": 9,
1541                "name": "orch required checks"
1542            })))
1543            .expect(1)
1544            .mount(&server)
1545            .await;
1546
1547        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1548        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
1549        let task_manager = Arc::new(TaskManager::with_store(
1550            backend,
1551            Arc::clone(&store),
1552            "o/r".to_string(),
1553        ));
1554        let id = store
1555            .upsert_external(&UpsertExternal {
1556                repo: "o/r",
1557                ext_id: "12",
1558                title: "CI-01 Set up CI and tests",
1559                body: &bootstrap_body(),
1560                author: "u",
1561                url: "",
1562                labels: &[],
1563                origin: "github",
1564            })
1565            .await
1566            .unwrap();
1567        store.update_status(id, TaskStatus::Done).await.unwrap();
1568        store
1569            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
1570            .await
1571            .unwrap();
1572
1573        let gh = GhHttp::for_test(server.uri(), None);
1574        apply_pending_with(&gh, &store, &task_manager, "o/r")
1575            .await
1576            .unwrap();
1577
1578        let stored = store.get(id).await.unwrap();
1579        assert_eq!(stored.status, TaskStatus::Done);
1580        assert_eq!(
1581            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1582            Some(KV_APPLIED)
1583        );
1584    }
1585
1586    #[tokio::test]
1587    async fn apply_pending_logs_and_holds_while_default_branch_checks_run() {
1588        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1589        let server = MockServer::start().await;
1590        mock_unprotected_main(&server, "deadbeef").await;
1591        mock_runs_and_empty_status(
1592            &server,
1593            "deadbeef",
1594            serde_json::json!([{
1595                "id": 1,
1596                "name": "test",
1597                "status": "in_progress",
1598                "conclusion": null,
1599                "app": { "id": 15368 }
1600            }]),
1601        )
1602        .await;
1603        mock_issue_comment(&server, 0).await;
1604        Mock::given(method("POST"))
1605            .and(path("/repos/o/r/rulesets"))
1606            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1607                "id": 9
1608            })))
1609            .expect(0)
1610            .mount(&server)
1611            .await;
1612
1613        let (store, task_manager, id) = pending_store().await;
1614        let gh = GhHttp::for_test(server.uri(), None);
1615        apply_pending_with(&gh, &store, &task_manager, "o/r")
1616            .await
1617            .unwrap();
1618
1619        let stored = store.get(id).await.unwrap();
1620        assert_eq!(stored.status, TaskStatus::Approved);
1621        assert_eq!(
1622            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1623            Some(KV_PENDING)
1624        );
1625    }
1626
1627    #[tokio::test]
1628    async fn apply_pending_comments_and_routes_when_default_branch_checks_fail() {
1629        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1630        let server = MockServer::start().await;
1631        mock_unprotected_main(&server, "deadbeef").await;
1632        mock_runs_and_empty_status(
1633            &server,
1634            "deadbeef",
1635            serde_json::json!([{
1636                "id": 1,
1637                "name": "test",
1638                "status": "completed",
1639                "conclusion": "failure",
1640                "app": { "id": 15368 }
1641            }]),
1642        )
1643        .await;
1644        mock_issue_comment(&server, 1).await;
1645
1646        let (store, task_manager, id) = pending_store().await;
1647        let gh = GhHttp::for_test(server.uri(), None);
1648        apply_pending_with(&gh, &store, &task_manager, "o/r")
1649            .await
1650            .unwrap();
1651
1652        let stored = store.get(id).await.unwrap();
1653        assert_eq!(stored.status, TaskStatus::Routed);
1654        assert_eq!(stored.ci_merge_failures, 1);
1655        assert!(stored.last_error.contains("test: failure"));
1656        assert_eq!(
1657            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1658            Some(KV_PENDING)
1659        );
1660
1661        apply_pending_with(&gh, &store, &task_manager, "o/r")
1662            .await
1663            .unwrap();
1664        let stored = store.get(id).await.unwrap();
1665        assert_eq!(stored.status, TaskStatus::Routed);
1666        assert_eq!(stored.ci_merge_failures, 1);
1667    }
1668
1669    #[tokio::test]
1670    async fn apply_pending_blocks_after_max_failed_default_branch_heads() {
1671        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1672        let server = MockServer::start().await;
1673        mock_unprotected_main(&server, "deadbeef").await;
1674        mock_runs_and_empty_status(
1675            &server,
1676            "deadbeef",
1677            serde_json::json!([{
1678                "id": 1,
1679                "name": "test",
1680                "status": "completed",
1681                "conclusion": "cancelled",
1682                "app": { "id": 15368 }
1683            }]),
1684        )
1685        .await;
1686        mock_issue_comment(&server, 1).await;
1687
1688        let (store, task_manager, id) = pending_store().await;
1689        store.increment(id, "ci_merge_failures").await.unwrap();
1690        store.increment(id, "ci_merge_failures").await.unwrap();
1691        let gh = GhHttp::for_test(server.uri(), None);
1692        apply_pending_with(&gh, &store, &task_manager, "o/r")
1693            .await
1694            .unwrap();
1695
1696        let stored = store.get(id).await.unwrap();
1697        assert_eq!(stored.status, TaskStatus::Blocked);
1698        assert_eq!(
1699            stored.ci_merge_failures,
1700            i32::try_from(auto_merge::MAX_CI_MERGE_FAILURES).unwrap()
1701        );
1702        assert!(
1703            stored
1704                .block_reason
1705                .as_deref()
1706                .unwrap_or("")
1707                .contains("CI failure limit")
1708        );
1709    }
1710
1711    #[tokio::test]
1712    async fn apply_pending_waits_before_commenting_when_no_check_starts() {
1713        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1714        let server = MockServer::start().await;
1715        mock_unprotected_main(&server, "deadbeef").await;
1716        mock_runs_and_empty_status(&server, "deadbeef", serde_json::json!([])).await;
1717        mock_issue_comment(&server, 0).await;
1718
1719        let (store, task_manager, id) = pending_store().await;
1720        let gh = GhHttp::for_test(server.uri(), None);
1721        apply_pending_with(&gh, &store, &task_manager, "o/r")
1722            .await
1723            .unwrap();
1724
1725        let stored = store.get(id).await.unwrap();
1726        assert_eq!(stored.status, TaskStatus::Approved);
1727        assert!(
1728            store
1729                .kv_get("ci_bootstrap_wait_since:o/r#12#deadbeef")
1730                .await
1731                .unwrap()
1732                .is_some()
1733        );
1734        assert!(
1735            store
1736                .kv_get(&no_checks_comment_key("o/r", "12", "deadbeef"))
1737                .await
1738                .unwrap()
1739                .is_none()
1740        );
1741    }
1742
1743    #[tokio::test]
1744    async fn apply_pending_comments_once_when_no_check_starts_after_the_wait() {
1745        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1746        let server = MockServer::start().await;
1747        mock_unprotected_main(&server, "deadbeef").await;
1748        mock_runs_and_empty_status(&server, "deadbeef", serde_json::json!([])).await;
1749        mock_issue_comment(&server, 1).await;
1750
1751        let (store, task_manager, id) = pending_store().await;
1752        let started = chrono::Utc::now().timestamp() - review_gate::NO_CHECKS_WAIT_SECS - 1;
1753        store
1754            .kv_set(
1755                "ci_bootstrap_wait_since:o/r#12#deadbeef",
1756                &started.to_string(),
1757            )
1758            .await
1759            .unwrap();
1760        let gh = GhHttp::for_test(server.uri(), None);
1761        apply_pending_with(&gh, &store, &task_manager, "o/r")
1762            .await
1763            .unwrap();
1764
1765        let stored = store.get(id).await.unwrap();
1766        assert_eq!(stored.status, TaskStatus::Approved);
1767        assert_eq!(
1768            store
1769                .kv_get(&no_checks_comment_key("o/r", "12", "deadbeef"))
1770                .await
1771                .unwrap()
1772                .as_deref(),
1773            Some("1")
1774        );
1775
1776        apply_pending_with(&gh, &store, &task_manager, "o/r")
1777            .await
1778            .unwrap();
1779        assert_eq!(store.get(id).await.unwrap().status, TaskStatus::Approved);
1780    }
1781
1782    #[tokio::test]
1783    async fn apply_pending_comments_once_when_github_says_the_check_never_ran() {
1784        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1785        let server = MockServer::start().await;
1786        mock_unprotected_main(&server, "deadbeef").await;
1787        mock_runs_and_empty_status(
1788            &server,
1789            "deadbeef",
1790            serde_json::json!([{
1791                "id": 1,
1792                "name": "test",
1793                "status": "completed",
1794                "conclusion": "success",
1795                "app": { "id": 15368 }
1796            }]),
1797        )
1798        .await;
1799        Mock::given(method("POST"))
1800            .and(path("/repos/o/r/rulesets"))
1801            .respond_with(ResponseTemplate::new(422).set_body_json(serde_json::json!({
1802                "message": "test was not a valid check"
1803            })))
1804            .mount(&server)
1805            .await;
1806        mock_issue_comment(&server, 1).await;
1807
1808        let (store, task_manager, id) = pending_store().await;
1809        let gh = GhHttp::for_test(server.uri(), None);
1810        apply_pending_with(&gh, &store, &task_manager, "o/r")
1811            .await
1812            .unwrap();
1813        apply_pending_with(&gh, &store, &task_manager, "o/r")
1814            .await
1815            .unwrap();
1816
1817        let stored = store.get(id).await.unwrap();
1818        assert_eq!(stored.status, TaskStatus::Approved);
1819        assert_eq!(
1820            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1821            Some(KV_PENDING)
1822        );
1823        assert_eq!(
1824            store
1825                .kv_get(&never_run_comment_key("o/r", "12"))
1826                .await
1827                .unwrap()
1828                .as_deref(),
1829            Some("1")
1830        );
1831    }
1832}