Skip to main content

orch/engine/
ci_bootstrap.rs

1//! After a CI bootstrap task merges, require its default-branch checks
2
3use crate::backends::{ExternalId, ExternalTask, Status};
4use crate::engine;
5use crate::engine::auto_merge;
6use crate::engine::review_gate;
7use crate::engine::tasks::TaskManager;
8use crate::github::http::{
9    GhHttp, REQUIRED_CHECK_RULESET_NAME, RulesetCreateKind, classify_ruleset_create_error,
10};
11use crate::github::types::{GitHubCheckRun, RequiredStatusCheck};
12use crate::plan;
13use crate::store;
14use crate::store::TaskStore;
15use std::sync::Arc;
16
17const KV_PENDING: &str = "pending";
18const KV_APPLIED: &str = "applied";
19const KV_REFUSED: &str = "refused";
20
21#[derive(Debug, PartialEq)]
22pub(crate) enum ChecksReady {
23    Ready(Vec<RequiredStatusCheck>),
24    /// A real check is still queued, in progress, or waiting on a person
25    Pending,
26    /// A real check finished as failure, error, cancelled, or timed out
27    Failed(Vec<FailedCheck>),
28    /// No real check has started
29    Empty,
30}
31
32#[derive(Debug, PartialEq)]
33pub(crate) struct FailedCheck {
34    name: String,
35    conclusion: String,
36}
37
38pub(crate) fn is_ci_bootstrap_task(body: &str) -> bool {
39    plan::is_ci_bootstrap_body(body)
40        || plan::marker_id(body).is_some_and(|id| plan::is_ci_bootstrap_id(&id))
41}
42
43pub(crate) fn kv_key(repo: &str, task_id: &str) -> String {
44    format!("ci-bootstrap:{repo}:{task_id}")
45}
46
47/// True when this merged CI bootstrap task must stay approved until a ruleset exists
48pub(crate) async fn hold_after_merge(
49    task: &ExternalTask,
50    store: &Arc<TaskStore>,
51    repo: &str,
52) -> bool {
53    if !is_ci_bootstrap_task(&task.body) {
54        return false;
55    }
56    let key = kv_key(repo, &task.id.0);
57    match store.kv_get(&key).await {
58        Ok(Some(state)) if state == KV_APPLIED => false,
59        Ok(Some(_)) => true,
60        Ok(None) => {
61            if let Err(e) = store.kv_set(&key, KV_PENDING).await {
62                tracing::warn!(
63                    task_id = %task.id.0,
64                    err = %e,
65                    "failed to record CI bootstrap pending"
66                );
67            }
68            true
69        }
70        Err(e) => {
71            tracing::warn!(
72                task_id = %task.id.0,
73                err = %e,
74                "failed to read CI bootstrap state"
75            );
76            true
77        }
78    }
79}
80
81pub(crate) async fn apply_pending(
82    store: &Arc<TaskStore>,
83    task_manager: &Arc<TaskManager>,
84    repo: &str,
85) -> anyhow::Result<()> {
86    let gh = GhHttp::owner()?;
87    apply_pending_with(&gh, store, task_manager, repo).await
88}
89
90pub(crate) async fn apply_pending_with(
91    gh: &GhHttp,
92    store: &Arc<TaskStore>,
93    task_manager: &Arc<TaskManager>,
94    repo: &str,
95) -> anyhow::Result<()> {
96    let prefix = format!("ci-bootstrap:{repo}:");
97    let entries = store.kv_list_prefix(&prefix).await?;
98    for (key, state) in entries {
99        if state == KV_APPLIED {
100            continue;
101        }
102        let Some(task_id) = key.strip_prefix(&prefix) else {
103            continue;
104        };
105        if let Err(e) = apply_one(gh, store, task_manager, repo, task_id, &state).await {
106            tracing::warn!(task_id, err = %e, "CI bootstrap ruleset apply failed");
107        }
108    }
109    Ok(())
110}
111
112pub(crate) fn checks_ready_for_ruleset(
113    runs: &[GitHubCheckRun],
114    statuses: &[(String, String)],
115) -> ChecksReady {
116    let mut ready: Vec<RequiredStatusCheck> = Vec::new();
117    let mut pending = false;
118    let mut failed: Vec<FailedCheck> = Vec::new();
119
120    for run in runs {
121        if review_gate::is_own_check(&run.name) {
122            continue;
123        }
124        if !run.is_finished() {
125            pending = true;
126            continue;
127        }
128        match run.conclusion.as_deref() {
129            Some("success") => {
130                let integration_id = run.app.as_ref().map(|app| app.id).filter(|&id| id > 0);
131                if !ready.iter().any(|check| check.context == run.name) {
132                    ready.push(RequiredStatusCheck {
133                        context: run.name.clone(),
134                        integration_id,
135                    });
136                }
137            }
138            Some("skipped" | "neutral") => {}
139            Some(conclusion) => failed.push(FailedCheck {
140                name: run.name.clone(),
141                conclusion: conclusion.to_string(),
142            }),
143            None => pending = true,
144        }
145    }
146
147    for (context, state) in statuses {
148        if review_gate::is_own_check(context) {
149            continue;
150        }
151        match state.as_str() {
152            "pending" => pending = true,
153            "success" => {
154                if !ready.iter().any(|check| &check.context == context) {
155                    ready.push(RequiredStatusCheck {
156                        context: context.clone(),
157                        integration_id: None,
158                    });
159                }
160            }
161            "failure" | "error" => failed.push(FailedCheck {
162                name: context.clone(),
163                conclusion: state.clone(),
164            }),
165            _ => {}
166        }
167    }
168
169    if !failed.is_empty() {
170        ChecksReady::Failed(failed)
171    } else if pending {
172        ChecksReady::Pending
173    } else if ready.is_empty() {
174        ChecksReady::Empty
175    } else {
176        ChecksReady::Ready(ready)
177    }
178}
179
180fn required_contexts_cover(
181    required: &[RequiredStatusCheck],
182    needed: &[RequiredStatusCheck],
183) -> bool {
184    needed.iter().all(|check| {
185        required
186            .iter()
187            .any(|existing| existing.context == check.context)
188    })
189}
190
191fn merge_missing_checks(
192    mut existing: Vec<RequiredStatusCheck>,
193    needed: &[RequiredStatusCheck],
194) -> Vec<RequiredStatusCheck> {
195    for check in needed {
196        if !existing.iter().any(|e| e.context == check.context) {
197            existing.push(check.clone());
198        }
199    }
200    existing
201}
202
203async fn apply_one(
204    gh: &GhHttp,
205    store: &Arc<TaskStore>,
206    task_manager: &Arc<TaskManager>,
207    repo: &str,
208    task_id: &str,
209    state: &str,
210) -> anyhow::Result<()> {
211    if store::opt_store_get_task(&Some(Arc::clone(store)), repo, task_id)
212        .await
213        .is_none()
214    {
215        return Ok(());
216    }
217
218    let branch = gh.get_default_branch(repo).await?;
219    let required = gh.get_required_status_check_contexts(repo, &branch).await?;
220    let sha = gh.get_branch_sha(repo, &branch).await?;
221    let runs = gh.get_check_runs(repo, &sha).await?;
222    let statuses = gh.get_commit_status_contexts(repo, &sha).await?;
223    let checks = match checks_ready_for_ruleset(&runs, &statuses) {
224        ChecksReady::Pending => {
225            tracing::info!(
226                task_id,
227                sha = %sha,
228                "CI bootstrap waiting for default-branch checks"
229            );
230            reopen_if_closed(gh, repo, task_id).await;
231            clear_wait(store, repo, task_id).await;
232            return Ok(());
233        }
234        ChecksReady::Empty => {
235            reopen_if_closed(gh, repo, task_id).await;
236            return wait_for_checks(gh, store, repo, task_id, &sha).await;
237        }
238        ChecksReady::Failed(failed) => {
239            reopen_if_closed(gh, repo, task_id).await;
240            clear_wait(store, repo, task_id).await;
241            return handle_failed_checks(gh, store, task_manager, repo, task_id, &sha, &failed)
242                .await;
243        }
244        ChecksReady::Ready(checks) => {
245            clear_wait(store, repo, task_id).await;
246            checks
247        }
248    };
249
250    if required_contexts_cover(&required, &checks) {
251        return finish_applied(store, task_manager, task_id, repo).await;
252    }
253
254    // GitHub may have closed the issue via Closes on the PR, keep it open until the ruleset exists
255    reopen_if_closed(gh, repo, task_id).await;
256    if state == KV_REFUSED {
257        return Ok(());
258    }
259
260    match gh.create_required_check_ruleset(repo, &checks).await {
261        Ok(()) => finish_applied(store, task_manager, task_id, repo).await,
262        Err(e) => {
263            let err = e.to_string();
264            match classify_ruleset_create_error(&err) {
265                RulesetCreateKind::NameTaken => {
266                    apply_existing_orch_ruleset(gh, store, task_manager, repo, task_id, &checks)
267                        .await
268                }
269                kind => handle_ruleset_write_error(gh, store, repo, task_id, &err, kind).await,
270            }
271        }
272    }
273}
274
275async fn apply_existing_orch_ruleset(
276    gh: &GhHttp,
277    store: &Arc<TaskStore>,
278    task_manager: &Arc<TaskManager>,
279    repo: &str,
280    task_id: &str,
281    needed: &[RequiredStatusCheck],
282) -> anyhow::Result<()> {
283    let Some((id, existing)) = gh
284        .get_named_ruleset_required_checks(repo, REQUIRED_CHECK_RULESET_NAME)
285        .await?
286    else {
287        tracing::info!(
288            task_id,
289            "CI bootstrap ruleset name taken, existing ruleset not found"
290        );
291        return Ok(());
292    };
293    if required_contexts_cover(&existing, needed) {
294        return finish_applied(store, task_manager, task_id, repo).await;
295    }
296    let merged = merge_missing_checks(existing, needed);
297    match gh.update_required_check_ruleset(repo, id, &merged).await {
298        Ok(()) => finish_applied(store, task_manager, task_id, repo).await,
299        Err(e) => {
300            let err = e.to_string();
301            handle_ruleset_write_error(
302                gh,
303                store,
304                repo,
305                task_id,
306                &err,
307                classify_ruleset_create_error(&err),
308            )
309            .await
310        }
311    }
312}
313
314async fn handle_ruleset_write_error(
315    gh: &GhHttp,
316    store: &Arc<TaskStore>,
317    repo: &str,
318    task_id: &str,
319    err: &str,
320    kind: RulesetCreateKind,
321) -> anyhow::Result<()> {
322    match kind {
323        RulesetCreateKind::NameTaken | RulesetCreateKind::Retryable | RulesetCreateKind::Other => {
324            tracing::info!(
325                task_id,
326                err = %err,
327                "CI bootstrap ruleset create retrying"
328            );
329            Ok(())
330        }
331        RulesetCreateKind::CheckNeverRun => {
332            tracing::info!(
333                task_id,
334                err = %err,
335                "CI bootstrap ruleset create waiting, check has not run"
336            );
337            comment_never_run_once(gh, store, repo, task_id, err).await;
338            Ok(())
339        }
340        RulesetCreateKind::Refused => {
341            comment_refused(gh, repo, task_id, err).await;
342            store.kv_set(&kv_key(repo, task_id), KV_REFUSED).await?;
343            Ok(())
344        }
345    }
346}
347
348async fn finish_applied(
349    store: &Arc<TaskStore>,
350    task_manager: &Arc<TaskManager>,
351    task_id: &str,
352    repo: &str,
353) -> anyhow::Result<()> {
354    store.kv_set(&kv_key(repo, task_id), KV_APPLIED).await?;
355    task_manager
356        .update_task_status(&ExternalId(task_id.to_string()), Status::Done)
357        .await?;
358    Ok(())
359}
360
361async fn reopen_if_closed(gh: &GhHttp, repo: &str, task_id: &str) {
362    match gh.get_issue(repo, task_id).await {
363        Ok(issue) if issue.state.eq_ignore_ascii_case("closed") => {
364            if let Err(e) = gh.reopen_issue(repo, task_id).await {
365                tracing::warn!(
366                    task_id,
367                    err = %e,
368                    "failed to reopen CI bootstrap issue"
369                );
370            }
371        }
372        Ok(_) => {}
373        Err(e) => {
374            tracing::warn!(
375                task_id,
376                err = %e,
377                "failed to read CI bootstrap issue"
378            );
379        }
380    }
381}
382
383fn wait_prefix(repo: &str, task_id: &str) -> String {
384    format!("ci_bootstrap_wait_since:{repo}#{task_id}#")
385}
386
387fn no_checks_comment_key(repo: &str, task_id: &str, sha: &str) -> String {
388    format!("ci_bootstrap_no_checks_comment:{repo}#{task_id}#{sha}")
389}
390
391fn never_run_comment_key(repo: &str, task_id: &str) -> String {
392    format!("ci_bootstrap_never_run_comment:{repo}#{task_id}")
393}
394
395fn format_failed(failed: &[FailedCheck]) -> String {
396    failed
397        .iter()
398        .map(|check| format!("{}: {}", check.name, check.conclusion))
399        .collect::<Vec<_>>()
400        .join(", ")
401}
402
403async fn clear_wait(store: &Arc<TaskStore>, repo: &str, task_id: &str) {
404    let prefix = wait_prefix(repo, task_id);
405    let keys = match store.kv_list_prefix(&prefix).await {
406        Ok(entries) => entries,
407        Err(e) => {
408            tracing::warn!(task_id, err = %e, "failed to list CI bootstrap wait keys");
409            return;
410        }
411    };
412    for (key, _) in keys {
413        if let Err(e) = store.kv_delete(&key).await {
414            tracing::warn!(task_id, err = %e, "failed to clear CI bootstrap wait key");
415        }
416    }
417}
418
419async fn wait_for_checks(
420    gh: &GhHttp,
421    store: &Arc<TaskStore>,
422    repo: &str,
423    task_id: &str,
424    sha: &str,
425) -> anyhow::Result<()> {
426    let prefix = wait_prefix(repo, task_id);
427    let key = format!("{prefix}{sha}");
428    let now = chrono::Utc::now().timestamp();
429    let since = review_gate::wait_since(store, &prefix, &key, now).await?;
430    if now - since < review_gate::NO_CHECKS_WAIT_SECS {
431        tracing::info!(
432            task_id,
433            sha = %sha,
434            "CI bootstrap waiting for a default-branch check to start"
435        );
436        return Ok(());
437    }
438    let wait_mins = review_gate::NO_CHECKS_WAIT_SECS / 60;
439    tracing::warn!(
440        task_id,
441        sha = %sha,
442        wait_mins,
443        "CI bootstrap has no default-branch check after the wait"
444    );
445    let comment_key = no_checks_comment_key(repo, task_id, sha);
446    if store.kv_get(&comment_key).await?.is_none() {
447        comment_issue(
448            gh,
449            repo,
450            task_id,
451            &format!(
452                "No default-branch check has started {wait_mins} minutes after the CI bootstrap merge. Later tasks stay waiting until a check runs."
453            ),
454        )
455        .await;
456        store.kv_set(&comment_key, "1").await?;
457    }
458    Ok(())
459}
460
461async fn handle_failed_checks(
462    gh: &GhHttp,
463    store: &Arc<TaskStore>,
464    task_manager: &Arc<TaskManager>,
465    repo: &str,
466    task_id: &str,
467    sha: &str,
468    failed: &[FailedCheck],
469) -> anyhow::Result<()> {
470    let details = format_failed(failed);
471    let failures = match auto_merge::count_ci_failure_for_head(store, repo, task_id, sha).await {
472        Ok(Some(v)) => v,
473        Ok(None) => {
474            tracing::info!(
475                task_id,
476                sha = %sha,
477                details,
478                "CI bootstrap default-branch checks still failed on the same head"
479            );
480            return Ok(());
481        }
482        Err(e) => {
483            tracing::warn!(
484                task_id,
485                err = %e,
486                "failed to increment ci_merge_failures for CI bootstrap"
487            );
488            return Ok(());
489        }
490    };
491
492    comment_issue(
493        gh,
494        repo,
495        task_id,
496        &format!(
497            "Default-branch checks failed after the CI bootstrap merge ({details}). Later tasks stay waiting until those checks pass."
498        ),
499    )
500    .await;
501
502    let max = auto_merge::MAX_CI_MERGE_FAILURES;
503    let id = ExternalId(task_id.to_string());
504    if failures >= max {
505        tracing::error!(
506            task_id,
507            failures,
508            details,
509            "CI bootstrap default-branch checks failed, CI failure limit reached"
510        );
511        let fields = [(
512            "block_reason",
513            serde_json::json!(format!(
514                "CI failure limit ({max}) reached after the CI bootstrap merge"
515            )),
516        )];
517        if let Err(e) = task_manager
518            .update_task_status_and_result(&id, Status::Blocked, &fields)
519            .await
520        {
521            tracing::error!(
522                task_id,
523                err = %e,
524                "failed to block CI bootstrap after failed default-branch checks"
525            );
526        }
527    } else {
528        tracing::warn!(
529            task_id,
530            failures,
531            details,
532            "CI bootstrap default-branch checks failed, routing back to the task agent"
533        );
534        let fields = [
535            (
536                "last_error",
537                serde_json::json!(format!(
538                    "CI failed on the default branch after the CI bootstrap merge: {details}. Fix the failing checks"
539                )),
540            ),
541            (
542                "route_reason",
543                serde_json::json!("re-dispatch after CI bootstrap default-branch checks failed"),
544            ),
545        ];
546        if let Err(e) = task_manager
547            .update_task_status_and_result(&id, Status::Routed, &fields)
548            .await
549        {
550            tracing::error!(
551                task_id,
552                err = %e,
553                "failed to re-route CI bootstrap after failed default-branch checks"
554            );
555        }
556    }
557    Ok(())
558}
559
560async fn comment_never_run_once(
561    gh: &GhHttp,
562    store: &Arc<TaskStore>,
563    repo: &str,
564    task_id: &str,
565    err: &str,
566) {
567    let key = never_run_comment_key(repo, task_id);
568    match store.kv_get(&key).await {
569        Ok(Some(_)) => return,
570        Ok(None) => {}
571        Err(e) => {
572            tracing::warn!(
573                task_id,
574                err = %e,
575                "failed to read CI bootstrap never-run comment key"
576            );
577            return;
578        }
579    }
580    comment_issue(
581        gh,
582        repo,
583        task_id,
584        &format!(
585            "GitHub does not treat the default-branch checks as valid required checks ({err}). Later tasks stay waiting until a required-checks ruleset can be created."
586        ),
587    )
588    .await;
589    if let Err(e) = store.kv_set(&key, "1").await {
590        tracing::warn!(
591            task_id,
592            err = %e,
593            "failed to record CI bootstrap never-run comment"
594        );
595    }
596}
597
598async fn comment_refused(gh: &GhHttp, repo: &str, task_id: &str, err: &str) {
599    comment_issue(
600        gh,
601        repo,
602        task_id,
603        &format!(
604            "GitHub refused a required-checks ruleset on the default branch ({err}). Turn branch protection on for this repository. Later tasks stay waiting until that happens."
605        ),
606    )
607    .await;
608}
609
610async fn comment_issue(gh: &GhHttp, repo: &str, task_id: &str, message: &str) {
611    let body = format!("{message}{}", engine::orch_footer());
612    if let Err(e) = gh.add_comment(repo, task_id, &body).await {
613        tracing::warn!(
614            task_id,
615            err = %e,
616            "failed to comment on CI bootstrap issue"
617        );
618    }
619}
620
621#[cfg(test)]
622mod tests {
623    use super::*;
624    use crate::backends::ExternalBackend;
625    use crate::backends::test_helpers::NoopBackend;
626    use crate::github::types::GitHubApp;
627    use crate::plan::{CI_BOOTSTRAP_MARKER, ci_bootstrap_task, issue_body};
628    use crate::store::TaskStatus;
629    use crate::store::UpsertExternal;
630    use std::env;
631    use std::ffi::OsString;
632    use wiremock::matchers::{body_partial_json, method, path};
633    use wiremock::{Mock, MockServer, ResponseTemplate};
634
635    struct EnvVarGuard {
636        key: &'static str,
637        previous: Option<OsString>,
638    }
639
640    impl EnvVarGuard {
641        fn set(key: &'static str, value: &str) -> Self {
642            let previous = env::var_os(key);
643            // SAFETY: nextest runs each test in its own process
644            unsafe { env::set_var(key, value) };
645            Self { key, previous }
646        }
647    }
648
649    impl Drop for EnvVarGuard {
650        fn drop(&mut self) {
651            match self.previous.take() {
652                // SAFETY: nextest runs each test in its own process
653                Some(value) => unsafe { env::set_var(self.key, value) },
654                // SAFETY: nextest runs each test in its own process
655                None => unsafe { env::remove_var(self.key) },
656            }
657        }
658    }
659
660    fn run(
661        name: &str,
662        status: &str,
663        conclusion: Option<&str>,
664        app_id: Option<i64>,
665    ) -> GitHubCheckRun {
666        GitHubCheckRun {
667            id: 1,
668            name: name.to_string(),
669            status: status.to_string(),
670            conclusion: conclusion.map(String::from),
671            app: app_id.map(|id| GitHubApp { id }),
672        }
673    }
674
675    fn bootstrap_body() -> String {
676        issue_body(
677            &ci_bootstrap_task(false),
678            "acme/app",
679            "main",
680            "docs/ROADMAP.md",
681        )
682    }
683
684    async fn pending_store() -> (Arc<TaskStore>, Arc<TaskManager>, i64) {
685        let store = Arc::new(TaskStore::open_memory().await.unwrap());
686        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
687        let task_manager = Arc::new(TaskManager::with_store(
688            backend,
689            Arc::clone(&store),
690            "o/r".to_string(),
691        ));
692        let id = store
693            .upsert_external(&UpsertExternal {
694                repo: "o/r",
695                ext_id: "12",
696                title: "CI-01 Set up CI and tests",
697                body: &bootstrap_body(),
698                author: "u",
699                url: "",
700                labels: &[],
701                origin: "github",
702            })
703            .await
704            .unwrap();
705        store.update_status(id, TaskStatus::Approved).await.unwrap();
706        store
707            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
708            .await
709            .unwrap();
710        (store, task_manager, id)
711    }
712
713    async fn mock_unprotected_main(server: &MockServer, sha: &str) {
714        Mock::given(method("GET"))
715            .and(path("/repos/o/r"))
716            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
717                "default_branch": "main"
718            })))
719            .mount(server)
720            .await;
721        Mock::given(method("GET"))
722            .and(path("/repos/o/r/branches/main/protection"))
723            .respond_with(
724                ResponseTemplate::new(404)
725                    .set_body_string("{\"message\":\"Branch not protected\"}"),
726            )
727            .mount(server)
728            .await;
729        Mock::given(method("GET"))
730            .and(path("/repos/o/r/rules/branches/main"))
731            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
732            .mount(server)
733            .await;
734        Mock::given(method("GET"))
735            .and(path("/repos/o/r/branches/main"))
736            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
737                "commit": { "sha": sha }
738            })))
739            .mount(server)
740            .await;
741    }
742
743    async fn mock_classic_required(server: &MockServer, context: &str, sha: &str) {
744        Mock::given(method("GET"))
745            .and(path("/repos/o/r"))
746            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
747                "default_branch": "main"
748            })))
749            .mount(server)
750            .await;
751        Mock::given(method("GET"))
752            .and(path("/repos/o/r/branches/main/protection"))
753            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
754                "required_status_checks": {
755                    "checks": [{ "context": context, "app_id": 15368 }]
756                }
757            })))
758            .mount(server)
759            .await;
760        Mock::given(method("GET"))
761            .and(path("/repos/o/r/rules/branches/main"))
762            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
763            .mount(server)
764            .await;
765        Mock::given(method("GET"))
766            .and(path("/repos/o/r/branches/main"))
767            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
768                "commit": { "sha": sha }
769            })))
770            .mount(server)
771            .await;
772    }
773
774    async fn mock_name_taken_ruleset(server: &MockServer, ruleset_checks: serde_json::Value) {
775        Mock::given(method("POST"))
776            .and(path("/repos/o/r/rulesets"))
777            .respond_with(ResponseTemplate::new(422).set_body_json(serde_json::json!({
778                "message": "Name has already been taken"
779            })))
780            .expect(1)
781            .mount(server)
782            .await;
783        Mock::given(method("GET"))
784            .and(path("/repos/o/r/rulesets"))
785            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([
786                { "id": 9, "name": REQUIRED_CHECK_RULESET_NAME }
787            ])))
788            .mount(server)
789            .await;
790        Mock::given(method("GET"))
791            .and(path("/repos/o/r/rulesets/9"))
792            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
793                "id": 9,
794                "name": REQUIRED_CHECK_RULESET_NAME,
795                "rules": [{
796                    "type": "required_status_checks",
797                    "parameters": {
798                        "required_status_checks": ruleset_checks
799                    }
800                }]
801            })))
802            .mount(server)
803            .await;
804    }
805
806    async fn mock_runs_and_empty_status(
807        server: &MockServer,
808        sha: &str,
809        check_runs: serde_json::Value,
810    ) {
811        Mock::given(method("GET"))
812            .and(path(format!("/repos/o/r/commits/{sha}/check-runs")))
813            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
814                "total_count": 1,
815                "check_runs": check_runs
816            })))
817            .mount(server)
818            .await;
819        Mock::given(method("GET"))
820            .and(path(format!("/repos/o/r/commits/{sha}/status")))
821            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
822                "state": "pending",
823                "total_count": 0,
824                "statuses": []
825            })))
826            .mount(server)
827            .await;
828    }
829
830    async fn mock_issue_comment(server: &MockServer, expected: u64) {
831        Mock::given(method("POST"))
832            .and(path("/repos/o/r/issues/12/comments"))
833            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
834                "id": 1
835            })))
836            .expect(expected)
837            .mount(server)
838            .await;
839    }
840
841    #[test]
842    fn ruleset_write_keeps_429_transport_and_422_with_401_digits_pending() {
843        for err in [
844            "GitHub API POST /rulesets failed (429 Too Many Requests): {\"message\":\"API rate limit exceeded\"}",
845            "HTTP request failed to send (transport error): connection reset by peer",
846            "failed (422 Unprocessable Entity): {\"message\":\"Ruleset validation failed for rule 401\"}",
847        ] {
848            assert!(
849                !matches!(
850                    classify_ruleset_create_error(err),
851                    RulesetCreateKind::Refused
852                ),
853                "{err}"
854            );
855        }
856    }
857
858    #[test]
859    fn is_ci_bootstrap_task_reads_the_body_marker() {
860        assert!(is_ci_bootstrap_task(&bootstrap_body()));
861        assert!(is_ci_bootstrap_task(&format!("{CI_BOOTSTRAP_MARKER}\n")));
862        assert!(is_ci_bootstrap_task("goal\n<!-- orch-plan: CI-01 -->"));
863        assert!(!is_ci_bootstrap_task(""));
864        assert!(!is_ci_bootstrap_task("goal"));
865    }
866
867    #[test]
868    fn checks_ready_for_ruleset_splits_pending_failed_and_empty() {
869        assert_eq!(checks_ready_for_ruleset(&[], &[]), ChecksReady::Empty);
870        assert_eq!(
871            checks_ready_for_ruleset(&[run("test", "in_progress", None, None)], &[]),
872            ChecksReady::Pending
873        );
874        assert_eq!(
875            checks_ready_for_ruleset(&[run("test", "completed", Some("failure"), None)], &[]),
876            ChecksReady::Failed(vec![FailedCheck {
877                name: "test".to_string(),
878                conclusion: "failure".to_string(),
879            }])
880        );
881        assert_eq!(
882            checks_ready_for_ruleset(
883                &[run("deploy", "completed", Some("action_required"), None)],
884                &[]
885            ),
886            ChecksReady::Pending
887        );
888        assert_eq!(
889            checks_ready_for_ruleset(
890                &[
891                    run("test", "completed", Some("success"), None),
892                    run("deploy", "completed", Some("action_required"), None),
893                ],
894                &[]
895            ),
896            ChecksReady::Pending
897        );
898        assert_eq!(
899            checks_ready_for_ruleset(&[run("test", "completed", Some("cancelled"), None)], &[]),
900            ChecksReady::Failed(vec![FailedCheck {
901                name: "test".to_string(),
902                conclusion: "cancelled".to_string(),
903            }])
904        );
905        assert_eq!(
906            checks_ready_for_ruleset(&[], &[("ci/test".to_string(), "failure".to_string())]),
907            ChecksReady::Failed(vec![FailedCheck {
908                name: "ci/test".to_string(),
909                conclusion: "failure".to_string(),
910            }])
911        );
912        assert_eq!(
913            checks_ready_for_ruleset(&[run("test", "completed", Some("skipped"), None)], &[]),
914            ChecksReady::Empty
915        );
916        assert_eq!(
917            checks_ready_for_ruleset(
918                &[run("orch/review", "completed", Some("success"), None)],
919                &[]
920            ),
921            ChecksReady::Empty
922        );
923        assert_eq!(
924            checks_ready_for_ruleset(
925                &[
926                    run("test", "completed", Some("failure"), None),
927                    run("lint", "in_progress", None, None),
928                ],
929                &[]
930            ),
931            ChecksReady::Failed(vec![FailedCheck {
932                name: "test".to_string(),
933                conclusion: "failure".to_string(),
934            }])
935        );
936        let ChecksReady::Ready(ready) = checks_ready_for_ruleset(
937            &[
938                run("test", "completed", Some("success"), Some(15368)),
939                run("review-gate", "completed", Some("success"), None),
940            ],
941            &[],
942        ) else {
943            panic!("expected ready checks");
944        };
945        assert_eq!(
946            ready,
947            vec![RequiredStatusCheck {
948                context: "test".to_string(),
949                integration_id: Some(15368),
950            }]
951        );
952    }
953
954    #[test]
955    fn required_contexts_cover_matches_by_context_name() {
956        let required = vec![RequiredStatusCheck {
957            context: "cla/check".to_string(),
958            integration_id: None,
959        }];
960        let needed = vec![RequiredStatusCheck {
961            context: "test".to_string(),
962            integration_id: Some(15368),
963        }];
964        assert!(!required_contexts_cover(&required, &needed));
965        let required = vec![
966            RequiredStatusCheck {
967                context: "cla/check".to_string(),
968                integration_id: None,
969            },
970            RequiredStatusCheck {
971                context: "test".to_string(),
972                integration_id: None,
973            },
974        ];
975        assert!(required_contexts_cover(&required, &needed));
976    }
977
978    #[test]
979    fn merge_missing_checks_appends_contexts_the_ruleset_lacks() {
980        let existing = vec![RequiredStatusCheck {
981            context: "cla/check".to_string(),
982            integration_id: None,
983        }];
984        let needed = vec![
985            RequiredStatusCheck {
986                context: "cla/check".to_string(),
987                integration_id: Some(1),
988            },
989            RequiredStatusCheck {
990                context: "test".to_string(),
991                integration_id: Some(15368),
992            },
993        ];
994        assert_eq!(
995            merge_missing_checks(existing, &needed),
996            vec![
997                RequiredStatusCheck {
998                    context: "cla/check".to_string(),
999                    integration_id: None,
1000                },
1001                RequiredStatusCheck {
1002                    context: "test".to_string(),
1003                    integration_id: Some(15368),
1004                },
1005            ]
1006        );
1007    }
1008
1009    #[tokio::test]
1010    async fn hold_after_merge_records_pending_for_a_bootstrap_task() {
1011        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1012        let task = ExternalTask {
1013            id: ExternalId("12".to_string()),
1014            title: "CI-01 Set up CI and tests".to_string(),
1015            body: bootstrap_body(),
1016            state: "open".to_string(),
1017            labels: vec![],
1018            author: String::new(),
1019            created_at: String::new(),
1020            updated_at: String::new(),
1021            url: String::new(),
1022        };
1023        assert!(hold_after_merge(&task, &store, "o/r").await);
1024        assert_eq!(
1025            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1026            Some(KV_PENDING)
1027        );
1028        store
1029            .kv_set(&kv_key("o/r", "12"), KV_APPLIED)
1030            .await
1031            .unwrap();
1032        assert!(!hold_after_merge(&task, &store, "o/r").await);
1033    }
1034
1035    #[tokio::test]
1036    async fn hold_after_merge_skips_a_title_without_a_plan_marker() {
1037        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1038        let task = ExternalTask {
1039            id: ExternalId("12".to_string()),
1040            title: "CI-01 Set up CI and tests".to_string(),
1041            body: "ordinary issue".to_string(),
1042            state: "open".to_string(),
1043            labels: vec![],
1044            author: String::new(),
1045            created_at: String::new(),
1046            updated_at: String::new(),
1047            url: String::new(),
1048        };
1049        assert!(!hold_after_merge(&task, &store, "o/r").await);
1050        assert!(store.kv_get(&kv_key("o/r", "12")).await.unwrap().is_none());
1051    }
1052
1053    #[tokio::test]
1054    async fn apply_pending_dones_when_required_checks_already_cover_default_branch_checks() {
1055        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1056        let server = MockServer::start().await;
1057        mock_classic_required(&server, "test", "deadbeef").await;
1058        mock_runs_and_empty_status(
1059            &server,
1060            "deadbeef",
1061            serde_json::json!([{
1062                "id": 1,
1063                "name": "test",
1064                "status": "completed",
1065                "conclusion": "success",
1066                "app": { "id": 15368 }
1067            }]),
1068        )
1069        .await;
1070        Mock::given(method("POST"))
1071            .and(path("/repos/o/r/rulesets"))
1072            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1073                "id": 9
1074            })))
1075            .expect(0)
1076            .mount(&server)
1077            .await;
1078
1079        let (store, task_manager, id) = pending_store().await;
1080        let gh = GhHttp::for_test(server.uri(), None);
1081        apply_pending_with(&gh, &store, &task_manager, "o/r")
1082            .await
1083            .unwrap();
1084
1085        let stored = store.get(id).await.unwrap();
1086        assert_eq!(stored.status, TaskStatus::Done);
1087        assert_eq!(
1088            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1089            Some(KV_APPLIED)
1090        );
1091    }
1092
1093    #[tokio::test]
1094    async fn apply_pending_creates_a_ruleset_when_an_unrelated_required_check_exists() {
1095        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1096        let server = MockServer::start().await;
1097        mock_classic_required(&server, "cla/check", "deadbeef").await;
1098        mock_runs_and_empty_status(
1099            &server,
1100            "deadbeef",
1101            serde_json::json!([{
1102                "id": 1,
1103                "name": "test",
1104                "status": "completed",
1105                "conclusion": "success",
1106                "app": { "id": 15368 }
1107            }]),
1108        )
1109        .await;
1110        Mock::given(method("POST"))
1111            .and(path("/repos/o/r/rulesets"))
1112            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1113                "id": 9,
1114                "name": REQUIRED_CHECK_RULESET_NAME
1115            })))
1116            .expect(1)
1117            .mount(&server)
1118            .await;
1119
1120        let (store, task_manager, id) = pending_store().await;
1121        let gh = GhHttp::for_test(server.uri(), None);
1122        apply_pending_with(&gh, &store, &task_manager, "o/r")
1123            .await
1124            .unwrap();
1125
1126        let stored = store.get(id).await.unwrap();
1127        assert_eq!(stored.status, TaskStatus::Done);
1128        assert_eq!(
1129            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1130            Some(KV_APPLIED)
1131        );
1132    }
1133
1134    #[tokio::test]
1135    async fn apply_pending_holds_when_an_unrelated_required_check_exists_and_checks_are_pending() {
1136        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1137        let server = MockServer::start().await;
1138        mock_classic_required(&server, "cla/check", "deadbeef").await;
1139        mock_runs_and_empty_status(
1140            &server,
1141            "deadbeef",
1142            serde_json::json!([{
1143                "id": 1,
1144                "name": "test",
1145                "status": "in_progress",
1146                "conclusion": null,
1147                "app": { "id": 15368 }
1148            }]),
1149        )
1150        .await;
1151        Mock::given(method("POST"))
1152            .and(path("/repos/o/r/rulesets"))
1153            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1154                "id": 9
1155            })))
1156            .expect(0)
1157            .mount(&server)
1158            .await;
1159
1160        let (store, task_manager, id) = pending_store().await;
1161        let gh = GhHttp::for_test(server.uri(), None);
1162        apply_pending_with(&gh, &store, &task_manager, "o/r")
1163            .await
1164            .unwrap();
1165
1166        let stored = store.get(id).await.unwrap();
1167        assert_eq!(stored.status, TaskStatus::Approved);
1168        assert_eq!(
1169            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1170            Some(KV_PENDING)
1171        );
1172    }
1173
1174    #[tokio::test]
1175    async fn apply_pending_dones_when_name_taken_and_orch_ruleset_lists_the_checks() {
1176        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1177        let server = MockServer::start().await;
1178        mock_unprotected_main(&server, "deadbeef").await;
1179        mock_runs_and_empty_status(
1180            &server,
1181            "deadbeef",
1182            serde_json::json!([{
1183                "id": 1,
1184                "name": "test",
1185                "status": "completed",
1186                "conclusion": "success",
1187                "app": { "id": 15368 }
1188            }]),
1189        )
1190        .await;
1191        mock_name_taken_ruleset(
1192            &server,
1193            serde_json::json!([{ "context": "test", "integration_id": 15368 }]),
1194        )
1195        .await;
1196        Mock::given(method("PUT"))
1197            .and(path("/repos/o/r/rulesets/9"))
1198            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1199                "id": 9
1200            })))
1201            .expect(0)
1202            .mount(&server)
1203            .await;
1204
1205        let (store, task_manager, id) = pending_store().await;
1206        let gh = GhHttp::for_test(server.uri(), None);
1207        apply_pending_with(&gh, &store, &task_manager, "o/r")
1208            .await
1209            .unwrap();
1210
1211        let stored = store.get(id).await.unwrap();
1212        assert_eq!(stored.status, TaskStatus::Done);
1213        assert_eq!(
1214            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1215            Some(KV_APPLIED)
1216        );
1217    }
1218
1219    #[tokio::test]
1220    async fn apply_pending_updates_the_orch_ruleset_when_name_taken_and_checks_are_missing() {
1221        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1222        let server = MockServer::start().await;
1223        mock_unprotected_main(&server, "deadbeef").await;
1224        mock_runs_and_empty_status(
1225            &server,
1226            "deadbeef",
1227            serde_json::json!([{
1228                "id": 1,
1229                "name": "test",
1230                "status": "completed",
1231                "conclusion": "success",
1232                "app": { "id": 15368 }
1233            }]),
1234        )
1235        .await;
1236        mock_name_taken_ruleset(&server, serde_json::json!([{ "context": "cla/check" }])).await;
1237        Mock::given(method("PUT"))
1238            .and(path("/repos/o/r/rulesets/9"))
1239            .and(body_partial_json(serde_json::json!({
1240                "name": REQUIRED_CHECK_RULESET_NAME,
1241                "rules": [{
1242                    "type": "required_status_checks",
1243                    "parameters": {
1244                        "required_status_checks": [
1245                            { "context": "cla/check" },
1246                            { "context": "test", "integration_id": 15368 }
1247                        ]
1248                    }
1249                }]
1250            })))
1251            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1252                "id": 9,
1253                "name": REQUIRED_CHECK_RULESET_NAME
1254            })))
1255            .expect(1)
1256            .mount(&server)
1257            .await;
1258
1259        let (store, task_manager, id) = pending_store().await;
1260        let gh = GhHttp::for_test(server.uri(), None);
1261        apply_pending_with(&gh, &store, &task_manager, "o/r")
1262            .await
1263            .unwrap();
1264
1265        let stored = store.get(id).await.unwrap();
1266        assert_eq!(stored.status, TaskStatus::Done);
1267        assert_eq!(
1268            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1269            Some(KV_APPLIED)
1270        );
1271    }
1272
1273    #[tokio::test]
1274    async fn apply_pending_creates_a_ruleset_when_default_branch_checks_pass() {
1275        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1276        let server = MockServer::start().await;
1277        Mock::given(method("GET"))
1278            .and(path("/repos/o/r"))
1279            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1280                "default_branch": "main"
1281            })))
1282            .mount(&server)
1283            .await;
1284        Mock::given(method("GET"))
1285            .and(path("/repos/o/r/branches/main/protection"))
1286            .respond_with(
1287                ResponseTemplate::new(404)
1288                    .set_body_string("{\"message\":\"Branch not protected\"}"),
1289            )
1290            .mount(&server)
1291            .await;
1292        Mock::given(method("GET"))
1293            .and(path("/repos/o/r/rules/branches/main"))
1294            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
1295            .mount(&server)
1296            .await;
1297        Mock::given(method("GET"))
1298            .and(path("/repos/o/r/branches/main"))
1299            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1300                "commit": { "sha": "deadbeef" }
1301            })))
1302            .mount(&server)
1303            .await;
1304        Mock::given(method("GET"))
1305            .and(path("/repos/o/r/commits/deadbeef/check-runs"))
1306            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1307                "total_count": 1,
1308                "check_runs": [{
1309                    "id": 1,
1310                    "name": "test",
1311                    "status": "completed",
1312                    "conclusion": "success",
1313                    "app": { "id": 15368 }
1314                }]
1315            })))
1316            .mount(&server)
1317            .await;
1318        Mock::given(method("GET"))
1319            .and(path("/repos/o/r/commits/deadbeef/status"))
1320            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1321                "state": "success",
1322                "total_count": 0,
1323                "statuses": []
1324            })))
1325            .mount(&server)
1326            .await;
1327        Mock::given(method("POST"))
1328            .and(path("/repos/o/r/rulesets"))
1329            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1330                "id": 9,
1331                "name": "orch required checks"
1332            })))
1333            .expect(1)
1334            .mount(&server)
1335            .await;
1336
1337        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1338        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
1339        let task_manager = Arc::new(TaskManager::with_store(
1340            backend,
1341            Arc::clone(&store),
1342            "o/r".to_string(),
1343        ));
1344        let id = store
1345            .upsert_external(&UpsertExternal {
1346                repo: "o/r",
1347                ext_id: "12",
1348                title: "CI-01 Set up CI and tests",
1349                body: &bootstrap_body(),
1350                author: "u",
1351                url: "",
1352                labels: &[],
1353                origin: "github",
1354            })
1355            .await
1356            .unwrap();
1357        store.update_status(id, TaskStatus::Approved).await.unwrap();
1358        store
1359            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
1360            .await
1361            .unwrap();
1362
1363        let gh = GhHttp::for_test(server.uri(), None);
1364        apply_pending_with(&gh, &store, &task_manager, "o/r")
1365            .await
1366            .unwrap();
1367
1368        let stored = store.get(id).await.unwrap();
1369        assert_eq!(stored.status, TaskStatus::Done);
1370        assert_eq!(
1371            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1372            Some(KV_APPLIED)
1373        );
1374    }
1375
1376    #[tokio::test]
1377    async fn apply_pending_comments_and_holds_when_github_refuses_the_ruleset() {
1378        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1379        let server = MockServer::start().await;
1380        Mock::given(method("GET"))
1381            .and(path("/repos/o/r"))
1382            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1383                "default_branch": "main"
1384            })))
1385            .mount(&server)
1386            .await;
1387        Mock::given(method("GET"))
1388            .and(path("/repos/o/r/branches/main/protection"))
1389            .respond_with(
1390                ResponseTemplate::new(404)
1391                    .set_body_string("{\"message\":\"Branch not protected\"}"),
1392            )
1393            .mount(&server)
1394            .await;
1395        Mock::given(method("GET"))
1396            .and(path("/repos/o/r/rules/branches/main"))
1397            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
1398            .mount(&server)
1399            .await;
1400        Mock::given(method("GET"))
1401            .and(path("/repos/o/r/branches/main"))
1402            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1403                "commit": { "sha": "deadbeef" }
1404            })))
1405            .mount(&server)
1406            .await;
1407        Mock::given(method("GET"))
1408            .and(path("/repos/o/r/commits/deadbeef/check-runs"))
1409            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1410                "total_count": 1,
1411                "check_runs": [{
1412                    "id": 1,
1413                    "name": "test",
1414                    "status": "completed",
1415                    "conclusion": "success",
1416                    "app": { "id": 15368 }
1417                }]
1418            })))
1419            .mount(&server)
1420            .await;
1421        Mock::given(method("GET"))
1422            .and(path("/repos/o/r/commits/deadbeef/status"))
1423            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1424                "state": "success",
1425                "total_count": 0,
1426                "statuses": []
1427            })))
1428            .mount(&server)
1429            .await;
1430        Mock::given(method("POST"))
1431            .and(path("/repos/o/r/rulesets"))
1432            .respond_with(ResponseTemplate::new(403).set_body_json(serde_json::json!({
1433                "message": "Upgrade to GitHub Pro or make this repository public to enable this feature."
1434            })))
1435            .mount(&server)
1436            .await;
1437        Mock::given(method("POST"))
1438            .and(path("/repos/o/r/issues/12/comments"))
1439            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1440                "id": 1
1441            })))
1442            .expect(1)
1443            .mount(&server)
1444            .await;
1445
1446        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1447        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
1448        let task_manager = Arc::new(TaskManager::with_store(
1449            backend,
1450            Arc::clone(&store),
1451            "o/r".to_string(),
1452        ));
1453        let id = store
1454            .upsert_external(&UpsertExternal {
1455                repo: "o/r",
1456                ext_id: "12",
1457                title: "CI-01 Set up CI and tests",
1458                body: &bootstrap_body(),
1459                author: "u",
1460                url: "",
1461                labels: &[],
1462                origin: "github",
1463            })
1464            .await
1465            .unwrap();
1466        store.update_status(id, TaskStatus::Approved).await.unwrap();
1467        store
1468            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
1469            .await
1470            .unwrap();
1471
1472        let gh = GhHttp::for_test(server.uri(), None);
1473        apply_pending_with(&gh, &store, &task_manager, "o/r")
1474            .await
1475            .unwrap();
1476
1477        let stored = store.get(id).await.unwrap();
1478        assert_eq!(stored.status, TaskStatus::Approved);
1479        assert_eq!(
1480            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1481            Some(KV_REFUSED)
1482        );
1483    }
1484
1485    #[tokio::test]
1486    async fn apply_pending_holds_when_ruleset_create_is_unrecognized_even_with_401_digits() {
1487        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1488        let server = MockServer::start().await;
1489        mock_unprotected_main(&server, "deadbeef").await;
1490        mock_runs_and_empty_status(
1491            &server,
1492            "deadbeef",
1493            serde_json::json!([{
1494                "id": 1,
1495                "name": "test",
1496                "status": "completed",
1497                "conclusion": "success",
1498                "app": { "id": 15368 }
1499            }]),
1500        )
1501        .await;
1502        Mock::given(method("POST"))
1503            .and(path("/repos/o/r/rulesets"))
1504            .respond_with(ResponseTemplate::new(422).set_body_json(serde_json::json!({
1505                "message": "Ruleset validation failed for rule 401"
1506            })))
1507            .mount(&server)
1508            .await;
1509        mock_issue_comment(&server, 0).await;
1510
1511        let (store, task_manager, id) = pending_store().await;
1512        let gh = GhHttp::for_test(server.uri(), None);
1513        apply_pending_with(&gh, &store, &task_manager, "o/r")
1514            .await
1515            .unwrap();
1516        apply_pending_with(&gh, &store, &task_manager, "o/r")
1517            .await
1518            .unwrap();
1519
1520        let stored = store.get(id).await.unwrap();
1521        assert_eq!(stored.status, TaskStatus::Approved);
1522        assert_eq!(
1523            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1524            Some(KV_PENDING)
1525        );
1526    }
1527
1528    #[tokio::test]
1529    async fn apply_pending_reopens_a_closed_issue_and_creates_the_ruleset() {
1530        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1531        let server = MockServer::start().await;
1532        Mock::given(method("GET"))
1533            .and(path("/repos/o/r"))
1534            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1535                "default_branch": "main"
1536            })))
1537            .mount(&server)
1538            .await;
1539        Mock::given(method("GET"))
1540            .and(path("/repos/o/r/branches/main/protection"))
1541            .respond_with(
1542                ResponseTemplate::new(404)
1543                    .set_body_string("{\"message\":\"Branch not protected\"}"),
1544            )
1545            .mount(&server)
1546            .await;
1547        Mock::given(method("GET"))
1548            .and(path("/repos/o/r/rules/branches/main"))
1549            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([])))
1550            .mount(&server)
1551            .await;
1552        Mock::given(method("GET"))
1553            .and(path("/repos/o/r/issues/12"))
1554            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1555                "number": 12,
1556                "id": 120,
1557                "title": "CI-01 Set up CI and tests",
1558                "body": "",
1559                "state": "closed",
1560                "labels": [],
1561                "user": { "login": "u" },
1562                "created_at": "",
1563                "updated_at": "",
1564                "html_url": ""
1565            })))
1566            .mount(&server)
1567            .await;
1568        Mock::given(method("PATCH"))
1569            .and(path("/repos/o/r/issues/12"))
1570            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1571                "number": 12,
1572                "id": 120,
1573                "title": "CI-01 Set up CI and tests",
1574                "body": "",
1575                "state": "open",
1576                "labels": [],
1577                "user": { "login": "u" },
1578                "created_at": "",
1579                "updated_at": "",
1580                "html_url": ""
1581            })))
1582            .expect(1)
1583            .mount(&server)
1584            .await;
1585        Mock::given(method("GET"))
1586            .and(path("/repos/o/r/branches/main"))
1587            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1588                "commit": { "sha": "deadbeef" }
1589            })))
1590            .mount(&server)
1591            .await;
1592        Mock::given(method("GET"))
1593            .and(path("/repos/o/r/commits/deadbeef/check-runs"))
1594            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1595                "total_count": 1,
1596                "check_runs": [{
1597                    "id": 1,
1598                    "name": "test",
1599                    "status": "completed",
1600                    "conclusion": "success",
1601                    "app": { "id": 15368 }
1602                }]
1603            })))
1604            .mount(&server)
1605            .await;
1606        Mock::given(method("GET"))
1607            .and(path("/repos/o/r/commits/deadbeef/status"))
1608            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
1609                "state": "success",
1610                "total_count": 0,
1611                "statuses": []
1612            })))
1613            .mount(&server)
1614            .await;
1615        Mock::given(method("POST"))
1616            .and(path("/repos/o/r/rulesets"))
1617            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1618                "id": 9,
1619                "name": "orch required checks"
1620            })))
1621            .expect(1)
1622            .mount(&server)
1623            .await;
1624
1625        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1626        let backend: Arc<dyn ExternalBackend> = Arc::new(NoopBackend);
1627        let task_manager = Arc::new(TaskManager::with_store(
1628            backend,
1629            Arc::clone(&store),
1630            "o/r".to_string(),
1631        ));
1632        let id = store
1633            .upsert_external(&UpsertExternal {
1634                repo: "o/r",
1635                ext_id: "12",
1636                title: "CI-01 Set up CI and tests",
1637                body: &bootstrap_body(),
1638                author: "u",
1639                url: "",
1640                labels: &[],
1641                origin: "github",
1642            })
1643            .await
1644            .unwrap();
1645        store.update_status(id, TaskStatus::Done).await.unwrap();
1646        store
1647            .kv_set(&kv_key("o/r", "12"), KV_PENDING)
1648            .await
1649            .unwrap();
1650
1651        let gh = GhHttp::for_test(server.uri(), None);
1652        apply_pending_with(&gh, &store, &task_manager, "o/r")
1653            .await
1654            .unwrap();
1655
1656        let stored = store.get(id).await.unwrap();
1657        assert_eq!(stored.status, TaskStatus::Done);
1658        assert_eq!(
1659            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1660            Some(KV_APPLIED)
1661        );
1662    }
1663
1664    #[tokio::test]
1665    async fn apply_pending_logs_and_holds_while_default_branch_checks_run() {
1666        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1667        let server = MockServer::start().await;
1668        mock_unprotected_main(&server, "deadbeef").await;
1669        mock_runs_and_empty_status(
1670            &server,
1671            "deadbeef",
1672            serde_json::json!([{
1673                "id": 1,
1674                "name": "test",
1675                "status": "in_progress",
1676                "conclusion": null,
1677                "app": { "id": 15368 }
1678            }]),
1679        )
1680        .await;
1681        mock_issue_comment(&server, 0).await;
1682        Mock::given(method("POST"))
1683            .and(path("/repos/o/r/rulesets"))
1684            .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({
1685                "id": 9
1686            })))
1687            .expect(0)
1688            .mount(&server)
1689            .await;
1690
1691        let (store, task_manager, id) = pending_store().await;
1692        let gh = GhHttp::for_test(server.uri(), None);
1693        apply_pending_with(&gh, &store, &task_manager, "o/r")
1694            .await
1695            .unwrap();
1696
1697        let stored = store.get(id).await.unwrap();
1698        assert_eq!(stored.status, TaskStatus::Approved);
1699        assert_eq!(
1700            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1701            Some(KV_PENDING)
1702        );
1703    }
1704
1705    #[tokio::test]
1706    async fn apply_pending_comments_and_routes_when_default_branch_checks_fail() {
1707        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1708        let server = MockServer::start().await;
1709        mock_unprotected_main(&server, "deadbeef").await;
1710        mock_runs_and_empty_status(
1711            &server,
1712            "deadbeef",
1713            serde_json::json!([{
1714                "id": 1,
1715                "name": "test",
1716                "status": "completed",
1717                "conclusion": "failure",
1718                "app": { "id": 15368 }
1719            }]),
1720        )
1721        .await;
1722        mock_issue_comment(&server, 1).await;
1723
1724        let (store, task_manager, id) = pending_store().await;
1725        let gh = GhHttp::for_test(server.uri(), None);
1726        apply_pending_with(&gh, &store, &task_manager, "o/r")
1727            .await
1728            .unwrap();
1729
1730        let stored = store.get(id).await.unwrap();
1731        assert_eq!(stored.status, TaskStatus::Routed);
1732        assert_eq!(stored.ci_merge_failures, 1);
1733        assert!(stored.last_error.contains("test: failure"));
1734        assert_eq!(
1735            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1736            Some(KV_PENDING)
1737        );
1738
1739        apply_pending_with(&gh, &store, &task_manager, "o/r")
1740            .await
1741            .unwrap();
1742        let stored = store.get(id).await.unwrap();
1743        assert_eq!(stored.status, TaskStatus::Routed);
1744        assert_eq!(stored.ci_merge_failures, 1);
1745    }
1746
1747    #[tokio::test]
1748    async fn apply_pending_blocks_after_max_failed_default_branch_heads() {
1749        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1750        let server = MockServer::start().await;
1751        mock_unprotected_main(&server, "deadbeef").await;
1752        mock_runs_and_empty_status(
1753            &server,
1754            "deadbeef",
1755            serde_json::json!([{
1756                "id": 1,
1757                "name": "test",
1758                "status": "completed",
1759                "conclusion": "cancelled",
1760                "app": { "id": 15368 }
1761            }]),
1762        )
1763        .await;
1764        mock_issue_comment(&server, 1).await;
1765
1766        let (store, task_manager, id) = pending_store().await;
1767        store.increment(id, "ci_merge_failures").await.unwrap();
1768        store.increment(id, "ci_merge_failures").await.unwrap();
1769        let gh = GhHttp::for_test(server.uri(), None);
1770        apply_pending_with(&gh, &store, &task_manager, "o/r")
1771            .await
1772            .unwrap();
1773
1774        let stored = store.get(id).await.unwrap();
1775        assert_eq!(stored.status, TaskStatus::Blocked);
1776        assert_eq!(
1777            stored.ci_merge_failures,
1778            i32::try_from(auto_merge::MAX_CI_MERGE_FAILURES).unwrap()
1779        );
1780        assert!(
1781            stored
1782                .block_reason
1783                .as_deref()
1784                .unwrap_or("")
1785                .contains("CI failure limit")
1786        );
1787    }
1788
1789    #[tokio::test]
1790    async fn apply_pending_waits_before_commenting_when_no_check_starts() {
1791        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1792        let server = MockServer::start().await;
1793        mock_unprotected_main(&server, "deadbeef").await;
1794        mock_runs_and_empty_status(&server, "deadbeef", serde_json::json!([])).await;
1795        mock_issue_comment(&server, 0).await;
1796
1797        let (store, task_manager, id) = pending_store().await;
1798        let gh = GhHttp::for_test(server.uri(), None);
1799        apply_pending_with(&gh, &store, &task_manager, "o/r")
1800            .await
1801            .unwrap();
1802
1803        let stored = store.get(id).await.unwrap();
1804        assert_eq!(stored.status, TaskStatus::Approved);
1805        assert!(
1806            store
1807                .kv_get("ci_bootstrap_wait_since:o/r#12#deadbeef")
1808                .await
1809                .unwrap()
1810                .is_some()
1811        );
1812        assert!(
1813            store
1814                .kv_get(&no_checks_comment_key("o/r", "12", "deadbeef"))
1815                .await
1816                .unwrap()
1817                .is_none()
1818        );
1819    }
1820
1821    #[tokio::test]
1822    async fn apply_pending_comments_once_when_no_check_starts_after_the_wait() {
1823        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1824        let server = MockServer::start().await;
1825        mock_unprotected_main(&server, "deadbeef").await;
1826        mock_runs_and_empty_status(&server, "deadbeef", serde_json::json!([])).await;
1827        mock_issue_comment(&server, 1).await;
1828
1829        let (store, task_manager, id) = pending_store().await;
1830        let started = chrono::Utc::now().timestamp() - review_gate::NO_CHECKS_WAIT_SECS - 1;
1831        store
1832            .kv_set(
1833                "ci_bootstrap_wait_since:o/r#12#deadbeef",
1834                &started.to_string(),
1835            )
1836            .await
1837            .unwrap();
1838        let gh = GhHttp::for_test(server.uri(), None);
1839        apply_pending_with(&gh, &store, &task_manager, "o/r")
1840            .await
1841            .unwrap();
1842
1843        let stored = store.get(id).await.unwrap();
1844        assert_eq!(stored.status, TaskStatus::Approved);
1845        assert_eq!(
1846            store
1847                .kv_get(&no_checks_comment_key("o/r", "12", "deadbeef"))
1848                .await
1849                .unwrap()
1850                .as_deref(),
1851            Some("1")
1852        );
1853
1854        apply_pending_with(&gh, &store, &task_manager, "o/r")
1855            .await
1856            .unwrap();
1857        assert_eq!(store.get(id).await.unwrap().status, TaskStatus::Approved);
1858    }
1859
1860    #[tokio::test]
1861    async fn apply_pending_comments_once_when_github_says_the_check_never_ran() {
1862        let _token = EnvVarGuard::set("GH_TOKEN", "test_token");
1863        let server = MockServer::start().await;
1864        mock_unprotected_main(&server, "deadbeef").await;
1865        mock_runs_and_empty_status(
1866            &server,
1867            "deadbeef",
1868            serde_json::json!([{
1869                "id": 1,
1870                "name": "test",
1871                "status": "completed",
1872                "conclusion": "success",
1873                "app": { "id": 15368 }
1874            }]),
1875        )
1876        .await;
1877        Mock::given(method("POST"))
1878            .and(path("/repos/o/r/rulesets"))
1879            .respond_with(ResponseTemplate::new(422).set_body_json(serde_json::json!({
1880                "message": "test was not a valid check"
1881            })))
1882            .mount(&server)
1883            .await;
1884        mock_issue_comment(&server, 1).await;
1885
1886        let (store, task_manager, id) = pending_store().await;
1887        let gh = GhHttp::for_test(server.uri(), None);
1888        apply_pending_with(&gh, &store, &task_manager, "o/r")
1889            .await
1890            .unwrap();
1891        apply_pending_with(&gh, &store, &task_manager, "o/r")
1892            .await
1893            .unwrap();
1894
1895        let stored = store.get(id).await.unwrap();
1896        assert_eq!(stored.status, TaskStatus::Approved);
1897        assert_eq!(
1898            store.kv_get(&kv_key("o/r", "12")).await.unwrap().as_deref(),
1899            Some(KV_PENDING)
1900        );
1901        assert_eq!(
1902            store
1903                .kv_get(&never_run_comment_key("o/r", "12"))
1904                .await
1905                .unwrap()
1906                .as_deref(),
1907            Some("1")
1908        );
1909    }
1910}