git_branchless_test/
lib.rs

1//! Run a user-provided command on each of a set of provided commits. This is
2//! useful to run checks on all commits in the current stack with caching,
3//! parallelization, out-of-tree execution, etc.
4
5#![warn(missing_docs)]
6#![warn(
7    clippy::all,
8    clippy::as_conversions,
9    clippy::clone_on_ref_ptr,
10    clippy::dbg_macro
11)]
12#![allow(clippy::too_many_arguments, clippy::blocks_in_if_conditions)]
13
14mod worker;
15
16use std::collections::{HashMap, HashSet};
17use std::fmt::Write as _;
18use std::fs::File;
19use std::path::{Path, PathBuf};
20use std::process::{Command, Stdio};
21use std::sync::Arc;
22use std::time::SystemTime;
23
24use bstr::ByteSlice;
25use clap::ValueEnum;
26use crossbeam::channel::{Receiver, RecvError};
27use cursive::theme::{BaseColor, Effect, Style};
28use cursive::utils::markup::StyledString;
29
30use eyre::WrapErr;
31use fslock::LockFile;
32use git_branchless_invoke::CommandContext;
33use indexmap::IndexMap;
34use itertools::Itertools;
35use lazy_static::lazy_static;
36use lib::core::check_out::CheckOutCommitOptions;
37use lib::core::config::{
38    get_hint_enabled, get_hint_string, get_restack_preserve_timestamps,
39    print_hint_suppression_notice, Hint,
40};
41use lib::core::dag::{sorted_commit_set, CommitSet, Dag};
42use lib::core::effects::{icons, Effects, OperationIcon, OperationType};
43use lib::core::eventlog::{
44    EventLogDb, EventReplayer, EventTransactionId, BRANCHLESS_TRANSACTION_ID_ENV_VAR,
45};
46use lib::core::formatting::{Glyphs, Pluralize, StyledStringBuilder};
47use lib::core::repo_ext::RepoExt;
48use lib::core::rewrite::{
49    execute_rebase_plan, BuildRebasePlanOptions, ExecuteRebasePlanOptions, ExecuteRebasePlanResult,
50    RebaseCommand, RebasePlan, RebasePlanBuilder, RebasePlanPermissions, RepoResource,
51};
52use lib::git::{
53    get_latest_test_command_path, get_test_locks_dir, get_test_tree_dir, get_test_worktrees_dir,
54    make_test_command_slug, Commit, ConfigRead, GitRunInfo, GitRunResult, MaybeZeroOid, NonZeroOid,
55    Repo, SerializedNonZeroOid, SerializedTestResult, TestCommand, WorkingCopyChangesType,
56    TEST_ABORT_EXIT_CODE, TEST_INDETERMINATE_EXIT_CODE, TEST_SUCCESS_EXIT_CODE,
57};
58use lib::try_exit_code;
59use lib::util::{get_sh, ExitCode, EyreExitOr};
60use rayon::ThreadPoolBuilder;
61use scm_bisect::basic::{BasicSourceControlGraph, BasicStrategy, BasicStrategyKind};
62use scm_bisect::search;
63use tempfile::TempDir;
64use thiserror::Error;
65use tracing::{debug, info, instrument, warn};
66
67use git_branchless_opts::{
68    MoveOptions, ResolveRevsetOptions, Revset, TestArgs, TestExecutionStrategy, TestSearchStrategy,
69    TestSubcommand,
70};
71use git_branchless_revset::resolve_commits;
72
73use crate::worker::{worker, JobResult, WorkQueue, WorkerId};
74
75lazy_static! {
76    static ref STYLE_SUCCESS: Style =
77        Style::merge(&[BaseColor::Green.light().into(), Effect::Bold.into()]);
78    static ref STYLE_FAILURE: Style =
79        Style::merge(&[BaseColor::Red.light().into(), Effect::Bold.into()]);
80    static ref STYLE_SKIPPED: Style =
81        Style::merge(&[BaseColor::Yellow.light().into(), Effect::Bold.into()]);
82}
83
84/// How verbose of output to produce.
85#[derive(Clone, Copy, Debug, Ord, PartialOrd, Eq, PartialEq)]
86pub enum Verbosity {
87    /// Do not include test output at all.
88    None,
89
90    /// Include truncated test output.
91    PartialOutput,
92
93    /// Include the full test output.
94    FullOutput,
95}
96
97impl From<u8> for Verbosity {
98    fn from(value: u8) -> Self {
99        match value {
100            0 => Self::None,
101            1 => Self::PartialOutput,
102            _ => Self::FullOutput,
103        }
104    }
105}
106
107/// The options for testing before they've assumed default values or been
108/// validated.
109#[derive(Debug)]
110pub struct RawTestOptions {
111    /// The command to execute, if any.
112    pub exec: Option<String>,
113
114    /// The command alias to execute, if any.
115    pub command: Option<String>,
116
117    /// Whether or not to execute as a "dry-run", i.e. don't rewrite any commits
118    /// if `true`.
119    pub dry_run: bool,
120
121    /// The execution strategy to use.
122    pub strategy: Option<TestExecutionStrategy>,
123
124    /// Search for the first commit that fails the test command, rather than
125    /// running on all commits.
126    pub search: Option<TestSearchStrategy>,
127
128    /// Shorthand for the binary search strategy.
129    pub bisect: bool,
130
131    /// Don't read or write to the cache when executing the test commands.
132    pub no_cache: bool,
133
134    /// Whether to run interactively.
135    pub interactive: bool,
136
137    /// The number of jobs to run in parallel.
138    pub jobs: Option<usize>,
139
140    /// The requested verbosity of the test output.
141    pub verbosity: Verbosity,
142
143    /// Whether to amend commits with the changes produced by the executed
144    /// command.
145    pub apply_fixes: bool,
146}
147
148fn resolve_test_command_alias(
149    effects: &Effects,
150    repo: &Repo,
151    alias: Option<&str>,
152) -> EyreExitOr<String> {
153    let config = repo.get_readonly_config()?;
154    let config_key = format!("branchless.test.alias.{}", alias.unwrap_or("default"));
155    let config_value: Option<String> = config.get(config_key).unwrap_or_default();
156    if let Some(command) = config_value {
157        return Ok(Ok(command));
158    }
159
160    match alias {
161        Some(alias) => {
162            writeln!(
163                effects.get_output_stream(),
164                "\
165The test command alias {alias:?} was not defined.
166
167To create it, run: git config branchless.test.alias.{alias} <command>
168Or use the -x/--exec flag instead to run a test command without first creating an alias."
169            )?;
170        }
171        None => {
172            writeln!(
173                effects.get_output_stream(),
174                "\
175Could not determine test command to run. No test command was provided with -c/--command or
176-x/--exec, and the configuration value 'branchless.test.alias.default' was not set.
177
178To configure a default test command, run: git config branchless.test.alias.default <command>
179To run a specific test command, run: git test run -x <command>
180To run a specific command alias, run: git test run -c <alias>",
181            )?;
182        }
183    }
184
185    let aliases = config.list("branchless.test.alias.*")?;
186    if !aliases.is_empty() {
187        writeln!(
188            effects.get_output_stream(),
189            "\nThese are the currently-configured command aliases:"
190        )?;
191        for (name, command) in aliases {
192            writeln!(
193                effects.get_output_stream(),
194                "{} {name} = {command:?}",
195                effects.get_glyphs().bullet_point,
196            )?;
197        }
198    }
199
200    Ok(Err(ExitCode(1)))
201}
202
203/// The values from a `RawTestOptions` but with defaults provided. See
204/// [`RawTestOptions`] for details on these options.
205#[allow(missing_docs)]
206#[derive(Debug)]
207pub struct ResolvedTestOptions {
208    pub command: TestCommand,
209    pub execution_strategy: TestExecutionStrategy,
210    pub search_strategy: Option<TestSearchStrategy>,
211    pub is_dry_run: bool,
212    pub use_cache: bool,
213    pub is_interactive: bool,
214    pub num_jobs: usize,
215    pub verbosity: Verbosity,
216    pub fix_options: Option<(ExecuteRebasePlanOptions, RebasePlanPermissions)>,
217}
218
219impl ResolvedTestOptions {
220    /// Resolve any missing test options with their defaults from the environment.
221    pub fn resolve(
222        now: SystemTime,
223        effects: &Effects,
224        dag: &Dag,
225        repo: &Repo,
226        event_tx_id: EventTransactionId,
227        commits: &CommitSet,
228        move_options: Option<&MoveOptions>,
229        options: &RawTestOptions,
230    ) -> EyreExitOr<Self> {
231        let config = repo.get_readonly_config()?;
232        let RawTestOptions {
233            exec: command,
234            command: command_alias,
235            dry_run,
236            strategy,
237            search,
238            bisect,
239            no_cache,
240            interactive,
241            jobs,
242            verbosity,
243            apply_fixes,
244        } = options;
245        let resolved_command = match (command, command_alias) {
246            (Some(command), None) => command.to_owned(),
247            (None, None) => match (interactive, std::env::var("SHELL")) {
248                (true, Ok(shell)) => shell,
249                _ => match resolve_test_command_alias(effects, repo, None)? {
250                    Ok(command) => command,
251                    Err(exit_code) => {
252                        return Ok(Err(exit_code));
253                    }
254                },
255            },
256            (None, Some(command_alias)) => {
257                match resolve_test_command_alias(effects, repo, Some(command_alias))? {
258                    Ok(command) => command,
259                    Err(exit_code) => {
260                        return Ok(Err(exit_code));
261                    }
262                }
263            }
264            (Some(command), Some(command_alias)) => unreachable!(
265                "Command ({:?}) and command alias ({:?}) are conflicting options",
266                command, command_alias
267            ),
268        };
269        let configured_execution_strategy = match strategy {
270            Some(strategy) => *strategy,
271            None => {
272                let strategy_config_key = "branchless.test.strategy";
273                let strategy: Option<String> = config.get(strategy_config_key)?;
274                match strategy {
275                    None => TestExecutionStrategy::WorkingCopy,
276                    Some(strategy) => {
277                        match TestExecutionStrategy::from_str(&strategy, true) {
278                            Ok(strategy) => strategy,
279                            Err(_) => {
280                                writeln!(effects.get_output_stream(), "Invalid value for config value {strategy_config_key}: {strategy}")?;
281                                writeln!(
282                                    effects.get_output_stream(),
283                                    "Expected one of: {}",
284                                    TestExecutionStrategy::value_variants()
285                                        .iter()
286                                        .filter_map(|variant| variant.to_possible_value())
287                                        .map(|value| value.get_name().to_owned())
288                                        .join(", ")
289                                )?;
290                                return Ok(Err(ExitCode(1)));
291                            }
292                        }
293                    }
294                }
295            }
296        };
297
298        let jobs_config_key = "branchless.test.jobs";
299        let configured_jobs: Option<i32> = config.get(jobs_config_key)?;
300        let configured_jobs = match configured_jobs {
301            None => None,
302            Some(configured_jobs) => match usize::try_from(configured_jobs) {
303                Ok(configured_jobs) => Some(configured_jobs),
304                Err(err) => {
305                    writeln!(
306                        effects.get_output_stream(),
307                        "Invalid value for config value for {jobs_config_key} ({configured_jobs}): {err}"
308                    )?;
309                    return Ok(Err(ExitCode(1)));
310                }
311            },
312        };
313        let (resolved_num_jobs, resolved_execution_strategy, resolved_interactive) = match jobs {
314            None => match (strategy, *interactive) {
315                (Some(TestExecutionStrategy::WorkingCopy), interactive) => {
316                    (1, TestExecutionStrategy::WorkingCopy, interactive)
317                }
318                (Some(TestExecutionStrategy::Worktree), true) => {
319                    (1, TestExecutionStrategy::Worktree, true)
320                }
321                (Some(TestExecutionStrategy::Worktree), false) => (
322                    configured_jobs.unwrap_or(1),
323                    TestExecutionStrategy::Worktree,
324                    false,
325                ),
326                (None, true) => (1, configured_execution_strategy, true),
327                (None, false) => (
328                    configured_jobs.unwrap_or(1),
329                    configured_execution_strategy,
330                    false,
331                ),
332            },
333            Some(1) => (1, configured_execution_strategy, *interactive),
334            Some(jobs) => {
335                if *interactive {
336                    writeln!(
337                        effects.get_output_stream(),
338                        "\
339The --jobs option cannot be used with the --interactive option."
340                    )?;
341                    return Ok(Err(ExitCode(1)));
342                }
343                // NB: match on the strategy passed on the command-line here, not the resolved strategy.
344                match strategy {
345                    None | Some(TestExecutionStrategy::Worktree) => {
346                        (*jobs, TestExecutionStrategy::Worktree, false)
347                    }
348                    Some(TestExecutionStrategy::WorkingCopy) => {
349                        writeln!(
350                            effects.get_output_stream(),
351                            "\
352The --jobs option can only be used with --strategy worktree, but --strategy working-copy was provided instead."
353                        )?;
354                        return Ok(Err(ExitCode(1)));
355                    }
356                }
357            }
358        };
359
360        if resolved_interactive != *interactive {
361            writeln!(effects.get_output_stream(),
362            "\
363BUG: Expected resolved_interactive ({resolved_interactive:?}) to match interactive ({interactive:?}). If it doesn't match, then multiple interactive jobs might inadvertently be launched in parallel."
364            )?;
365            return Ok(Err(ExitCode(1)));
366        }
367
368        let resolved_num_jobs = if resolved_num_jobs == 0 {
369            num_cpus::get_physical()
370        } else {
371            resolved_num_jobs
372        };
373        assert!(resolved_num_jobs > 0);
374
375        let fix_options = if *apply_fixes {
376            let move_options = match move_options {
377                Some(move_options) => move_options,
378                None => {
379                    writeln!(effects.get_output_stream(), "BUG: fixes were requested to be applied, but no `BuildRebasePlanOptions` were provided.")?;
380                    return Ok(Err(ExitCode(1)));
381                }
382            };
383            let MoveOptions {
384                force_rewrite_public_commits,
385                force_in_memory: _,
386                force_on_disk,
387                detect_duplicate_commits_via_patch_id,
388                resolve_merge_conflicts,
389                dump_rebase_constraints,
390                dump_rebase_plan,
391            } = move_options;
392
393            let force_in_memory = true;
394            if *force_on_disk {
395                writeln!(
396                    effects.get_output_stream(),
397                    "The --on-disk option cannot be provided for fixes. Use the --in-memory option instead."
398                )?;
399                return Ok(Err(ExitCode(1)));
400            }
401
402            let build_options = BuildRebasePlanOptions {
403                force_rewrite_public_commits: *force_rewrite_public_commits,
404                dump_rebase_constraints: *dump_rebase_constraints,
405                dump_rebase_plan: *dump_rebase_plan,
406                detect_duplicate_commits_via_patch_id: *detect_duplicate_commits_via_patch_id,
407            };
408            let execute_options = ExecuteRebasePlanOptions {
409                now,
410                event_tx_id,
411                preserve_timestamps: get_restack_preserve_timestamps(repo)?,
412                force_in_memory,
413                force_on_disk: *force_on_disk,
414                resolve_merge_conflicts: *resolve_merge_conflicts,
415                check_out_commit_options: CheckOutCommitOptions {
416                    render_smartlog: false,
417                    ..Default::default()
418                },
419            };
420            let permissions =
421                match RebasePlanPermissions::verify_rewrite_set(dag, build_options, commits)? {
422                    Ok(permissions) => permissions,
423                    Err(err) => {
424                        err.describe(effects, repo, dag)?;
425                        return Ok(Err(ExitCode(1)));
426                    }
427                };
428            Some((execute_options, permissions))
429        } else {
430            None
431        };
432
433        let resolved_search_strategy = if *bisect {
434            Some(TestSearchStrategy::Binary)
435        } else {
436            *search
437        };
438
439        let resolved_test_options = ResolvedTestOptions {
440            command: TestCommand::String(resolved_command),
441            execution_strategy: resolved_execution_strategy,
442            search_strategy: resolved_search_strategy,
443            use_cache: !no_cache,
444            is_dry_run: *dry_run,
445            is_interactive: resolved_interactive,
446            num_jobs: resolved_num_jobs,
447            verbosity: *verbosity,
448            fix_options,
449        };
450        debug!(?resolved_test_options, "Resolved test options");
451        Ok(Ok(resolved_test_options))
452    }
453
454    fn make_command_slug(&self) -> String {
455        make_test_command_slug(self.command.to_string())
456    }
457}
458
459/// `test` command.
460#[instrument]
461pub fn command_main(ctx: CommandContext, args: TestArgs) -> EyreExitOr<()> {
462    let CommandContext {
463        effects,
464        git_run_info,
465    } = ctx;
466    let TestArgs { subcommand } = args;
467    match subcommand {
468        TestSubcommand::Clean {
469            revset,
470            resolve_revset_options,
471        } => subcommand_clean(&effects, revset, &resolve_revset_options),
472
473        TestSubcommand::Run {
474            exec: command,
475            command: command_alias,
476            revset,
477            resolve_revset_options,
478            verbosity,
479            strategy,
480            search,
481            bisect,
482            no_cache,
483            interactive,
484            jobs,
485        } => subcommand_run(
486            &effects,
487            &git_run_info,
488            &RawTestOptions {
489                exec: command,
490                command: command_alias,
491                dry_run: false,
492                strategy,
493                search,
494                bisect,
495                no_cache,
496                interactive,
497                jobs,
498                verbosity: Verbosity::from(verbosity),
499                apply_fixes: false,
500            },
501            revset,
502            &resolve_revset_options,
503            None,
504        ),
505
506        TestSubcommand::Show {
507            exec: command,
508            command: command_alias,
509            revset,
510            resolve_revset_options,
511            verbosity,
512        } => subcommand_show(
513            &effects,
514            &RawTestOptions {
515                exec: command,
516                command: command_alias,
517                dry_run: false,
518                strategy: None,
519                search: None,
520                bisect: false,
521                no_cache: false,
522                interactive: false,
523                jobs: None,
524                verbosity: Verbosity::from(verbosity),
525                apply_fixes: false,
526            },
527            revset,
528            &resolve_revset_options,
529        ),
530
531        TestSubcommand::Fix {
532            exec: command,
533            command: command_alias,
534            dry_run,
535            revset,
536            resolve_revset_options,
537            verbosity,
538            strategy,
539            no_cache,
540            jobs,
541            move_options,
542        } => subcommand_run(
543            &effects,
544            &git_run_info,
545            &RawTestOptions {
546                exec: command,
547                command: command_alias,
548                dry_run,
549                strategy,
550                search: None,
551                bisect: false,
552                no_cache,
553                interactive: false,
554                jobs,
555                verbosity: Verbosity::from(verbosity),
556                apply_fixes: true,
557            },
558            revset,
559            &resolve_revset_options,
560            Some(&move_options),
561        ),
562    }
563}
564
565/// Run the command provided in `options` on each of the commits in `revset`.
566#[instrument]
567fn subcommand_run(
568    effects: &Effects,
569    git_run_info: &GitRunInfo,
570    options: &RawTestOptions,
571    revset: Revset,
572    resolve_revset_options: &ResolveRevsetOptions,
573    move_options: Option<&MoveOptions>,
574) -> EyreExitOr<()> {
575    let now = SystemTime::now();
576    let repo = Repo::from_current_dir()?;
577    let conn = repo.get_db_conn()?;
578    let event_log_db = EventLogDb::new(&conn)?;
579    let event_tx_id = event_log_db.make_transaction_id(now, "test run")?;
580    let event_replayer = EventReplayer::from_event_log_db(effects, &repo, &event_log_db)?;
581    let event_cursor = event_replayer.make_default_cursor();
582    let references_snapshot = repo.get_references_snapshot()?;
583    let mut dag = Dag::open_and_sync(
584        effects,
585        &repo,
586        &event_replayer,
587        event_cursor,
588        &references_snapshot,
589    )?;
590
591    let commit_set = match resolve_commits(
592        effects,
593        &repo,
594        &mut dag,
595        &[revset.clone()],
596        resolve_revset_options,
597    ) {
598        Ok(mut commit_sets) => commit_sets.pop().unwrap(),
599        Err(err) => {
600            err.describe(effects)?;
601            return Ok(Err(ExitCode(1)));
602        }
603    };
604
605    let options = try_exit_code!(ResolvedTestOptions::resolve(
606        now,
607        effects,
608        &dag,
609        &repo,
610        event_tx_id,
611        &commit_set,
612        move_options,
613        options,
614    )?);
615
616    let commits = sorted_commit_set(&repo, &dag, &commit_set)?;
617    let test_results = try_exit_code!(run_tests(
618        now,
619        effects,
620        git_run_info,
621        &dag,
622        &repo,
623        &event_log_db,
624        &revset,
625        &commits,
626        &options,
627    )?);
628
629    try_exit_code!(print_summary(
630        effects,
631        &dag,
632        &repo,
633        &revset,
634        &options.command,
635        &test_results,
636        options.search_strategy.is_some(),
637        options.fix_options.is_some(),
638        &options.verbosity,
639    )?);
640
641    if let Some((execute_options, permissions)) = &options.fix_options {
642        try_exit_code!(apply_fixes(
643            effects,
644            git_run_info,
645            &mut dag,
646            &repo,
647            &event_log_db,
648            execute_options,
649            permissions.clone(),
650            options.is_dry_run,
651            &options.command,
652            &test_results,
653        )?);
654    }
655
656    Ok(Ok(()))
657}
658
659#[must_use]
660#[derive(Debug)]
661struct AbortTrap {
662    is_active: bool,
663}
664
665/// Ensure that no commit operation is currently underway (such as a merge or
666/// rebase), and start a rebase.  In the event that the test invocation is
667/// interrupted, this will prevent the user from starting another commit
668/// operation without first running `git rebase --abort` to get back to their
669/// original commit.
670#[instrument]
671fn set_abort_trap(
672    now: SystemTime,
673    effects: &Effects,
674    git_run_info: &GitRunInfo,
675    repo: &Repo,
676    event_log_db: &EventLogDb,
677    event_tx_id: EventTransactionId,
678    strategy: TestExecutionStrategy,
679) -> EyreExitOr<AbortTrap> {
680    match strategy {
681        TestExecutionStrategy::Worktree => return Ok(Ok(AbortTrap { is_active: false })),
682        TestExecutionStrategy::WorkingCopy => {}
683    }
684
685    if let Some(operation_type) = repo.get_current_operation_type() {
686        writeln!(
687            effects.get_output_stream(),
688            "A {operation_type} operation is already in progress."
689        )?;
690        writeln!(
691            effects.get_output_stream(),
692            "Run git {operation_type} --continue or git {operation_type} --abort to resolve it and proceed."
693        )?;
694        return Ok(Err(ExitCode(1)));
695    }
696
697    let head_info = repo.get_head_info()?;
698    let head_oid = match head_info.oid {
699        Some(head_oid) => head_oid,
700        None => {
701            writeln!(
702                effects.get_output_stream(),
703                "No commit is currently checked out; cannot start on-disk rebase."
704            )?;
705            writeln!(
706                effects.get_output_stream(),
707                "Check out a commit and try again."
708            )?;
709            return Ok(Err(ExitCode(1)));
710        }
711    };
712
713    let rebase_plan = RebasePlan {
714        first_dest_oid: head_oid,
715        commands: vec![RebaseCommand::Break],
716    };
717    match execute_rebase_plan(
718        effects,
719        git_run_info,
720        repo,
721        event_log_db,
722        &rebase_plan,
723        &ExecuteRebasePlanOptions {
724            now,
725            event_tx_id,
726            preserve_timestamps: true,
727            force_in_memory: false,
728            force_on_disk: true,
729            resolve_merge_conflicts: false,
730            check_out_commit_options: CheckOutCommitOptions {
731                render_smartlog: false,
732                ..Default::default()
733            },
734        },
735    )? {
736        ExecuteRebasePlanResult::Succeeded { rewritten_oids: _ } => {
737            // Do nothing.
738        }
739        ExecuteRebasePlanResult::DeclinedToMerge { failed_merge_info } => {
740            writeln!(
741                effects.get_output_stream(),
742                "BUG: Encountered unexpected merge failure: {failed_merge_info:?}"
743            )?;
744            return Ok(Err(ExitCode(1)));
745        }
746        ExecuteRebasePlanResult::Failed { exit_code } => {
747            return Ok(Err(exit_code));
748        }
749    }
750
751    Ok(Ok(AbortTrap { is_active: true }))
752}
753
754#[instrument]
755fn clear_abort_trap(
756    effects: &Effects,
757    git_run_info: &GitRunInfo,
758    event_tx_id: EventTransactionId,
759    abort_trap: AbortTrap,
760) -> EyreExitOr<()> {
761    let AbortTrap { is_active } = abort_trap;
762    if is_active {
763        try_exit_code!(git_run_info.run(effects, Some(event_tx_id), &["rebase", "--abort"])?);
764    }
765    Ok(Ok(()))
766}
767
768/// The result of running a test.
769#[derive(Debug)]
770pub struct TestOutput {
771    /// Only used for `--no-cache` invocations, to ensure that we don't clobber
772    /// an existing cached entry.
773    pub temp_dir: Option<TempDir>,
774
775    /// The path to the file containing the serialized test result information
776    /// (such as the exit code).
777    pub result_path: PathBuf,
778
779    /// The path to the file containing the stdout of the test command.
780    pub stdout_path: PathBuf,
781
782    /// The path to the file containing the stderr of the test command.
783    pub stderr_path: PathBuf,
784
785    /// The resulting status of the test.
786    pub test_status: TestStatus,
787}
788
789/// The possible results of attempting to run a test.
790#[derive(Clone, Debug)]
791pub enum TestStatus {
792    /// Attempting to set up the working directory for the repository failed.
793    CheckoutFailed,
794
795    /// Invoking the test command failed.
796    SpawnTestFailed(String),
797
798    /// The test command was invoked successfully, but was terminated by a signal, rather than
799    /// returning an exit code normally.
800    TerminatedBySignal,
801
802    /// It appears that some other process is already running the test for a commit with the given
803    /// tree. (If that process crashed, then the test may need to be re-run.)
804    AlreadyInProgress,
805
806    /// Attempting to read cached data failed.
807    ReadCacheFailed(String),
808
809    /// The test command indicated that the commit should be skipped for testing.
810    Indeterminate {
811        /// The exit code of the command.
812        exit_code: i32,
813    },
814
815    /// The test command indicated that the process should be aborted entirely.
816    Abort {
817        /// The exit code of the command.
818        exit_code: i32,
819    },
820
821    /// The test failed and returned the provided (non-zero) exit code.
822    Failed {
823        /// Whether or not the result was cached (indicating that we didn't
824        /// actually re-run the test).
825        cached: bool,
826
827        /// The exit code of the process.
828        exit_code: i32,
829
830        /// Whether the test was run interactively (the user executed the
831        /// command via `--interactive`).
832        interactive: bool,
833    },
834
835    /// The test passed and returned a successful exit code.
836    Passed {
837        /// Whether or not the result was cached (indicating that we didn't
838        /// actually re-run the test).
839        cached: bool,
840
841        /// Information about the working copy state after running the test command.
842        fix_info: FixInfo,
843
844        /// Whether the test was run interactively (the user executed the
845        /// command via `--interactive`).
846        interactive: bool,
847    },
848}
849
850/// Information about the working copy state after running the test command.
851#[derive(Clone, Debug)]
852pub struct FixInfo {
853    /// The resulting commit which was checked out as `HEAD`. This is usually
854    /// the same commit as was being tested, but the test command could amend or
855    /// switch to a different commit.
856    pub head_commit_oid: Option<NonZeroOid>,
857
858    /// The resulting contents of the working copy after the command was
859    /// executed (if taking a working copy snapshot succeeded and there were no
860    /// merge conflicts, etc.).
861    pub snapshot_tree_oid: Option<NonZeroOid>,
862}
863
864impl TestStatus {
865    #[instrument]
866    fn get_icon(&self) -> &'static str {
867        match self {
868            TestStatus::CheckoutFailed
869            | TestStatus::SpawnTestFailed(_)
870            | TestStatus::AlreadyInProgress
871            | TestStatus::ReadCacheFailed(_)
872            | TestStatus::TerminatedBySignal
873            | TestStatus::Indeterminate { .. } => icons::EXCLAMATION,
874            TestStatus::Failed { .. } | TestStatus::Abort { .. } => icons::CROSS,
875            TestStatus::Passed { .. } => icons::CHECKMARK,
876        }
877    }
878
879    #[instrument]
880    fn get_style(&self) -> Style {
881        match self {
882            TestStatus::CheckoutFailed
883            | TestStatus::SpawnTestFailed(_)
884            | TestStatus::AlreadyInProgress
885            | TestStatus::ReadCacheFailed(_)
886            | TestStatus::TerminatedBySignal
887            | TestStatus::Indeterminate { .. } => *STYLE_SKIPPED,
888            TestStatus::Failed { .. } | TestStatus::Abort { .. } => *STYLE_FAILURE,
889            TestStatus::Passed { .. } => *STYLE_SUCCESS,
890        }
891    }
892
893    /// Produce a friendly description of the test status.
894    #[instrument]
895    pub fn describe(
896        &self,
897        glyphs: &Glyphs,
898        commit: &Commit,
899        apply_fixes: bool,
900    ) -> eyre::Result<StyledString> {
901        let description = match self {
902            TestStatus::CheckoutFailed => StyledStringBuilder::new()
903                .append_styled("Failed to check out: ", self.get_style())
904                .append(commit.friendly_describe(glyphs)?)
905                .build(),
906
907            TestStatus::SpawnTestFailed(err) => StyledStringBuilder::new()
908                .append_styled(
909                    format!("Failed to spawn command: {err}: "),
910                    self.get_style(),
911                )
912                .append(commit.friendly_describe(glyphs)?)
913                .build(),
914
915            TestStatus::TerminatedBySignal => StyledStringBuilder::new()
916                .append_styled("Command terminated by signal: ", self.get_style())
917                .append(commit.friendly_describe(glyphs)?)
918                .build(),
919
920            TestStatus::AlreadyInProgress => StyledStringBuilder::new()
921                .append_styled("Command already in progress? ", self.get_style())
922                .append(commit.friendly_describe(glyphs)?)
923                .build(),
924
925            TestStatus::ReadCacheFailed(_) => StyledStringBuilder::new()
926                .append_styled("Could not read cached command result: ", self.get_style())
927                .append(commit.friendly_describe(glyphs)?)
928                .build(),
929
930            TestStatus::Indeterminate { exit_code } => StyledStringBuilder::new()
931                .append_styled(
932                    format!("Exit code indicated to skip this commit (exit code {exit_code}): "),
933                    self.get_style(),
934                )
935                .append(commit.friendly_describe(glyphs)?)
936                .build(),
937
938            TestStatus::Abort { exit_code } => StyledStringBuilder::new()
939                .append_styled(
940                    format!("Exit code indicated to abort command (exit code {exit_code}): "),
941                    self.get_style(),
942                )
943                .append(commit.friendly_describe(glyphs)?)
944                .build(),
945
946            TestStatus::Failed {
947                cached,
948                interactive,
949                exit_code,
950            } => {
951                let mut descriptors = Vec::new();
952                if *cached {
953                    descriptors.push("cached".to_string());
954                }
955                descriptors.push(format!("exit code {exit_code}"));
956                if *interactive {
957                    descriptors.push("interactive".to_string());
958                }
959                let descriptors = descriptors.join(", ");
960                StyledStringBuilder::new()
961                    .append_styled(format!("Failed ({descriptors}): "), self.get_style())
962                    .append(commit.friendly_describe(glyphs)?)
963                    .build()
964            }
965
966            TestStatus::Passed {
967                cached,
968                interactive,
969                fix_info:
970                    FixInfo {
971                        head_commit_oid: _,
972                        snapshot_tree_oid,
973                    },
974            } => {
975                let mut descriptors = Vec::new();
976                if *cached {
977                    descriptors.push("cached".to_string());
978                }
979                match (snapshot_tree_oid, commit.get_tree_oid()) {
980                    (Some(snapshot_tree_oid), MaybeZeroOid::NonZero(original_tree_oid)) => {
981                        if *snapshot_tree_oid != original_tree_oid {
982                            descriptors.push(if apply_fixes {
983                                "fixed".to_string()
984                            } else {
985                                "fixable".to_string()
986                            });
987                        }
988                    }
989                    (None, _) | (_, MaybeZeroOid::Zero) => {}
990                }
991                if *interactive {
992                    descriptors.push("interactive".to_string());
993                }
994                let descriptors = if descriptors.is_empty() {
995                    "".to_string()
996                } else {
997                    format!(" ({})", descriptors.join(", "))
998                };
999                StyledStringBuilder::new()
1000                    .append_styled(format!("Passed{descriptors}: "), self.get_style())
1001                    .append(commit.friendly_describe(glyphs)?)
1002                    .build()
1003            }
1004        };
1005        Ok(description)
1006    }
1007}
1008
1009/// An error produced when testing is aborted due to a certain commit.
1010#[derive(Debug)]
1011pub struct TestingAbortedError {
1012    /// The commit which aborted testing.
1013    pub commit_oid: NonZeroOid,
1014
1015    /// The exit code of the test command when run on that commit.
1016    pub exit_code: i32,
1017}
1018
1019impl TestOutput {
1020    #[instrument]
1021    fn describe(
1022        &self,
1023        effects: &Effects,
1024        commit: &Commit,
1025        apply_fixes: bool,
1026        verbosity: Verbosity,
1027    ) -> eyre::Result<StyledString> {
1028        let description = StyledStringBuilder::new()
1029            .append_styled(self.test_status.get_icon(), self.test_status.get_style())
1030            .append_plain(" ")
1031            .append(
1032                self.test_status
1033                    .describe(effects.get_glyphs(), commit, apply_fixes)?,
1034            )
1035            .build();
1036
1037        if verbosity == Verbosity::None {
1038            return Ok(StyledStringBuilder::from_lines(vec![description]));
1039        }
1040
1041        fn abbreviate_lines(path: &Path, verbosity: Verbosity) -> Vec<StyledString> {
1042            let should_show_all_lines = match verbosity {
1043                Verbosity::None => return Vec::new(),
1044                Verbosity::PartialOutput => false,
1045                Verbosity::FullOutput => true,
1046            };
1047
1048            // FIXME: don't read entire file into memory
1049            let contents = match std::fs::read_to_string(path) {
1050                Ok(contents) => contents,
1051                Err(_) => {
1052                    return vec![StyledStringBuilder::new()
1053                        .append_plain("<failed to read file>")
1054                        .build()]
1055                }
1056            };
1057
1058            const NUM_CONTEXT_LINES: usize = 5;
1059            let lines = contents.lines().collect_vec();
1060            let num_missing_lines = lines.len().saturating_sub(2 * NUM_CONTEXT_LINES);
1061            let num_missing_lines_message = format!("<{num_missing_lines} more lines>");
1062            let lines = if lines.is_empty() {
1063                vec!["<no output>"]
1064            } else if num_missing_lines == 0 || should_show_all_lines {
1065                lines
1066            } else {
1067                [
1068                    &lines[..NUM_CONTEXT_LINES],
1069                    &[num_missing_lines_message.as_str()],
1070                    &lines[lines.len() - NUM_CONTEXT_LINES..],
1071                ]
1072                .concat()
1073            };
1074            lines
1075                .into_iter()
1076                .map(|line| StyledStringBuilder::new().append_plain(line).build())
1077                .collect()
1078        }
1079
1080        let interactive = match self.test_status {
1081            TestStatus::CheckoutFailed
1082            | TestStatus::SpawnTestFailed(_)
1083            | TestStatus::TerminatedBySignal
1084            | TestStatus::AlreadyInProgress
1085            | TestStatus::ReadCacheFailed(_)
1086            | TestStatus::Indeterminate { .. }
1087            | TestStatus::Abort { .. } => false,
1088            TestStatus::Failed { interactive, .. } | TestStatus::Passed { interactive, .. } => {
1089                interactive
1090            }
1091        };
1092
1093        let stdout_lines = {
1094            let mut lines = Vec::new();
1095            if !interactive {
1096                lines.push(
1097                    StyledStringBuilder::new()
1098                        .append_styled("Stdout: ", Effect::Bold)
1099                        .append_plain(self.stdout_path.to_string_lossy())
1100                        .build(),
1101                );
1102                lines.extend(abbreviate_lines(&self.stdout_path, verbosity));
1103            }
1104            lines
1105        };
1106        let stderr_lines = {
1107            let mut lines = Vec::new();
1108            if !interactive {
1109                lines.push(
1110                    StyledStringBuilder::new()
1111                        .append_styled("Stderr: ", Effect::Bold)
1112                        .append_plain(self.stderr_path.to_string_lossy())
1113                        .build(),
1114                );
1115                lines.extend(abbreviate_lines(&self.stderr_path, verbosity));
1116            }
1117            lines
1118        };
1119
1120        Ok(StyledStringBuilder::from_lines(
1121            [
1122                &[description],
1123                stdout_lines.as_slice(),
1124                stderr_lines.as_slice(),
1125            ]
1126            .concat(),
1127        ))
1128    }
1129}
1130
1131fn shell_escape(s: impl AsRef<str>) -> String {
1132    let s = s.as_ref();
1133    let mut escaped = String::new();
1134    escaped.push('"');
1135    for c in s.chars() {
1136        match c {
1137            '"' => escaped.push_str(r#"\""#),
1138            '\\' => escaped.push_str(r"\\\\"),
1139            c => escaped.push(c),
1140        }
1141    }
1142    escaped.push('"');
1143    escaped
1144}
1145
1146#[derive(Clone, Debug, Eq, Hash, PartialEq)]
1147struct TestJob {
1148    commit_oid: NonZeroOid,
1149    operation_type: OperationType,
1150}
1151
1152#[derive(Debug, Error)]
1153enum SearchGraphError {
1154    #[error(transparent)]
1155    Dag(#[from] eden_dag::Error),
1156
1157    #[error(transparent)]
1158    Other(#[from] eyre::Error),
1159}
1160
1161#[derive(Debug)]
1162struct SearchGraph<'a> {
1163    dag: &'a Dag,
1164    commit_set: CommitSet,
1165}
1166
1167impl<'a> BasicSourceControlGraph for SearchGraph<'a> {
1168    type Node = NonZeroOid;
1169    type Error = SearchGraphError;
1170
1171    #[instrument]
1172    fn ancestors(&self, node: Self::Node) -> Result<HashSet<Self::Node>, Self::Error> {
1173        let ancestors = self.dag.query_ancestors(CommitSet::from(node))?;
1174        let ancestors = ancestors.intersection(&self.commit_set);
1175        let ancestors = self.dag.commit_set_to_vec(&ancestors)?;
1176        Ok(ancestors.into_iter().collect())
1177    }
1178
1179    #[instrument]
1180    fn descendants(&self, node: Self::Node) -> Result<HashSet<Self::Node>, Self::Error> {
1181        let descendants = self.dag.query_descendants(CommitSet::from(node))?;
1182        let descendants = descendants.intersection(&self.commit_set);
1183        let descendants = self.dag.commit_set_to_vec(&descendants)?;
1184        Ok(descendants.into_iter().collect())
1185    }
1186}
1187
1188/// The results of running all tests.
1189#[derive(Debug)]
1190pub struct TestResults {
1191    /// If a search strategy was provided, the calculated bounds on the input
1192    /// commit set.
1193    pub search_bounds: search::Bounds<NonZeroOid>,
1194
1195    /// The test output for each commit.
1196    pub test_outputs: IndexMap<NonZeroOid, TestOutput>,
1197
1198    /// If testing was aborted, the corresponding error.
1199    pub testing_aborted_error: Option<TestingAbortedError>,
1200}
1201
1202/// Run tests on the provided set of commits.
1203#[instrument]
1204pub fn run_tests<'a>(
1205    now: SystemTime,
1206    effects: &Effects,
1207    git_run_info: &GitRunInfo,
1208    dag: &Dag,
1209    repo: &Repo,
1210    event_log_db: &EventLogDb,
1211    revset: &Revset,
1212    commits: &[Commit],
1213    options: &ResolvedTestOptions,
1214) -> EyreExitOr<TestResults> {
1215    let event_tx_id = EventTransactionId::Suppressed;
1216    let abort_trap = match set_abort_trap(
1217        now,
1218        effects,
1219        git_run_info,
1220        repo,
1221        event_log_db,
1222        event_tx_id,
1223        options.execution_strategy,
1224    )? {
1225        Ok(abort_trap) => abort_trap,
1226        Err(exit_code) => return Ok(Err(exit_code)),
1227    };
1228    let test_results: Result<_, _> = {
1229        let effects = if options.is_interactive {
1230            effects.suppress()
1231        } else {
1232            effects.clone()
1233        };
1234        run_tests_inner(
1235            &effects,
1236            git_run_info,
1237            dag,
1238            repo,
1239            event_log_db,
1240            event_tx_id,
1241            revset,
1242            commits,
1243            options,
1244        )
1245    };
1246
1247    try_exit_code!(clear_abort_trap(
1248        effects,
1249        git_run_info,
1250        event_tx_id,
1251        abort_trap
1252    )?);
1253    test_results
1254}
1255
1256#[instrument]
1257fn run_tests_inner<'a>(
1258    effects: &Effects,
1259    git_run_info: &GitRunInfo,
1260    dag: &Dag,
1261    repo: &Repo,
1262    event_log_db: &EventLogDb,
1263    event_tx_id: EventTransactionId,
1264    revset: &Revset,
1265    commits: &[Commit],
1266    options: &ResolvedTestOptions,
1267) -> EyreExitOr<TestResults> {
1268    let ResolvedTestOptions {
1269        command,
1270        execution_strategy,
1271        search_strategy,
1272        use_cache: _,      // Used only in `make_test_files`.
1273        is_dry_run: _,     // Used only in `apply_fixes`.
1274        is_interactive: _, // Used in `test_commit`.
1275        num_jobs,
1276        verbosity: _,   // Verbosity used by caller to print results.
1277        fix_options: _, // Whether to apply fixes is checked by `test_commit`, after the working directory is set up.
1278    } = &options;
1279
1280    let shell_path = match get_sh() {
1281        Some(shell_path) => shell_path,
1282        None => {
1283            writeln!(
1284                effects.get_output_stream(),
1285                "{}",
1286                effects.get_glyphs().render(
1287                    StyledStringBuilder::new()
1288                        .append_styled(
1289                            "Error: Could not determine path to shell.",
1290                            BaseColor::Red.light()
1291                        )
1292                        .build()
1293                )?
1294            )?;
1295            return Ok(Err(ExitCode(1)));
1296        }
1297    };
1298
1299    if let Some(strategy_value) = execution_strategy.to_possible_value() {
1300        writeln!(
1301            effects.get_output_stream(),
1302            "Using command execution strategy: {}",
1303            effects.get_glyphs().render(
1304                StyledStringBuilder::new()
1305                    .append_styled(strategy_value.get_name(), Effect::Bold)
1306                    .build()
1307            )?,
1308        )?;
1309    }
1310
1311    if let Some(strategy_value) = search_strategy.and_then(|opt| opt.to_possible_value()) {
1312        writeln!(
1313            effects.get_output_stream(),
1314            "Using test search strategy: {}",
1315            effects.get_glyphs().render(
1316                StyledStringBuilder::new()
1317                    .append_styled(strategy_value.get_name(), Effect::Bold)
1318                    .build()
1319            )?,
1320        )?;
1321    }
1322    let search_strategy = match search_strategy {
1323        None => None,
1324        Some(TestSearchStrategy::Linear) => Some(BasicStrategyKind::Linear),
1325        Some(TestSearchStrategy::Reverse) => Some(BasicStrategyKind::LinearReverse),
1326        Some(TestSearchStrategy::Binary) => Some(BasicStrategyKind::Binary),
1327    };
1328    let search_strategy = search_strategy.map(BasicStrategy::new);
1329
1330    let latest_test_command_path = get_latest_test_command_path(repo)?;
1331    if let Some(parent) = latest_test_command_path.parent() {
1332        if let Err(err) = std::fs::create_dir_all(parent) {
1333            warn!(
1334                ?err,
1335                ?latest_test_command_path,
1336                "Failed to create containing directory for latest test command"
1337            );
1338        }
1339    }
1340    if let Err(err) = std::fs::write(&latest_test_command_path, command.to_string()) {
1341        warn!(
1342            ?err,
1343            ?latest_test_command_path,
1344            "Failed to write latest test command to disk"
1345        );
1346    }
1347
1348    let EventLoopOutput {
1349        search,
1350        test_outputs: test_outputs_unordered,
1351        testing_aborted_error,
1352    } = {
1353        let (effects, progress) =
1354            effects.start_operation(OperationType::RunTests(Arc::new(command.to_string())));
1355        progress.notify_progress(0, commits.len());
1356        let commit_jobs = {
1357            let mut results = IndexMap::new();
1358            for commit in commits {
1359                // Create the progress entries in the multiprogress meter without starting them.
1360                // They'll be resumed later in the loop below.
1361                let commit_description = effects
1362                    .get_glyphs()
1363                    .render(commit.friendly_describe(effects.get_glyphs())?)?;
1364                let operation_type =
1365                    OperationType::RunTestOnCommit(Arc::new(commit_description.clone()));
1366                let (_effects, progress) = effects.start_operation(operation_type.clone());
1367                progress.notify_status(
1368                    OperationIcon::InProgress,
1369                    format!("Waiting to run on {commit_description}"),
1370                );
1371                results.insert(
1372                    commit.get_oid(),
1373                    TestJob {
1374                        commit_oid: commit.get_oid(),
1375                        operation_type,
1376                    },
1377                );
1378            }
1379            results
1380        };
1381
1382        let graph = SearchGraph {
1383            dag,
1384            commit_set: commits.iter().map(|c| c.get_oid()).collect(),
1385        };
1386        let search = search::Search::new(graph, commits.iter().map(|c| c.get_oid()));
1387
1388        let work_queue = WorkQueue::new();
1389        let repo_dir = repo.get_path();
1390        crossbeam::thread::scope(|scope| -> eyre::Result<_> {
1391            let (result_tx, result_rx) = crossbeam::channel::unbounded();
1392            let workers: HashMap<WorkerId, crossbeam::thread::ScopedJoinHandle<()>> = {
1393                let mut result = HashMap::new();
1394                for worker_id in 1..=*num_jobs {
1395                    let effects = &effects;
1396                    let progress = &progress;
1397                    let shell_path = &shell_path;
1398                    let work_queue = work_queue.clone();
1399                    let result_tx = result_tx.clone();
1400                    let setup = move || -> eyre::Result<Repo> {
1401                        let repo = Repo::from_dir(repo_dir)?;
1402                        Ok(repo)
1403                    };
1404                    let f = move |job: TestJob, repo: &Repo| -> eyre::Result<TestOutput> {
1405                        let TestJob {
1406                            commit_oid,
1407                            operation_type,
1408                        } = job;
1409                        let commit = repo.find_commit_or_fail(commit_oid)?;
1410                        run_test(
1411                            effects,
1412                            operation_type,
1413                            git_run_info,
1414                            shell_path,
1415                            repo,
1416                            event_tx_id,
1417                            options,
1418                            worker_id,
1419                            &commit,
1420                        )
1421                    };
1422                    result.insert(
1423                        worker_id,
1424                        scope.spawn(move |_scope| {
1425                            worker(progress, worker_id, work_queue, result_tx, setup, f);
1426                            debug!("Exiting spawned thread closure");
1427                        }),
1428                    );
1429                }
1430                result
1431            };
1432
1433            // We rely on `result_rx.recv()` returning `Err` once all the threads have exited
1434            // (whether that reason should be panicking or having finished all work). We have to
1435            // drop our local reference to ensure that we don't deadlock, since otherwise there
1436            // would still be a live receiver for the sender.
1437            drop(result_tx);
1438
1439            let test_results = event_loop(
1440                commit_jobs,
1441                search,
1442                search_strategy.clone(),
1443                *num_jobs,
1444                work_queue.clone(),
1445                result_rx,
1446            );
1447            work_queue.close();
1448            let test_results = test_results?;
1449
1450            if test_results.testing_aborted_error.is_none() && search_strategy.is_none() {
1451                debug!("Waiting for workers");
1452                progress.notify_status(OperationIcon::InProgress, "Waiting for workers");
1453                for (worker_id, worker) in workers {
1454                    worker
1455                        .join()
1456                        .map_err(|_err| eyre::eyre!("Waiting for worker {worker_id} to exit"))?;
1457                }
1458            }
1459
1460            debug!("About to return from thread scope");
1461            Ok(test_results)
1462        })
1463        .map_err(|_| eyre::eyre!("Could not spawn workers"))?
1464        .wrap_err("Failed waiting on workers")?
1465    };
1466    debug!("Returned from thread scope");
1467
1468    // The results may be returned in an arbitrary order if they were produced
1469    // in parallel, so recover the input order to produce deterministic output.
1470    let test_outputs_ordered: IndexMap<NonZeroOid, TestOutput> = {
1471        let mut test_outputs_unordered = test_outputs_unordered;
1472        let mut test_outputs_ordered = IndexMap::new();
1473        for commit_oid in commits.iter().map(|commit| commit.get_oid()) {
1474            match test_outputs_unordered.remove(&commit_oid) {
1475                Some(result) => {
1476                    test_outputs_ordered.insert(commit_oid, result);
1477                }
1478                None => {
1479                    if search_strategy.is_none() && testing_aborted_error.is_none() {
1480                        warn!(?commit_oid, "No result was returned for commit");
1481                    }
1482                }
1483            }
1484        }
1485        if !test_outputs_unordered.is_empty() {
1486            warn!(
1487                ?test_outputs_unordered,
1488                ?commits,
1489                "There were extra results for commits not appearing in the input list"
1490            );
1491        }
1492        test_outputs_ordered
1493    };
1494
1495    Ok(Ok(TestResults {
1496        search_bounds: match search_strategy {
1497            None => Default::default(),
1498            Some(search_strategy) => search.search(&search_strategy)?.bounds,
1499        },
1500        test_outputs: test_outputs_ordered,
1501        testing_aborted_error,
1502    }))
1503}
1504
1505struct EventLoopOutput<'a> {
1506    search: search::Search<SearchGraph<'a>>,
1507    test_outputs: HashMap<NonZeroOid, TestOutput>,
1508    testing_aborted_error: Option<TestingAbortedError>,
1509}
1510
1511fn event_loop(
1512    commit_jobs: IndexMap<NonZeroOid, TestJob>,
1513    mut search: search::Search<SearchGraph>,
1514    search_strategy: Option<BasicStrategy>,
1515    num_jobs: usize,
1516    work_queue: WorkQueue<TestJob>,
1517    result_rx: Receiver<JobResult<TestJob, TestOutput>>,
1518) -> eyre::Result<EventLoopOutput> {
1519    #[derive(Debug)]
1520    enum ScheduledJob {
1521        Scheduled(TestJob),
1522        Complete(TestOutput),
1523    }
1524    let mut scheduled_jobs: HashMap<NonZeroOid, ScheduledJob> = Default::default();
1525    let mut testing_aborted_error = None;
1526
1527    if search_strategy.is_none() {
1528        let jobs_to_schedule = commit_jobs
1529            .keys()
1530            .map(|commit_oid| commit_jobs[commit_oid].clone())
1531            .collect_vec();
1532        debug!(
1533            ?jobs_to_schedule,
1534            "Scheduling all jobs (since no search strategy was specified)"
1535        );
1536        for job in &jobs_to_schedule {
1537            scheduled_jobs.insert(job.commit_oid, ScheduledJob::Scheduled(job.clone()));
1538        }
1539        work_queue.set(jobs_to_schedule);
1540    }
1541
1542    loop {
1543        if let Some(err) = &testing_aborted_error {
1544            debug!(?err, "Testing aborted");
1545            break;
1546        }
1547
1548        if let Some(search_strategy) = &search_strategy {
1549            scheduled_jobs = scheduled_jobs
1550                .into_iter()
1551                .filter_map(|(commit_oid, scheduled_job)| match scheduled_job {
1552                    ScheduledJob::Scheduled(_) => None,
1553                    scheduled_job @ ScheduledJob::Complete(_) => Some((commit_oid, scheduled_job)),
1554                })
1555                .collect();
1556
1557            let solution = search.search(search_strategy)?;
1558            let next_to_search: Vec<_> = solution
1559                .next_to_search
1560                .filter(|commit_oid| {
1561                    let commit_oid = match commit_oid {
1562                        Ok(commit_oid) => commit_oid,
1563                        Err(_) => {
1564                            // Collect `Err` below.
1565                            return true;
1566                        }
1567                    };
1568
1569                    // At this point, `scheduled_jobs` should only contain
1570                    // completed jobs.
1571                    match scheduled_jobs.get(commit_oid) {
1572                        Some(ScheduledJob::Complete(_)) => false,
1573                        Some(ScheduledJob::Scheduled(_)) => {
1574                            warn!(
1575                                ?commit_oid,
1576                                "Left-over scheduled job; this should have already been filtered out."
1577                            );
1578                            true
1579                        }
1580                        None => true,
1581                    }
1582                })
1583                .take(num_jobs)
1584                .try_collect()?;
1585            if next_to_search.is_empty() {
1586                debug!("Search completed, exiting.");
1587                break;
1588            }
1589            let jobs_to_schedule = next_to_search
1590                .into_iter()
1591                .map(|commit_oid| commit_jobs[&commit_oid].clone())
1592                .collect_vec();
1593            debug!(
1594                ?search_strategy,
1595                ?jobs_to_schedule,
1596                "Jobs to schedule for search"
1597            );
1598            for job in &jobs_to_schedule {
1599                if let Some(previous_job) =
1600                    scheduled_jobs.insert(job.commit_oid, ScheduledJob::Scheduled(job.clone()))
1601                {
1602                    warn!(?job, ?previous_job, "Overwriting previously-scheduled job");
1603                }
1604            }
1605            work_queue.set(jobs_to_schedule);
1606        }
1607
1608        let message = {
1609            let jobs_in_progress = scheduled_jobs
1610                .values()
1611                .filter_map(|scheduled_job| match scheduled_job {
1612                    ScheduledJob::Scheduled(job) => Some(job),
1613                    ScheduledJob::Complete(_) => None,
1614                })
1615                .collect_vec();
1616            if jobs_in_progress.is_empty() {
1617                debug!("No more in-progress jobs to wait on, exiting");
1618                break;
1619            }
1620
1621            // If there is work to be done, then block on the next result to
1622            // be received from a worker. This is okay because we won't
1623            // adjust the work queue until we've received the next result.
1624            // (If we wanted to support cancellation or adjusting the number
1625            // of running workers then this wouldn't be sufficient.)
1626            debug!(?jobs_in_progress, "Event loop waiting for new job result");
1627            let result = result_rx.recv();
1628            debug!(?result, "Event loop got new job result");
1629            result
1630        };
1631        let (job, test_output) = match message {
1632            Err(RecvError) => {
1633                debug!("No more job results could be received because result_rx closed");
1634                break;
1635            }
1636
1637            Ok(JobResult::Error(worker_id, job, error_message)) => {
1638                let TestJob {
1639                    commit_oid,
1640                    operation_type: _,
1641                } = job;
1642                eyre::bail!("Worker {worker_id} failed when processing commit {commit_oid}: {error_message}");
1643            }
1644
1645            Ok(JobResult::Done(job, test_output)) => (job, test_output),
1646        };
1647
1648        let TestJob {
1649            commit_oid,
1650            operation_type: _,
1651        } = job;
1652        let (maybe_testing_aborted_error, search_status) = match &test_output.test_status {
1653            TestStatus::CheckoutFailed
1654            | TestStatus::SpawnTestFailed(_)
1655            | TestStatus::TerminatedBySignal
1656            | TestStatus::AlreadyInProgress
1657            | TestStatus::ReadCacheFailed(_)
1658            | TestStatus::Indeterminate { .. } => (None, search::Status::Indeterminate),
1659
1660            TestStatus::Abort { exit_code } => (
1661                Some(TestingAbortedError {
1662                    commit_oid,
1663                    exit_code: *exit_code,
1664                }),
1665                search::Status::Indeterminate,
1666            ),
1667
1668            TestStatus::Failed {
1669                cached: _,
1670                interactive: _,
1671                exit_code: _,
1672            } => (None, search::Status::Failure),
1673
1674            TestStatus::Passed {
1675                cached: _,
1676                fix_info: _,
1677                interactive: _,
1678            } => (None, search::Status::Success),
1679        };
1680        if search_strategy.is_some() {
1681            search.notify(commit_oid, search_status)?;
1682        }
1683        if scheduled_jobs
1684            .insert(commit_oid, ScheduledJob::Complete(test_output))
1685            .is_none()
1686        {
1687            warn!(
1688                ?commit_oid,
1689                "Received test result for commit that was not scheduled"
1690            );
1691        }
1692
1693        if let Some(err) = maybe_testing_aborted_error {
1694            testing_aborted_error = Some(err);
1695        }
1696    }
1697
1698    let test_outputs = scheduled_jobs
1699        .into_iter()
1700        .filter_map(|(commit_oid, scheduled_job)| match scheduled_job {
1701            ScheduledJob::Scheduled(_) => None,
1702            ScheduledJob::Complete(test_output) => Some((commit_oid, test_output)),
1703        })
1704        .collect();
1705    Ok(EventLoopOutput {
1706        search,
1707        test_outputs,
1708        testing_aborted_error,
1709    })
1710}
1711
1712#[instrument]
1713fn print_summary(
1714    effects: &Effects,
1715    dag: &Dag,
1716    repo: &Repo,
1717    revset: &Revset,
1718    command: &TestCommand,
1719    test_results: &TestResults,
1720    is_search: bool,
1721    apply_fixes: bool,
1722    verbosity: &Verbosity,
1723) -> EyreExitOr<()> {
1724    let mut num_passed = 0;
1725    let mut num_failed = 0;
1726    let mut num_skipped = 0;
1727    let mut num_cached_results = 0;
1728    for (commit_oid, test_output) in &test_results.test_outputs {
1729        let commit = repo.find_commit_or_fail(*commit_oid)?;
1730        write!(
1731            effects.get_output_stream(),
1732            "{}",
1733            effects.get_glyphs().render(test_output.describe(
1734                effects,
1735                &commit,
1736                apply_fixes,
1737                *verbosity,
1738            )?)?
1739        )?;
1740        match test_output.test_status {
1741            TestStatus::CheckoutFailed
1742            | TestStatus::SpawnTestFailed(_)
1743            | TestStatus::AlreadyInProgress
1744            | TestStatus::ReadCacheFailed(_)
1745            | TestStatus::TerminatedBySignal
1746            | TestStatus::Indeterminate { .. } => num_skipped += 1,
1747
1748            TestStatus::Abort { .. } => {
1749                num_failed += 1;
1750            }
1751            TestStatus::Failed {
1752                cached,
1753                exit_code: _,
1754                interactive: _,
1755            } => {
1756                num_failed += 1;
1757                if cached {
1758                    num_cached_results += 1;
1759                }
1760            }
1761            TestStatus::Passed {
1762                cached,
1763                fix_info: _,
1764                interactive: _,
1765            } => {
1766                num_passed += 1;
1767                if cached {
1768                    num_cached_results += 1;
1769                }
1770            }
1771        }
1772    }
1773
1774    writeln!(
1775        effects.get_output_stream(),
1776        "Ran command on {}: {}",
1777        Pluralize {
1778            determiner: None,
1779            amount: test_results.test_outputs.len(),
1780            unit: ("commit", "commits")
1781        },
1782        effects.get_glyphs().render(
1783            StyledStringBuilder::new()
1784                .append_styled(command.to_string(), Effect::Bold)
1785                .build()
1786        )?,
1787    )?;
1788
1789    let passed = effects.get_glyphs().render(
1790        StyledStringBuilder::new()
1791            .append_styled(format!("{num_passed} passed"), *STYLE_SUCCESS)
1792            .build(),
1793    )?;
1794    let failed = effects.get_glyphs().render(
1795        StyledStringBuilder::new()
1796            .append_styled(format!("{num_failed} failed"), *STYLE_FAILURE)
1797            .build(),
1798    )?;
1799    let skipped = effects.get_glyphs().render(
1800        StyledStringBuilder::new()
1801            .append_styled(format!("{num_skipped} skipped"), *STYLE_SKIPPED)
1802            .build(),
1803    )?;
1804    writeln!(effects.get_output_stream(), "{passed}, {failed}, {skipped}")?;
1805
1806    if is_search {
1807        let success_commits: CommitSet =
1808            test_results.search_bounds.success.iter().copied().collect();
1809        let success_commits = sorted_commit_set(repo, dag, &success_commits)?;
1810        if success_commits.is_empty() {
1811            writeln!(
1812                effects.get_output_stream(),
1813                "There were no passing commits in the provided set."
1814            )?;
1815        } else {
1816            writeln!(
1817                effects.get_output_stream(),
1818                "Last passing {commits}:",
1819                commits = if success_commits.len() == 1 {
1820                    "commit"
1821                } else {
1822                    "commits"
1823                },
1824            )?;
1825            for commit in success_commits {
1826                writeln!(
1827                    effects.get_output_stream(),
1828                    "{} {}",
1829                    effects.get_glyphs().bullet_point,
1830                    effects
1831                        .get_glyphs()
1832                        .render(commit.friendly_describe(effects.get_glyphs())?)?
1833                )?;
1834            }
1835        }
1836
1837        let failure_commits: CommitSet =
1838            test_results.search_bounds.failure.iter().copied().collect();
1839        let failure_commits = sorted_commit_set(repo, dag, &failure_commits)?;
1840        if failure_commits.is_empty() {
1841            writeln!(
1842                effects.get_output_stream(),
1843                "There were no failing commits in the provided set."
1844            )?;
1845        } else {
1846            writeln!(
1847                effects.get_output_stream(),
1848                "First failing {commits}:",
1849                commits = if failure_commits.len() == 1 {
1850                    "commit"
1851                } else {
1852                    "commits"
1853                },
1854            )?;
1855            for commit in failure_commits {
1856                writeln!(
1857                    effects.get_output_stream(),
1858                    "{} {}",
1859                    effects.get_glyphs().bullet_point,
1860                    effects
1861                        .get_glyphs()
1862                        .render(commit.friendly_describe(effects.get_glyphs())?)?
1863                )?;
1864            }
1865        }
1866    }
1867
1868    if num_cached_results > 0 && get_hint_enabled(repo, Hint::CleanCachedTestResults)? {
1869        writeln!(
1870            effects.get_output_stream(),
1871            "{}: there {}",
1872            effects.get_glyphs().render(get_hint_string())?,
1873            Pluralize {
1874                determiner: Some(("was", "were")),
1875                amount: num_cached_results,
1876                unit: ("cached test result", "cached test results")
1877            }
1878        )?;
1879        writeln!(
1880            effects.get_output_stream(),
1881            "{}: to clear these cached results, run: git test clean {}",
1882            effects.get_glyphs().render(get_hint_string())?,
1883            shell_escape(revset.to_string()),
1884        )?;
1885        print_hint_suppression_notice(effects, Hint::CleanCachedTestResults)?;
1886    }
1887
1888    if let Some(testing_aborted_error) = &test_results.testing_aborted_error {
1889        let TestingAbortedError {
1890            commit_oid,
1891            exit_code,
1892        } = testing_aborted_error;
1893        let commit = repo.find_commit_or_fail(*commit_oid)?;
1894        writeln!(
1895            effects.get_output_stream(),
1896            "Aborted running commands with exit code {} at commit: {}",
1897            exit_code,
1898            effects
1899                .get_glyphs()
1900                .render(commit.friendly_describe(effects.get_glyphs())?)?
1901        )?;
1902        return Ok(Err(ExitCode(1)));
1903    }
1904
1905    if is_search {
1906        Ok(Ok(()))
1907    } else if num_failed > 0 || num_skipped > 0 {
1908        Ok(Err(ExitCode(1)))
1909    } else {
1910        Ok(Ok(()))
1911    }
1912}
1913
1914#[instrument(skip(permissions))]
1915fn apply_fixes(
1916    effects: &Effects,
1917    git_run_info: &GitRunInfo,
1918    dag: &mut Dag,
1919    repo: &Repo,
1920    event_log_db: &EventLogDb,
1921    execute_options: &ExecuteRebasePlanOptions,
1922    permissions: RebasePlanPermissions,
1923    dry_run: bool,
1924    command: &TestCommand,
1925    test_results: &TestResults,
1926) -> EyreExitOr<()> {
1927    let fixed_tree_oids: Vec<(NonZeroOid, NonZeroOid)> = test_results
1928        .test_outputs
1929        .iter()
1930        .filter_map(|(commit_oid, test_output)| match test_output.test_status {
1931            TestStatus::Passed {
1932                cached: _,
1933                fix_info:
1934                    FixInfo {
1935                        head_commit_oid: _,
1936                        snapshot_tree_oid: Some(snapshot_tree_oid),
1937                    },
1938                interactive: _,
1939            } => Some((*commit_oid, snapshot_tree_oid)),
1940
1941            TestStatus::Passed {
1942                cached: _,
1943                fix_info:
1944                    FixInfo {
1945                        head_commit_oid: _,
1946                        snapshot_tree_oid: None,
1947                    },
1948                interactive: _,
1949            }
1950            | TestStatus::CheckoutFailed
1951            | TestStatus::SpawnTestFailed(_)
1952            | TestStatus::TerminatedBySignal
1953            | TestStatus::AlreadyInProgress
1954            | TestStatus::ReadCacheFailed(_)
1955            | TestStatus::Indeterminate { .. }
1956            | TestStatus::Failed { .. }
1957            | TestStatus::Abort { .. } => None,
1958        })
1959        .collect();
1960
1961    #[derive(Debug)]
1962    struct Fix {
1963        original_commit_oid: NonZeroOid,
1964        original_commit_parent_oids: Vec<NonZeroOid>,
1965        fixed_commit_oid: NonZeroOid,
1966    }
1967    let fixes: Vec<Fix> = {
1968        let mut fixes = Vec::new();
1969        for (original_commit_oid, fixed_tree_oid) in fixed_tree_oids {
1970            let original_commit = repo.find_commit_or_fail(original_commit_oid)?;
1971            let original_tree_oid = original_commit.get_tree_oid();
1972            let commit_message = original_commit.get_message_raw();
1973            let commit_message = commit_message.to_str().with_context(|| {
1974                eyre::eyre!(
1975                    "Could not decode commit message for commit: {:?}",
1976                    original_commit_oid
1977                )
1978            })?;
1979            let parents: Vec<Commit> = original_commit
1980                .get_parent_oids()
1981                .into_iter()
1982                .map(|parent_oid| repo.find_commit_or_fail(parent_oid))
1983                .try_collect()?;
1984            let fixed_tree = repo.find_tree_or_fail(fixed_tree_oid)?;
1985            let fixed_commit_oid = repo.create_commit(
1986                None,
1987                &original_commit.get_author(),
1988                &original_commit.get_committer(),
1989                commit_message,
1990                &fixed_tree,
1991                parents.iter().collect(),
1992            )?;
1993            if original_commit_oid == fixed_commit_oid {
1994                continue;
1995            }
1996
1997            let fix = Fix {
1998                original_commit_oid,
1999                original_commit_parent_oids: original_commit.get_parent_oids(),
2000                fixed_commit_oid,
2001            };
2002            debug!(
2003                ?fix,
2004                ?original_tree_oid,
2005                ?fixed_tree_oid,
2006                "Generated fix to apply"
2007            );
2008            fixes.push(fix);
2009        }
2010        fixes
2011    };
2012
2013    dag.sync_from_oids(
2014        effects,
2015        repo,
2016        CommitSet::empty(),
2017        fixes
2018            .iter()
2019            .map(|fix| {
2020                let Fix {
2021                    original_commit_oid: _,
2022                    original_commit_parent_oids: _,
2023                    fixed_commit_oid,
2024                } = fix;
2025                fixed_commit_oid
2026            })
2027            .copied()
2028            .collect(),
2029    )?;
2030
2031    let rebase_plan = {
2032        let mut builder = RebasePlanBuilder::new(dag, permissions);
2033        for fix in &fixes {
2034            let Fix {
2035                original_commit_oid,
2036                original_commit_parent_oids,
2037                fixed_commit_oid,
2038            } = fix;
2039            builder.replace_commit(*original_commit_oid, *fixed_commit_oid)?;
2040            builder.move_subtree(*original_commit_oid, original_commit_parent_oids.clone())?;
2041        }
2042
2043        let original_oids: CommitSet = fixes
2044            .iter()
2045            .map(|fix| {
2046                let Fix {
2047                    original_commit_oid,
2048                    original_commit_parent_oids: _,
2049                    fixed_commit_oid: _,
2050                } = fix;
2051                original_commit_oid
2052            })
2053            .copied()
2054            .collect();
2055        let descendant_oids = dag.query_descendants(original_oids.clone())?;
2056        let descendant_oids = dag
2057            .filter_visible_commits(descendant_oids)?
2058            .difference(&original_oids);
2059        for descendant_oid in dag.commit_set_to_vec(&descendant_oids)? {
2060            let descendant_commit = repo.find_commit_or_fail(descendant_oid)?;
2061            builder.replace_commit(descendant_oid, descendant_oid)?;
2062            builder.move_subtree(descendant_oid, descendant_commit.get_parent_oids())?;
2063        }
2064
2065        let thread_pool = ThreadPoolBuilder::new().build()?;
2066        let repo_pool = RepoResource::new_pool(repo)?;
2067        builder.build(effects, &thread_pool, &repo_pool)?
2068    };
2069
2070    let rebase_plan = match rebase_plan {
2071        Ok(Some(plan)) => plan,
2072        Ok(None) => {
2073            writeln!(effects.get_output_stream(), "No commits to fix.")?;
2074            return Ok(Ok(()));
2075        }
2076        Err(err) => {
2077            err.describe(effects, repo, dag)?;
2078            return Ok(Err(ExitCode(1)));
2079        }
2080    };
2081
2082    let rewritten_oids = if dry_run {
2083        Default::default()
2084    } else {
2085        match execute_rebase_plan(
2086            effects,
2087            git_run_info,
2088            repo,
2089            event_log_db,
2090            &rebase_plan,
2091            execute_options,
2092        )? {
2093            ExecuteRebasePlanResult::Succeeded { rewritten_oids } => rewritten_oids,
2094            ExecuteRebasePlanResult::DeclinedToMerge { failed_merge_info } => {
2095                writeln!(effects.get_output_stream(), "BUG: encountered merge conflicts during git test fix, but we should not be applying any patches: {failed_merge_info:?}")?;
2096                return Ok(Err(ExitCode(1)));
2097            }
2098            ExecuteRebasePlanResult::Failed { exit_code } => return Ok(Err(exit_code)),
2099        }
2100    };
2101    let rewritten_oids = match rewritten_oids {
2102        Some(rewritten_oids) => rewritten_oids,
2103
2104        // Can happen during a dry-run; just produce our rewritten commits which
2105        // haven't been rebased on top of each other yet.
2106        // FIXME: it should be possible to execute the rebase plan but not
2107        // commit the branch moves so that we can preview it.
2108        None => fixes
2109            .iter()
2110            .map(|fix| {
2111                let Fix {
2112                    original_commit_oid,
2113                    original_commit_parent_oids: _,
2114                    fixed_commit_oid,
2115                } = fix;
2116                (
2117                    *original_commit_oid,
2118                    MaybeZeroOid::NonZero(*fixed_commit_oid),
2119                )
2120            })
2121            .collect(),
2122    };
2123
2124    writeln!(
2125        effects.get_output_stream(),
2126        "Fixed {} with {}:",
2127        Pluralize {
2128            determiner: None,
2129            amount: fixes.len(),
2130            unit: ("commit", "commits")
2131        },
2132        effects.get_glyphs().render(
2133            StyledStringBuilder::new()
2134                .append_styled(command.to_string(), Effect::Bold)
2135                .build()
2136        )?,
2137    )?;
2138    for fix in fixes {
2139        let Fix {
2140            original_commit_oid,
2141            original_commit_parent_oids: _,
2142            fixed_commit_oid,
2143        } = fix;
2144        let original_commit = repo.find_commit_or_fail(original_commit_oid)?;
2145        let fixed_commit_oid = rewritten_oids
2146            .get(&original_commit_oid)
2147            .copied()
2148            .unwrap_or(MaybeZeroOid::NonZero(fixed_commit_oid));
2149        match fixed_commit_oid {
2150            MaybeZeroOid::NonZero(fixed_commit_oid) => {
2151                let fixed_commit = repo.find_commit_or_fail(fixed_commit_oid)?;
2152                writeln!(
2153                    effects.get_output_stream(),
2154                    "{} -> {}",
2155                    effects
2156                        .get_glyphs()
2157                        .render(original_commit.friendly_describe_oid(effects.get_glyphs())?)?,
2158                    effects
2159                        .get_glyphs()
2160                        .render(fixed_commit.friendly_describe(effects.get_glyphs())?)?
2161                )?;
2162            }
2163
2164            MaybeZeroOid::Zero => {
2165                // Shouldn't happen.
2166                writeln!(
2167                    effects.get_output_stream(),
2168                    "(deleted) {}",
2169                    effects
2170                        .get_glyphs()
2171                        .render(original_commit.friendly_describe_oid(effects.get_glyphs())?)?,
2172                )?;
2173            }
2174        }
2175    }
2176
2177    if dry_run {
2178        writeln!(effects.get_output_stream(), "(This was a dry-run, so no commits were rewritten. Re-run without the --dry-run option to apply fixes.)")?;
2179    }
2180
2181    Ok(Ok(()))
2182}
2183
2184#[instrument]
2185fn run_test(
2186    effects: &Effects,
2187    operation_type: OperationType,
2188    git_run_info: &GitRunInfo,
2189    shell_path: &Path,
2190    repo: &Repo,
2191    event_tx_id: EventTransactionId,
2192    options: &ResolvedTestOptions,
2193    worker_id: WorkerId,
2194    commit: &Commit,
2195) -> eyre::Result<TestOutput> {
2196    let ResolvedTestOptions {
2197        command: _, // Used in `test_commit`.
2198        execution_strategy,
2199        search_strategy: _, // Caller handles which commits to test.
2200        use_cache: _,       // Used only in `make_test_files`.
2201        is_dry_run: _,      // Used only in `apply_fixes`.
2202        is_interactive: _,  // Used in `test_commit`.
2203        num_jobs: _,        // Caller handles job management.
2204        verbosity: _,
2205        fix_options,
2206    } = options;
2207    let (effects, progress) = effects.start_operation(operation_type);
2208    progress.notify_status(
2209        OperationIcon::InProgress,
2210        format!(
2211            "Preparing {}",
2212            effects
2213                .get_glyphs()
2214                .render(commit.friendly_describe(effects.get_glyphs())?)?
2215        ),
2216    );
2217
2218    let test_output = match make_test_files(repo, commit, options)? {
2219        TestFilesResult::Cached(test_output) => test_output,
2220        TestFilesResult::NotCached(test_files) => {
2221            match prepare_working_directory(
2222                git_run_info,
2223                repo,
2224                event_tx_id,
2225                commit,
2226                *execution_strategy,
2227                worker_id,
2228            )? {
2229                Err(err) => {
2230                    info!(?err, "Failed to prepare working directory for testing");
2231                    let TestFiles {
2232                        temp_dir,
2233                        lock_file: _, // Drop lock.
2234                        result_path,
2235                        result_file: _,
2236                        stdout_path,
2237                        stdout_file: _,
2238                        stderr_path,
2239                        stderr_file: _,
2240                    } = test_files;
2241                    TestOutput {
2242                        temp_dir,
2243                        result_path,
2244                        stdout_path,
2245                        stderr_path,
2246                        test_status: TestStatus::CheckoutFailed,
2247                    }
2248                }
2249                Ok(PreparedWorkingDirectory {
2250                    lock_file: mut working_directory_lock_file,
2251                    path,
2252                }) => {
2253                    progress.notify_status(
2254                        OperationIcon::InProgress,
2255                        format!(
2256                            "Running on {}",
2257                            effects
2258                                .get_glyphs()
2259                                .render(commit.friendly_describe(effects.get_glyphs())?)?
2260                        ),
2261                    );
2262
2263                    let result = test_commit(
2264                        &effects,
2265                        git_run_info,
2266                        repo,
2267                        event_tx_id,
2268                        test_files,
2269                        &path,
2270                        shell_path,
2271                        options,
2272                        commit,
2273                    )?;
2274                    working_directory_lock_file
2275                        .unlock()
2276                        .wrap_err_with(|| format!("Unlocking working directory at {path:?}"))?;
2277                    drop(working_directory_lock_file);
2278                    result
2279                }
2280            }
2281        }
2282    };
2283
2284    let description = StyledStringBuilder::new()
2285        .append(test_output.test_status.describe(
2286            effects.get_glyphs(),
2287            commit,
2288            fix_options.is_some(),
2289        )?)
2290        .build();
2291    progress.notify_status(
2292        match test_output.test_status {
2293            TestStatus::CheckoutFailed
2294            | TestStatus::SpawnTestFailed(_)
2295            | TestStatus::AlreadyInProgress
2296            | TestStatus::ReadCacheFailed(_)
2297            | TestStatus::Indeterminate { .. } => OperationIcon::Warning,
2298
2299            TestStatus::TerminatedBySignal
2300            | TestStatus::Failed { .. }
2301            | TestStatus::Abort { .. } => OperationIcon::Failure,
2302
2303            TestStatus::Passed { .. } => OperationIcon::Success,
2304        },
2305        effects.get_glyphs().render(description)?,
2306    );
2307    Ok(test_output)
2308}
2309
2310#[derive(Debug)]
2311struct TestFiles {
2312    temp_dir: Option<TempDir>,
2313    lock_file: LockFile,
2314    result_path: PathBuf,
2315    result_file: File,
2316    stdout_path: PathBuf,
2317    stdout_file: File,
2318    stderr_path: PathBuf,
2319    stderr_file: File,
2320}
2321
2322#[derive(Debug)]
2323enum TestFilesResult {
2324    Cached(TestOutput),
2325    NotCached(TestFiles),
2326}
2327
2328#[instrument]
2329fn make_test_files(
2330    repo: &Repo,
2331    commit: &Commit,
2332    options: &ResolvedTestOptions,
2333) -> eyre::Result<TestFilesResult> {
2334    if !options.use_cache {
2335        let temp_dir = tempfile::tempdir().context("Creating temporary directory")?;
2336        let lock_path = temp_dir.path().join("pid.lock");
2337        let mut lock_file = LockFile::open(&lock_path)
2338            .wrap_err_with(|| format!("Opening lock file {lock_path:?}"))?;
2339        if !lock_file.try_lock_with_pid()? {
2340            warn!(
2341                ?temp_dir,
2342                ?lock_file,
2343                "Could not acquire lock despite being in a temporary directory"
2344            );
2345        }
2346
2347        let result_path = temp_dir.path().join("result");
2348        let stdout_path = temp_dir.path().join("stdout");
2349        let stderr_path = temp_dir.path().join("stderr");
2350        let result_file = File::create(&result_path)
2351            .wrap_err_with(|| format!("Opening result file {result_path:?}"))?;
2352        let stdout_file = File::create(&stdout_path)
2353            .wrap_err_with(|| format!("Opening stdout file {stdout_path:?}"))?;
2354        let stderr_file = File::create(&stderr_path)
2355            .wrap_err_with(|| format!("Opening stderr file {stderr_path:?}"))?;
2356        return Ok(TestFilesResult::NotCached(TestFiles {
2357            temp_dir: Some(temp_dir),
2358            lock_file,
2359            result_path,
2360            result_file,
2361            stdout_path,
2362            stdout_file,
2363            stderr_path,
2364            stderr_file,
2365        }));
2366    }
2367
2368    let tree_dir = get_test_tree_dir(repo, commit)?;
2369    std::fs::create_dir_all(&tree_dir)
2370        .wrap_err_with(|| format!("Creating tree directory {tree_dir:?}"))?;
2371
2372    let command_dir = tree_dir.join(options.make_command_slug());
2373    std::fs::create_dir_all(&command_dir)
2374        .wrap_err_with(|| format!("Creating command directory {command_dir:?}"))?;
2375
2376    let result_path = command_dir.join("result");
2377    let stdout_path = command_dir.join("stdout");
2378    let stderr_path = command_dir.join("stderr");
2379    let lock_path = command_dir.join("pid.lock");
2380
2381    let mut lock_file =
2382        LockFile::open(&lock_path).wrap_err_with(|| format!("Opening lock file {lock_path:?}"))?;
2383    if !lock_file
2384        .try_lock_with_pid()
2385        .wrap_err_with(|| format!("Locking file {lock_path:?}"))?
2386    {
2387        return Ok(TestFilesResult::Cached(TestOutput {
2388            temp_dir: None,
2389            result_path,
2390            stdout_path,
2391            stderr_path,
2392            test_status: TestStatus::AlreadyInProgress,
2393        }));
2394    }
2395
2396    if let Ok(contents) = std::fs::read_to_string(&result_path) {
2397        // If the file exists but was empty, this indicates that a previous
2398        // attempt did not complete successfully. However, we successfully took
2399        // the lock, so it should be the case that we are the exclusive writers
2400        // to the contents of this directory (i.e. the previous attempt is not
2401        // still running), so it's safe to proceed and overwrite these files.
2402        if !contents.is_empty() {
2403            let serialized_result: Result<SerializedTestResult, _> =
2404                serde_json::from_str(&contents);
2405            let test_status = match serialized_result {
2406                Ok(SerializedTestResult {
2407                    command: _,
2408                    exit_code: 0,
2409                    head_commit_oid,
2410                    snapshot_tree_oid,
2411                    interactive,
2412                }) => TestStatus::Passed {
2413                    cached: true,
2414                    fix_info: FixInfo {
2415                        head_commit_oid: head_commit_oid.map(|SerializedNonZeroOid(oid)| oid),
2416                        snapshot_tree_oid: snapshot_tree_oid.map(|SerializedNonZeroOid(oid)| oid),
2417                    },
2418
2419                    interactive,
2420                },
2421
2422                Ok(SerializedTestResult {
2423                    command: _,
2424                    exit_code,
2425                    head_commit_oid: _,
2426                    snapshot_tree_oid: _,
2427                    interactive: _,
2428                }) if exit_code == TEST_INDETERMINATE_EXIT_CODE => {
2429                    TestStatus::Indeterminate { exit_code }
2430                }
2431
2432                Ok(SerializedTestResult {
2433                    command: _,
2434                    exit_code,
2435                    head_commit_oid: _,
2436                    snapshot_tree_oid: _,
2437                    interactive: _,
2438                }) if exit_code == TEST_ABORT_EXIT_CODE => TestStatus::Abort { exit_code },
2439
2440                Ok(SerializedTestResult {
2441                    command: _,
2442                    exit_code,
2443                    head_commit_oid: _,
2444                    snapshot_tree_oid: _,
2445                    interactive,
2446                }) => TestStatus::Failed {
2447                    cached: true,
2448                    exit_code,
2449                    interactive,
2450                },
2451                Err(err) => TestStatus::ReadCacheFailed(err.to_string()),
2452            };
2453            return Ok(TestFilesResult::Cached(TestOutput {
2454                temp_dir: None,
2455                result_path,
2456                stdout_path,
2457                stderr_path,
2458                test_status,
2459            }));
2460        }
2461    }
2462
2463    let result_file = File::create(&result_path)
2464        .wrap_err_with(|| format!("Opening result file {result_path:?}"))?;
2465    let stdout_file = File::create(&stdout_path)
2466        .wrap_err_with(|| format!("Opening stdout file {stdout_path:?}"))?;
2467    let stderr_file = File::create(&stderr_path)
2468        .wrap_err_with(|| format!("Opening stderr file {stderr_path:?}"))?;
2469    Ok(TestFilesResult::NotCached(TestFiles {
2470        temp_dir: None,
2471        lock_file,
2472        result_path,
2473        result_file,
2474        stdout_path,
2475        stdout_file,
2476        stderr_path,
2477        stderr_file,
2478    }))
2479}
2480
2481#[derive(Debug)]
2482struct PreparedWorkingDirectory {
2483    lock_file: LockFile,
2484    path: PathBuf,
2485}
2486
2487#[allow(dead_code)] // fields are not read except by `Debug` implementation`
2488#[derive(Debug)]
2489enum PrepareWorkingDirectoryError {
2490    LockFailed(PathBuf),
2491    NoWorkingCopy,
2492    CheckoutFailed(NonZeroOid),
2493    CreateWorktreeFailed(PathBuf),
2494}
2495
2496#[instrument]
2497fn prepare_working_directory(
2498    git_run_info: &GitRunInfo,
2499    repo: &Repo,
2500    event_tx_id: EventTransactionId,
2501    commit: &Commit,
2502    strategy: TestExecutionStrategy,
2503    worker_id: WorkerId,
2504) -> eyre::Result<Result<PreparedWorkingDirectory, PrepareWorkingDirectoryError>> {
2505    let test_lock_dir_path = get_test_locks_dir(repo)?;
2506    std::fs::create_dir_all(&test_lock_dir_path)
2507        .wrap_err_with(|| format!("Creating test lock dir path: {test_lock_dir_path:?}"))?;
2508
2509    let lock_file_name = match strategy {
2510        TestExecutionStrategy::WorkingCopy => "working-copy.lock".to_string(),
2511        TestExecutionStrategy::Worktree => {
2512            format!("worktree-{worker_id}.lock")
2513        }
2514    };
2515    let lock_path = test_lock_dir_path.join(lock_file_name);
2516    let mut lock_file = LockFile::open(&lock_path)
2517        .wrap_err_with(|| format!("Opening working copy lock at {lock_path:?}"))?;
2518    if !lock_file
2519        .try_lock_with_pid()
2520        .wrap_err_with(|| format!("Locking working copy with {lock_path:?}"))?
2521    {
2522        return Ok(Err(PrepareWorkingDirectoryError::LockFailed(lock_path)));
2523    }
2524
2525    match strategy {
2526        TestExecutionStrategy::WorkingCopy => {
2527            let working_copy_path = match repo.get_working_copy_path() {
2528                None => return Ok(Err(PrepareWorkingDirectoryError::NoWorkingCopy)),
2529                Some(working_copy_path) => working_copy_path.to_owned(),
2530            };
2531
2532            let GitRunResult { exit_code, stdout: _, stderr: _ } =
2533                // Don't show the `git reset` operation among the progress bars,
2534                // as we only want to see the testing status.
2535                git_run_info.run_silent(
2536                    repo,
2537                    Some(event_tx_id),
2538                    &["reset", "--hard", &commit.get_oid().to_string()],
2539                    Default::default()
2540                ).context("Checking out commit to prepare working directory")?;
2541            if exit_code.is_success() {
2542                Ok(Ok(PreparedWorkingDirectory {
2543                    lock_file,
2544                    path: working_copy_path,
2545                }))
2546            } else {
2547                Ok(Err(PrepareWorkingDirectoryError::CheckoutFailed(
2548                    commit.get_oid(),
2549                )))
2550            }
2551        }
2552
2553        TestExecutionStrategy::Worktree => {
2554            let parent_dir = get_test_worktrees_dir(repo)?;
2555            std::fs::create_dir_all(&parent_dir)
2556                .wrap_err_with(|| format!("Creating worktree parent dir at {parent_dir:?}"))?;
2557
2558            let worktree_dir_name = format!("testing-worktree-{worker_id}");
2559            let worktree_dir = parent_dir.join(worktree_dir_name);
2560            let worktree_dir_str = match worktree_dir.to_str() {
2561                Some(worktree_dir) => worktree_dir,
2562                None => {
2563                    return Ok(Err(PrepareWorkingDirectoryError::CreateWorktreeFailed(
2564                        worktree_dir,
2565                    )));
2566                }
2567            };
2568
2569            if !worktree_dir.exists() {
2570                let GitRunResult {
2571                    exit_code,
2572                    stdout: _,
2573                    stderr: _,
2574                } = git_run_info.run_silent(
2575                    repo,
2576                    Some(event_tx_id),
2577                    &["worktree", "add", worktree_dir_str, "--force", "--detach"],
2578                    Default::default(),
2579                )?;
2580                if !exit_code.is_success() {
2581                    return Ok(Err(PrepareWorkingDirectoryError::CreateWorktreeFailed(
2582                        worktree_dir,
2583                    )));
2584                }
2585            }
2586
2587            let GitRunResult {
2588                exit_code,
2589                stdout: _,
2590                stderr: _,
2591            } = git_run_info.run_silent(
2592                repo,
2593                Some(event_tx_id),
2594                &[
2595                    "-C",
2596                    worktree_dir_str,
2597                    "checkout",
2598                    "--force",
2599                    &commit.get_oid().to_string(),
2600                ],
2601                Default::default(),
2602            )?;
2603            if !exit_code.is_success() {
2604                return Ok(Err(PrepareWorkingDirectoryError::CheckoutFailed(
2605                    commit.get_oid(),
2606                )));
2607            }
2608            Ok(Ok(PreparedWorkingDirectory {
2609                lock_file,
2610                path: worktree_dir,
2611            }))
2612        }
2613    }
2614}
2615
2616#[instrument]
2617fn test_commit(
2618    effects: &Effects,
2619    git_run_info: &GitRunInfo,
2620    repo: &Repo,
2621    event_tx_id: EventTransactionId,
2622    test_files: TestFiles,
2623    working_directory: &Path,
2624    shell_path: &Path,
2625    options: &ResolvedTestOptions,
2626    commit: &Commit,
2627) -> eyre::Result<TestOutput> {
2628    let TestFiles {
2629        temp_dir,
2630        lock_file: _lock_file, // Make sure not to drop lock.
2631        result_path,
2632        result_file,
2633        stdout_path,
2634        stdout_file,
2635        stderr_path,
2636        stderr_file,
2637    } = test_files;
2638
2639    let mut command = Command::new(shell_path);
2640    command
2641        .arg("-c")
2642        .arg(options.command.to_string())
2643        .current_dir(working_directory)
2644        .env(BRANCHLESS_TRANSACTION_ID_ENV_VAR, event_tx_id.to_string())
2645        .env("BRANCHLESS_TEST_COMMIT", commit.get_oid().to_string())
2646        .env("BRANCHLESS_TEST_COMMAND", options.command.to_string());
2647
2648    if options.is_interactive {
2649        let commit_desc = effects
2650            .get_glyphs()
2651            .render(commit.friendly_describe(effects.get_glyphs())?)?;
2652        let passed = "passed";
2653        let exit0 = effects
2654            .get_glyphs()
2655            .render(StyledString::styled("exit 0", *STYLE_SUCCESS))?;
2656        let failed = "failed";
2657        let exit1 = effects
2658            .get_glyphs()
2659            .render(StyledString::styled("exit 1", *STYLE_FAILURE))?;
2660        let skipped = "skipped";
2661        let exit125 = effects
2662            .get_glyphs()
2663            .render(StyledString::styled("exit 125", *STYLE_SKIPPED))?;
2664        let exit127 = effects
2665            .get_glyphs()
2666            .render(StyledString::styled("exit 127", *STYLE_FAILURE))?;
2667
2668        // NB: use `println` here instead of
2669        // `writeln!(effects.get_output_stream(), ...)` because the effects are
2670        // suppressed in interactive mode.
2671        println!(
2672            "\
2673You are now at: {commit_desc}
2674To mark this commit as {passed},run:   {exit0}
2675To mark this commit as {failed}, run:  {exit1}
2676To mark this commit as {skipped}, run: {exit125}
2677To abort testing entirely, run:      {exit127}",
2678        );
2679        match options.execution_strategy {
2680            TestExecutionStrategy::WorkingCopy => {}
2681            TestExecutionStrategy::Worktree => {
2682                let warning = effects
2683                    .get_glyphs()
2684                    .render(StyledString::styled(
2685                        "Warning: You are in a worktree. Your changes will not be propagated between the worktree and the main repository.",
2686                        *STYLE_SKIPPED
2687                    ))?;
2688                println!("{warning}");
2689                println!("To save your changes, create a new branch or note the commit hash.");
2690                println!("To incorporate the changes from the main repository, switch to the main repository's current commit or branch.");
2691            }
2692        }
2693    } else {
2694        command
2695            .stdin(Stdio::null())
2696            .stdout(stdout_file)
2697            .stderr(stderr_file);
2698    }
2699
2700    let exit_code = match command.status() {
2701        Ok(status) => status.code(),
2702        Err(err) => {
2703            return Ok(TestOutput {
2704                temp_dir,
2705                result_path,
2706                stdout_path,
2707                stderr_path,
2708                test_status: TestStatus::SpawnTestFailed(err.to_string()),
2709            });
2710        }
2711    };
2712    let exit_code = match exit_code {
2713        Some(exit_code) => exit_code,
2714        None => {
2715            return Ok(TestOutput {
2716                temp_dir,
2717                result_path,
2718                stdout_path,
2719                stderr_path,
2720                test_status: TestStatus::TerminatedBySignal,
2721            });
2722        }
2723    };
2724    let test_status = match exit_code {
2725        TEST_SUCCESS_EXIT_CODE => {
2726            let fix_info = {
2727                let repo = Repo::from_dir(working_directory)?;
2728                let (head_commit_oid, snapshot) = {
2729                    let index = repo.get_index()?;
2730                    let head_info = repo.get_head_info()?;
2731                    let (snapshot, _status) = repo.get_status(
2732                        &effects.suppress(),
2733                        git_run_info,
2734                        &index,
2735                        &head_info,
2736                        Some(event_tx_id),
2737                    )?;
2738                    (head_info.oid, snapshot)
2739                };
2740                let snapshot_tree_oid = match snapshot.get_working_copy_changes_type()? {
2741                    WorkingCopyChangesType::None | WorkingCopyChangesType::Unstaged => {
2742                        let fixed_tree_oid: MaybeZeroOid = snapshot.commit_unstaged.get_tree_oid();
2743                        fixed_tree_oid.into()
2744                    }
2745                    changes_type @ (WorkingCopyChangesType::Staged
2746                    | WorkingCopyChangesType::Conflicts) => {
2747                        // FIXME: surface information about the fix that failed to be applied.
2748                        warn!(
2749                            ?changes_type,
2750                            "There were staged changes or conflicts in the resulting working copy"
2751                        );
2752                        None
2753                    }
2754                };
2755                FixInfo {
2756                    head_commit_oid,
2757                    snapshot_tree_oid,
2758                }
2759            };
2760            TestStatus::Passed {
2761                cached: false,
2762                fix_info,
2763                interactive: options.is_interactive,
2764            }
2765        }
2766
2767        exit_code @ TEST_INDETERMINATE_EXIT_CODE => TestStatus::Indeterminate { exit_code },
2768        exit_code @ TEST_ABORT_EXIT_CODE => TestStatus::Abort { exit_code },
2769
2770        exit_code => TestStatus::Failed {
2771            cached: false,
2772            exit_code,
2773            interactive: options.is_interactive,
2774        },
2775    };
2776
2777    let fix_info = match &test_status {
2778        TestStatus::Passed {
2779            cached: _,
2780            fix_info,
2781            interactive: _,
2782        } => Some(fix_info),
2783        TestStatus::CheckoutFailed
2784        | TestStatus::SpawnTestFailed(_)
2785        | TestStatus::TerminatedBySignal
2786        | TestStatus::AlreadyInProgress
2787        | TestStatus::ReadCacheFailed(_)
2788        | TestStatus::Failed { .. }
2789        | TestStatus::Abort { .. }
2790        | TestStatus::Indeterminate { .. } => None,
2791    };
2792    let serialized_test_result = SerializedTestResult {
2793        command: options.command.clone(),
2794        exit_code,
2795        head_commit_oid: fix_info
2796            .and_then(|fix_info| fix_info.head_commit_oid.map(SerializedNonZeroOid)),
2797        snapshot_tree_oid: fix_info
2798            .and_then(|fix_info| fix_info.snapshot_tree_oid.map(SerializedNonZeroOid)),
2799        interactive: options.is_interactive,
2800    };
2801    serde_json::to_writer_pretty(result_file, &serialized_test_result)
2802        .wrap_err_with(|| format!("Writing test status {test_status:?} to {result_path:?}"))?;
2803
2804    Ok(TestOutput {
2805        temp_dir,
2806        result_path,
2807        stdout_path,
2808        stderr_path,
2809        test_status,
2810    })
2811}
2812
2813/// Show test output for the command provided in `options` for each of the
2814/// commits in `revset`.
2815#[instrument]
2816fn subcommand_show(
2817    effects: &Effects,
2818    options: &RawTestOptions,
2819    revset: Revset,
2820    resolve_revset_options: &ResolveRevsetOptions,
2821) -> EyreExitOr<()> {
2822    let now = SystemTime::now();
2823    let repo = Repo::from_current_dir()?;
2824    let conn = repo.get_db_conn()?;
2825    let event_log_db = EventLogDb::new(&conn)?;
2826    let event_tx_id = event_log_db.make_transaction_id(now, "test show")?;
2827    let event_replayer = EventReplayer::from_event_log_db(effects, &repo, &event_log_db)?;
2828    let event_cursor = event_replayer.make_default_cursor();
2829    let references_snapshot = repo.get_references_snapshot()?;
2830    let mut dag = Dag::open_and_sync(
2831        effects,
2832        &repo,
2833        &event_replayer,
2834        event_cursor,
2835        &references_snapshot,
2836    )?;
2837
2838    let commit_set =
2839        match resolve_commits(effects, &repo, &mut dag, &[revset], resolve_revset_options) {
2840            Ok(mut commit_sets) => commit_sets.pop().unwrap(),
2841            Err(err) => {
2842                err.describe(effects)?;
2843                return Ok(Err(ExitCode(1)));
2844            }
2845        };
2846
2847    let options = try_exit_code!(ResolvedTestOptions::resolve(
2848        now,
2849        effects,
2850        &dag,
2851        &repo,
2852        event_tx_id,
2853        &commit_set,
2854        None,
2855        options,
2856    )?);
2857
2858    let commits = sorted_commit_set(&repo, &dag, &commit_set)?;
2859    for commit in commits {
2860        let test_files = make_test_files(&repo, &commit, &options)?;
2861        match test_files {
2862            TestFilesResult::NotCached(_) => {
2863                writeln!(
2864                    effects.get_output_stream(),
2865                    "No cached test data for {}",
2866                    effects
2867                        .get_glyphs()
2868                        .render(commit.friendly_describe(effects.get_glyphs())?)?
2869                )?;
2870            }
2871            TestFilesResult::Cached(test_output) => {
2872                write!(
2873                    effects.get_output_stream(),
2874                    "{}",
2875                    effects.get_glyphs().render(test_output.describe(
2876                        effects,
2877                        &commit,
2878                        false,
2879                        options.verbosity
2880                    )?)?,
2881                )?;
2882            }
2883        }
2884    }
2885
2886    if get_hint_enabled(&repo, Hint::TestShowVerbose)? {
2887        match options.verbosity {
2888            Verbosity::None => {
2889                writeln!(
2890                    effects.get_output_stream(),
2891                    "{}: to see more detailed output, re-run with -v/--verbose",
2892                    effects.get_glyphs().render(get_hint_string())?,
2893                )?;
2894                print_hint_suppression_notice(effects, Hint::TestShowVerbose)?;
2895            }
2896            Verbosity::PartialOutput => {
2897                writeln!(
2898                    effects.get_output_stream(),
2899                    "{}: to see more detailed output, re-run with -vv/--verbose --verbose",
2900                    effects.get_glyphs().render(get_hint_string())?,
2901                )?;
2902                print_hint_suppression_notice(effects, Hint::TestShowVerbose)?;
2903            }
2904            Verbosity::FullOutput => {}
2905        }
2906    }
2907
2908    Ok(Ok(()))
2909}
2910
2911/// Delete cached test output for the commits in `revset`.
2912#[instrument]
2913pub fn subcommand_clean(
2914    effects: &Effects,
2915    revset: Revset,
2916    resolve_revset_options: &ResolveRevsetOptions,
2917) -> EyreExitOr<()> {
2918    let repo = Repo::from_current_dir()?;
2919    let conn = repo.get_db_conn()?;
2920    let event_log_db = EventLogDb::new(&conn)?;
2921    let event_replayer = EventReplayer::from_event_log_db(effects, &repo, &event_log_db)?;
2922    let event_cursor = event_replayer.make_default_cursor();
2923    let references_snapshot = repo.get_references_snapshot()?;
2924    let mut dag = Dag::open_and_sync(
2925        effects,
2926        &repo,
2927        &event_replayer,
2928        event_cursor,
2929        &references_snapshot,
2930    )?;
2931
2932    let commit_set =
2933        match resolve_commits(effects, &repo, &mut dag, &[revset], resolve_revset_options) {
2934            Ok(mut commit_sets) => commit_sets.pop().unwrap(),
2935            Err(err) => {
2936                err.describe(effects)?;
2937                return Ok(Err(ExitCode(1)));
2938            }
2939        };
2940
2941    let mut num_cleaned_commits = 0;
2942    for commit in sorted_commit_set(&repo, &dag, &commit_set)? {
2943        let tree_dir = get_test_tree_dir(&repo, &commit)?;
2944        if tree_dir.exists() {
2945            writeln!(
2946                effects.get_output_stream(),
2947                "Cleaning results for {}",
2948                effects
2949                    .get_glyphs()
2950                    .render(commit.friendly_describe(effects.get_glyphs())?)?,
2951            )?;
2952            std::fs::remove_dir_all(&tree_dir)
2953                .with_context(|| format!("Cleaning test dir: {tree_dir:?}"))?;
2954            num_cleaned_commits += 1;
2955        } else {
2956            writeln!(
2957                effects.get_output_stream(),
2958                "Nothing to clean for {}",
2959                effects
2960                    .get_glyphs()
2961                    .render(commit.friendly_describe(effects.get_glyphs())?)?,
2962            )?;
2963        }
2964    }
2965    writeln!(
2966        effects.get_output_stream(),
2967        "Cleaned {}.",
2968        Pluralize {
2969            determiner: None,
2970            amount: num_cleaned_commits,
2971            unit: ("cached test result", "cached test results")
2972        }
2973    )?;
2974    Ok(Ok(()))
2975}
2976
2977#[cfg(test)]
2978mod tests {
2979    use lib::testing::make_git;
2980
2981    use super::*;
2982
2983    #[test]
2984    fn test_lock_prepared_working_directory() -> eyre::Result<()> {
2985        let git = make_git()?;
2986        git.init_repo()?;
2987
2988        let git_run_info = git.get_git_run_info();
2989        let repo = git.get_repo()?;
2990        let conn = repo.get_db_conn()?;
2991        let event_log_db = EventLogDb::new(&conn)?;
2992        let event_tx_id = event_log_db.make_transaction_id(SystemTime::now(), "test")?;
2993        let head_oid = repo.get_head_info()?.oid.unwrap();
2994        let head_commit = repo.find_commit_or_fail(head_oid)?;
2995        let worker_id = 1;
2996
2997        let _prepared_working_copy = prepare_working_directory(
2998            &git_run_info,
2999            &repo,
3000            event_tx_id,
3001            &head_commit,
3002            TestExecutionStrategy::WorkingCopy,
3003            worker_id,
3004        )?
3005        .unwrap();
3006        assert!(matches!(
3007            prepare_working_directory(
3008                &git_run_info,
3009                &repo,
3010                event_tx_id,
3011                &head_commit,
3012                TestExecutionStrategy::WorkingCopy,
3013                worker_id
3014            )?,
3015            Err(PrepareWorkingDirectoryError::LockFailed(_))
3016        ));
3017
3018        let _prepared_worktree = prepare_working_directory(
3019            &git_run_info,
3020            &repo,
3021            event_tx_id,
3022            &head_commit,
3023            TestExecutionStrategy::Worktree,
3024            worker_id,
3025        )?
3026        .unwrap();
3027        assert!(matches!(
3028            prepare_working_directory(
3029                &git_run_info,
3030                &repo,
3031                event_tx_id,
3032                &head_commit,
3033                TestExecutionStrategy::Worktree,
3034                worker_id
3035            )?,
3036            Err(PrepareWorkingDirectoryError::LockFailed(_))
3037        ));
3038
3039        Ok(())
3040    }
3041}