1#![warn(missing_docs)]
6#![warn(
7 clippy::all,
8 clippy::as_conversions,
9 clippy::clone_on_ref_ptr,
10 clippy::dbg_macro
11)]
12#![allow(clippy::too_many_arguments, clippy::blocks_in_if_conditions)]
13
14mod worker;
15
16use std::collections::{HashMap, HashSet};
17use std::fmt::Write as _;
18use std::fs::File;
19use std::path::{Path, PathBuf};
20use std::process::{Command, Stdio};
21use std::sync::Arc;
22use std::time::SystemTime;
23
24use bstr::ByteSlice;
25use clap::ValueEnum;
26use crossbeam::channel::{Receiver, RecvError};
27use cursive::theme::{BaseColor, Effect, Style};
28use cursive::utils::markup::StyledString;
29
30use eyre::WrapErr;
31use fslock::LockFile;
32use git_branchless_invoke::CommandContext;
33use indexmap::IndexMap;
34use itertools::Itertools;
35use lazy_static::lazy_static;
36use lib::core::check_out::CheckOutCommitOptions;
37use lib::core::config::{
38 get_hint_enabled, get_hint_string, get_restack_preserve_timestamps,
39 print_hint_suppression_notice, Hint,
40};
41use lib::core::dag::{sorted_commit_set, CommitSet, Dag};
42use lib::core::effects::{icons, Effects, OperationIcon, OperationType};
43use lib::core::eventlog::{
44 EventLogDb, EventReplayer, EventTransactionId, BRANCHLESS_TRANSACTION_ID_ENV_VAR,
45};
46use lib::core::formatting::{Glyphs, Pluralize, StyledStringBuilder};
47use lib::core::repo_ext::RepoExt;
48use lib::core::rewrite::{
49 execute_rebase_plan, BuildRebasePlanOptions, ExecuteRebasePlanOptions, ExecuteRebasePlanResult,
50 RebaseCommand, RebasePlan, RebasePlanBuilder, RebasePlanPermissions, RepoResource,
51};
52use lib::git::{
53 get_latest_test_command_path, get_test_locks_dir, get_test_tree_dir, get_test_worktrees_dir,
54 make_test_command_slug, Commit, ConfigRead, GitRunInfo, GitRunResult, MaybeZeroOid, NonZeroOid,
55 Repo, SerializedNonZeroOid, SerializedTestResult, TestCommand, WorkingCopyChangesType,
56 TEST_ABORT_EXIT_CODE, TEST_INDETERMINATE_EXIT_CODE, TEST_SUCCESS_EXIT_CODE,
57};
58use lib::try_exit_code;
59use lib::util::{get_sh, ExitCode, EyreExitOr};
60use rayon::ThreadPoolBuilder;
61use scm_bisect::basic::{BasicSourceControlGraph, BasicStrategy, BasicStrategyKind};
62use scm_bisect::search;
63use tempfile::TempDir;
64use thiserror::Error;
65use tracing::{debug, info, instrument, warn};
66
67use git_branchless_opts::{
68 MoveOptions, ResolveRevsetOptions, Revset, TestArgs, TestExecutionStrategy, TestSearchStrategy,
69 TestSubcommand,
70};
71use git_branchless_revset::resolve_commits;
72
73use crate::worker::{worker, JobResult, WorkQueue, WorkerId};
74
75lazy_static! {
76 static ref STYLE_SUCCESS: Style =
77 Style::merge(&[BaseColor::Green.light().into(), Effect::Bold.into()]);
78 static ref STYLE_FAILURE: Style =
79 Style::merge(&[BaseColor::Red.light().into(), Effect::Bold.into()]);
80 static ref STYLE_SKIPPED: Style =
81 Style::merge(&[BaseColor::Yellow.light().into(), Effect::Bold.into()]);
82}
83
84#[derive(Clone, Copy, Debug, Ord, PartialOrd, Eq, PartialEq)]
86pub enum Verbosity {
87 None,
89
90 PartialOutput,
92
93 FullOutput,
95}
96
97impl From<u8> for Verbosity {
98 fn from(value: u8) -> Self {
99 match value {
100 0 => Self::None,
101 1 => Self::PartialOutput,
102 _ => Self::FullOutput,
103 }
104 }
105}
106
107#[derive(Debug)]
110pub struct RawTestOptions {
111 pub exec: Option<String>,
113
114 pub command: Option<String>,
116
117 pub dry_run: bool,
120
121 pub strategy: Option<TestExecutionStrategy>,
123
124 pub search: Option<TestSearchStrategy>,
127
128 pub bisect: bool,
130
131 pub no_cache: bool,
133
134 pub interactive: bool,
136
137 pub jobs: Option<usize>,
139
140 pub verbosity: Verbosity,
142
143 pub apply_fixes: bool,
146}
147
148fn resolve_test_command_alias(
149 effects: &Effects,
150 repo: &Repo,
151 alias: Option<&str>,
152) -> EyreExitOr<String> {
153 let config = repo.get_readonly_config()?;
154 let config_key = format!("branchless.test.alias.{}", alias.unwrap_or("default"));
155 let config_value: Option<String> = config.get(config_key).unwrap_or_default();
156 if let Some(command) = config_value {
157 return Ok(Ok(command));
158 }
159
160 match alias {
161 Some(alias) => {
162 writeln!(
163 effects.get_output_stream(),
164 "\
165The test command alias {alias:?} was not defined.
166
167To create it, run: git config branchless.test.alias.{alias} <command>
168Or use the -x/--exec flag instead to run a test command without first creating an alias."
169 )?;
170 }
171 None => {
172 writeln!(
173 effects.get_output_stream(),
174 "\
175Could not determine test command to run. No test command was provided with -c/--command or
176-x/--exec, and the configuration value 'branchless.test.alias.default' was not set.
177
178To configure a default test command, run: git config branchless.test.alias.default <command>
179To run a specific test command, run: git test run -x <command>
180To run a specific command alias, run: git test run -c <alias>",
181 )?;
182 }
183 }
184
185 let aliases = config.list("branchless.test.alias.*")?;
186 if !aliases.is_empty() {
187 writeln!(
188 effects.get_output_stream(),
189 "\nThese are the currently-configured command aliases:"
190 )?;
191 for (name, command) in aliases {
192 writeln!(
193 effects.get_output_stream(),
194 "{} {name} = {command:?}",
195 effects.get_glyphs().bullet_point,
196 )?;
197 }
198 }
199
200 Ok(Err(ExitCode(1)))
201}
202
203#[allow(missing_docs)]
206#[derive(Debug)]
207pub struct ResolvedTestOptions {
208 pub command: TestCommand,
209 pub execution_strategy: TestExecutionStrategy,
210 pub search_strategy: Option<TestSearchStrategy>,
211 pub is_dry_run: bool,
212 pub use_cache: bool,
213 pub is_interactive: bool,
214 pub num_jobs: usize,
215 pub verbosity: Verbosity,
216 pub fix_options: Option<(ExecuteRebasePlanOptions, RebasePlanPermissions)>,
217}
218
219impl ResolvedTestOptions {
220 pub fn resolve(
222 now: SystemTime,
223 effects: &Effects,
224 dag: &Dag,
225 repo: &Repo,
226 event_tx_id: EventTransactionId,
227 commits: &CommitSet,
228 move_options: Option<&MoveOptions>,
229 options: &RawTestOptions,
230 ) -> EyreExitOr<Self> {
231 let config = repo.get_readonly_config()?;
232 let RawTestOptions {
233 exec: command,
234 command: command_alias,
235 dry_run,
236 strategy,
237 search,
238 bisect,
239 no_cache,
240 interactive,
241 jobs,
242 verbosity,
243 apply_fixes,
244 } = options;
245 let resolved_command = match (command, command_alias) {
246 (Some(command), None) => command.to_owned(),
247 (None, None) => match (interactive, std::env::var("SHELL")) {
248 (true, Ok(shell)) => shell,
249 _ => match resolve_test_command_alias(effects, repo, None)? {
250 Ok(command) => command,
251 Err(exit_code) => {
252 return Ok(Err(exit_code));
253 }
254 },
255 },
256 (None, Some(command_alias)) => {
257 match resolve_test_command_alias(effects, repo, Some(command_alias))? {
258 Ok(command) => command,
259 Err(exit_code) => {
260 return Ok(Err(exit_code));
261 }
262 }
263 }
264 (Some(command), Some(command_alias)) => unreachable!(
265 "Command ({:?}) and command alias ({:?}) are conflicting options",
266 command, command_alias
267 ),
268 };
269 let configured_execution_strategy = match strategy {
270 Some(strategy) => *strategy,
271 None => {
272 let strategy_config_key = "branchless.test.strategy";
273 let strategy: Option<String> = config.get(strategy_config_key)?;
274 match strategy {
275 None => TestExecutionStrategy::WorkingCopy,
276 Some(strategy) => {
277 match TestExecutionStrategy::from_str(&strategy, true) {
278 Ok(strategy) => strategy,
279 Err(_) => {
280 writeln!(effects.get_output_stream(), "Invalid value for config value {strategy_config_key}: {strategy}")?;
281 writeln!(
282 effects.get_output_stream(),
283 "Expected one of: {}",
284 TestExecutionStrategy::value_variants()
285 .iter()
286 .filter_map(|variant| variant.to_possible_value())
287 .map(|value| value.get_name().to_owned())
288 .join(", ")
289 )?;
290 return Ok(Err(ExitCode(1)));
291 }
292 }
293 }
294 }
295 }
296 };
297
298 let jobs_config_key = "branchless.test.jobs";
299 let configured_jobs: Option<i32> = config.get(jobs_config_key)?;
300 let configured_jobs = match configured_jobs {
301 None => None,
302 Some(configured_jobs) => match usize::try_from(configured_jobs) {
303 Ok(configured_jobs) => Some(configured_jobs),
304 Err(err) => {
305 writeln!(
306 effects.get_output_stream(),
307 "Invalid value for config value for {jobs_config_key} ({configured_jobs}): {err}"
308 )?;
309 return Ok(Err(ExitCode(1)));
310 }
311 },
312 };
313 let (resolved_num_jobs, resolved_execution_strategy, resolved_interactive) = match jobs {
314 None => match (strategy, *interactive) {
315 (Some(TestExecutionStrategy::WorkingCopy), interactive) => {
316 (1, TestExecutionStrategy::WorkingCopy, interactive)
317 }
318 (Some(TestExecutionStrategy::Worktree), true) => {
319 (1, TestExecutionStrategy::Worktree, true)
320 }
321 (Some(TestExecutionStrategy::Worktree), false) => (
322 configured_jobs.unwrap_or(1),
323 TestExecutionStrategy::Worktree,
324 false,
325 ),
326 (None, true) => (1, configured_execution_strategy, true),
327 (None, false) => (
328 configured_jobs.unwrap_or(1),
329 configured_execution_strategy,
330 false,
331 ),
332 },
333 Some(1) => (1, configured_execution_strategy, *interactive),
334 Some(jobs) => {
335 if *interactive {
336 writeln!(
337 effects.get_output_stream(),
338 "\
339The --jobs option cannot be used with the --interactive option."
340 )?;
341 return Ok(Err(ExitCode(1)));
342 }
343 match strategy {
345 None | Some(TestExecutionStrategy::Worktree) => {
346 (*jobs, TestExecutionStrategy::Worktree, false)
347 }
348 Some(TestExecutionStrategy::WorkingCopy) => {
349 writeln!(
350 effects.get_output_stream(),
351 "\
352The --jobs option can only be used with --strategy worktree, but --strategy working-copy was provided instead."
353 )?;
354 return Ok(Err(ExitCode(1)));
355 }
356 }
357 }
358 };
359
360 if resolved_interactive != *interactive {
361 writeln!(effects.get_output_stream(),
362 "\
363BUG: Expected resolved_interactive ({resolved_interactive:?}) to match interactive ({interactive:?}). If it doesn't match, then multiple interactive jobs might inadvertently be launched in parallel."
364 )?;
365 return Ok(Err(ExitCode(1)));
366 }
367
368 let resolved_num_jobs = if resolved_num_jobs == 0 {
369 num_cpus::get_physical()
370 } else {
371 resolved_num_jobs
372 };
373 assert!(resolved_num_jobs > 0);
374
375 let fix_options = if *apply_fixes {
376 let move_options = match move_options {
377 Some(move_options) => move_options,
378 None => {
379 writeln!(effects.get_output_stream(), "BUG: fixes were requested to be applied, but no `BuildRebasePlanOptions` were provided.")?;
380 return Ok(Err(ExitCode(1)));
381 }
382 };
383 let MoveOptions {
384 force_rewrite_public_commits,
385 force_in_memory: _,
386 force_on_disk,
387 detect_duplicate_commits_via_patch_id,
388 resolve_merge_conflicts,
389 dump_rebase_constraints,
390 dump_rebase_plan,
391 } = move_options;
392
393 let force_in_memory = true;
394 if *force_on_disk {
395 writeln!(
396 effects.get_output_stream(),
397 "The --on-disk option cannot be provided for fixes. Use the --in-memory option instead."
398 )?;
399 return Ok(Err(ExitCode(1)));
400 }
401
402 let build_options = BuildRebasePlanOptions {
403 force_rewrite_public_commits: *force_rewrite_public_commits,
404 dump_rebase_constraints: *dump_rebase_constraints,
405 dump_rebase_plan: *dump_rebase_plan,
406 detect_duplicate_commits_via_patch_id: *detect_duplicate_commits_via_patch_id,
407 };
408 let execute_options = ExecuteRebasePlanOptions {
409 now,
410 event_tx_id,
411 preserve_timestamps: get_restack_preserve_timestamps(repo)?,
412 force_in_memory,
413 force_on_disk: *force_on_disk,
414 resolve_merge_conflicts: *resolve_merge_conflicts,
415 check_out_commit_options: CheckOutCommitOptions {
416 render_smartlog: false,
417 ..Default::default()
418 },
419 };
420 let permissions =
421 match RebasePlanPermissions::verify_rewrite_set(dag, build_options, commits)? {
422 Ok(permissions) => permissions,
423 Err(err) => {
424 err.describe(effects, repo, dag)?;
425 return Ok(Err(ExitCode(1)));
426 }
427 };
428 Some((execute_options, permissions))
429 } else {
430 None
431 };
432
433 let resolved_search_strategy = if *bisect {
434 Some(TestSearchStrategy::Binary)
435 } else {
436 *search
437 };
438
439 let resolved_test_options = ResolvedTestOptions {
440 command: TestCommand::String(resolved_command),
441 execution_strategy: resolved_execution_strategy,
442 search_strategy: resolved_search_strategy,
443 use_cache: !no_cache,
444 is_dry_run: *dry_run,
445 is_interactive: resolved_interactive,
446 num_jobs: resolved_num_jobs,
447 verbosity: *verbosity,
448 fix_options,
449 };
450 debug!(?resolved_test_options, "Resolved test options");
451 Ok(Ok(resolved_test_options))
452 }
453
454 fn make_command_slug(&self) -> String {
455 make_test_command_slug(self.command.to_string())
456 }
457}
458
459#[instrument]
461pub fn command_main(ctx: CommandContext, args: TestArgs) -> EyreExitOr<()> {
462 let CommandContext {
463 effects,
464 git_run_info,
465 } = ctx;
466 let TestArgs { subcommand } = args;
467 match subcommand {
468 TestSubcommand::Clean {
469 revset,
470 resolve_revset_options,
471 } => subcommand_clean(&effects, revset, &resolve_revset_options),
472
473 TestSubcommand::Run {
474 exec: command,
475 command: command_alias,
476 revset,
477 resolve_revset_options,
478 verbosity,
479 strategy,
480 search,
481 bisect,
482 no_cache,
483 interactive,
484 jobs,
485 } => subcommand_run(
486 &effects,
487 &git_run_info,
488 &RawTestOptions {
489 exec: command,
490 command: command_alias,
491 dry_run: false,
492 strategy,
493 search,
494 bisect,
495 no_cache,
496 interactive,
497 jobs,
498 verbosity: Verbosity::from(verbosity),
499 apply_fixes: false,
500 },
501 revset,
502 &resolve_revset_options,
503 None,
504 ),
505
506 TestSubcommand::Show {
507 exec: command,
508 command: command_alias,
509 revset,
510 resolve_revset_options,
511 verbosity,
512 } => subcommand_show(
513 &effects,
514 &RawTestOptions {
515 exec: command,
516 command: command_alias,
517 dry_run: false,
518 strategy: None,
519 search: None,
520 bisect: false,
521 no_cache: false,
522 interactive: false,
523 jobs: None,
524 verbosity: Verbosity::from(verbosity),
525 apply_fixes: false,
526 },
527 revset,
528 &resolve_revset_options,
529 ),
530
531 TestSubcommand::Fix {
532 exec: command,
533 command: command_alias,
534 dry_run,
535 revset,
536 resolve_revset_options,
537 verbosity,
538 strategy,
539 no_cache,
540 jobs,
541 move_options,
542 } => subcommand_run(
543 &effects,
544 &git_run_info,
545 &RawTestOptions {
546 exec: command,
547 command: command_alias,
548 dry_run,
549 strategy,
550 search: None,
551 bisect: false,
552 no_cache,
553 interactive: false,
554 jobs,
555 verbosity: Verbosity::from(verbosity),
556 apply_fixes: true,
557 },
558 revset,
559 &resolve_revset_options,
560 Some(&move_options),
561 ),
562 }
563}
564
565#[instrument]
567fn subcommand_run(
568 effects: &Effects,
569 git_run_info: &GitRunInfo,
570 options: &RawTestOptions,
571 revset: Revset,
572 resolve_revset_options: &ResolveRevsetOptions,
573 move_options: Option<&MoveOptions>,
574) -> EyreExitOr<()> {
575 let now = SystemTime::now();
576 let repo = Repo::from_current_dir()?;
577 let conn = repo.get_db_conn()?;
578 let event_log_db = EventLogDb::new(&conn)?;
579 let event_tx_id = event_log_db.make_transaction_id(now, "test run")?;
580 let event_replayer = EventReplayer::from_event_log_db(effects, &repo, &event_log_db)?;
581 let event_cursor = event_replayer.make_default_cursor();
582 let references_snapshot = repo.get_references_snapshot()?;
583 let mut dag = Dag::open_and_sync(
584 effects,
585 &repo,
586 &event_replayer,
587 event_cursor,
588 &references_snapshot,
589 )?;
590
591 let commit_set = match resolve_commits(
592 effects,
593 &repo,
594 &mut dag,
595 &[revset.clone()],
596 resolve_revset_options,
597 ) {
598 Ok(mut commit_sets) => commit_sets.pop().unwrap(),
599 Err(err) => {
600 err.describe(effects)?;
601 return Ok(Err(ExitCode(1)));
602 }
603 };
604
605 let options = try_exit_code!(ResolvedTestOptions::resolve(
606 now,
607 effects,
608 &dag,
609 &repo,
610 event_tx_id,
611 &commit_set,
612 move_options,
613 options,
614 )?);
615
616 let commits = sorted_commit_set(&repo, &dag, &commit_set)?;
617 let test_results = try_exit_code!(run_tests(
618 now,
619 effects,
620 git_run_info,
621 &dag,
622 &repo,
623 &event_log_db,
624 &revset,
625 &commits,
626 &options,
627 )?);
628
629 try_exit_code!(print_summary(
630 effects,
631 &dag,
632 &repo,
633 &revset,
634 &options.command,
635 &test_results,
636 options.search_strategy.is_some(),
637 options.fix_options.is_some(),
638 &options.verbosity,
639 )?);
640
641 if let Some((execute_options, permissions)) = &options.fix_options {
642 try_exit_code!(apply_fixes(
643 effects,
644 git_run_info,
645 &mut dag,
646 &repo,
647 &event_log_db,
648 execute_options,
649 permissions.clone(),
650 options.is_dry_run,
651 &options.command,
652 &test_results,
653 )?);
654 }
655
656 Ok(Ok(()))
657}
658
659#[must_use]
660#[derive(Debug)]
661struct AbortTrap {
662 is_active: bool,
663}
664
665#[instrument]
671fn set_abort_trap(
672 now: SystemTime,
673 effects: &Effects,
674 git_run_info: &GitRunInfo,
675 repo: &Repo,
676 event_log_db: &EventLogDb,
677 event_tx_id: EventTransactionId,
678 strategy: TestExecutionStrategy,
679) -> EyreExitOr<AbortTrap> {
680 match strategy {
681 TestExecutionStrategy::Worktree => return Ok(Ok(AbortTrap { is_active: false })),
682 TestExecutionStrategy::WorkingCopy => {}
683 }
684
685 if let Some(operation_type) = repo.get_current_operation_type() {
686 writeln!(
687 effects.get_output_stream(),
688 "A {operation_type} operation is already in progress."
689 )?;
690 writeln!(
691 effects.get_output_stream(),
692 "Run git {operation_type} --continue or git {operation_type} --abort to resolve it and proceed."
693 )?;
694 return Ok(Err(ExitCode(1)));
695 }
696
697 let head_info = repo.get_head_info()?;
698 let head_oid = match head_info.oid {
699 Some(head_oid) => head_oid,
700 None => {
701 writeln!(
702 effects.get_output_stream(),
703 "No commit is currently checked out; cannot start on-disk rebase."
704 )?;
705 writeln!(
706 effects.get_output_stream(),
707 "Check out a commit and try again."
708 )?;
709 return Ok(Err(ExitCode(1)));
710 }
711 };
712
713 let rebase_plan = RebasePlan {
714 first_dest_oid: head_oid,
715 commands: vec![RebaseCommand::Break],
716 };
717 match execute_rebase_plan(
718 effects,
719 git_run_info,
720 repo,
721 event_log_db,
722 &rebase_plan,
723 &ExecuteRebasePlanOptions {
724 now,
725 event_tx_id,
726 preserve_timestamps: true,
727 force_in_memory: false,
728 force_on_disk: true,
729 resolve_merge_conflicts: false,
730 check_out_commit_options: CheckOutCommitOptions {
731 render_smartlog: false,
732 ..Default::default()
733 },
734 },
735 )? {
736 ExecuteRebasePlanResult::Succeeded { rewritten_oids: _ } => {
737 }
739 ExecuteRebasePlanResult::DeclinedToMerge { failed_merge_info } => {
740 writeln!(
741 effects.get_output_stream(),
742 "BUG: Encountered unexpected merge failure: {failed_merge_info:?}"
743 )?;
744 return Ok(Err(ExitCode(1)));
745 }
746 ExecuteRebasePlanResult::Failed { exit_code } => {
747 return Ok(Err(exit_code));
748 }
749 }
750
751 Ok(Ok(AbortTrap { is_active: true }))
752}
753
754#[instrument]
755fn clear_abort_trap(
756 effects: &Effects,
757 git_run_info: &GitRunInfo,
758 event_tx_id: EventTransactionId,
759 abort_trap: AbortTrap,
760) -> EyreExitOr<()> {
761 let AbortTrap { is_active } = abort_trap;
762 if is_active {
763 try_exit_code!(git_run_info.run(effects, Some(event_tx_id), &["rebase", "--abort"])?);
764 }
765 Ok(Ok(()))
766}
767
768#[derive(Debug)]
770pub struct TestOutput {
771 pub temp_dir: Option<TempDir>,
774
775 pub result_path: PathBuf,
778
779 pub stdout_path: PathBuf,
781
782 pub stderr_path: PathBuf,
784
785 pub test_status: TestStatus,
787}
788
789#[derive(Clone, Debug)]
791pub enum TestStatus {
792 CheckoutFailed,
794
795 SpawnTestFailed(String),
797
798 TerminatedBySignal,
801
802 AlreadyInProgress,
805
806 ReadCacheFailed(String),
808
809 Indeterminate {
811 exit_code: i32,
813 },
814
815 Abort {
817 exit_code: i32,
819 },
820
821 Failed {
823 cached: bool,
826
827 exit_code: i32,
829
830 interactive: bool,
833 },
834
835 Passed {
837 cached: bool,
840
841 fix_info: FixInfo,
843
844 interactive: bool,
847 },
848}
849
850#[derive(Clone, Debug)]
852pub struct FixInfo {
853 pub head_commit_oid: Option<NonZeroOid>,
857
858 pub snapshot_tree_oid: Option<NonZeroOid>,
862}
863
864impl TestStatus {
865 #[instrument]
866 fn get_icon(&self) -> &'static str {
867 match self {
868 TestStatus::CheckoutFailed
869 | TestStatus::SpawnTestFailed(_)
870 | TestStatus::AlreadyInProgress
871 | TestStatus::ReadCacheFailed(_)
872 | TestStatus::TerminatedBySignal
873 | TestStatus::Indeterminate { .. } => icons::EXCLAMATION,
874 TestStatus::Failed { .. } | TestStatus::Abort { .. } => icons::CROSS,
875 TestStatus::Passed { .. } => icons::CHECKMARK,
876 }
877 }
878
879 #[instrument]
880 fn get_style(&self) -> Style {
881 match self {
882 TestStatus::CheckoutFailed
883 | TestStatus::SpawnTestFailed(_)
884 | TestStatus::AlreadyInProgress
885 | TestStatus::ReadCacheFailed(_)
886 | TestStatus::TerminatedBySignal
887 | TestStatus::Indeterminate { .. } => *STYLE_SKIPPED,
888 TestStatus::Failed { .. } | TestStatus::Abort { .. } => *STYLE_FAILURE,
889 TestStatus::Passed { .. } => *STYLE_SUCCESS,
890 }
891 }
892
893 #[instrument]
895 pub fn describe(
896 &self,
897 glyphs: &Glyphs,
898 commit: &Commit,
899 apply_fixes: bool,
900 ) -> eyre::Result<StyledString> {
901 let description = match self {
902 TestStatus::CheckoutFailed => StyledStringBuilder::new()
903 .append_styled("Failed to check out: ", self.get_style())
904 .append(commit.friendly_describe(glyphs)?)
905 .build(),
906
907 TestStatus::SpawnTestFailed(err) => StyledStringBuilder::new()
908 .append_styled(
909 format!("Failed to spawn command: {err}: "),
910 self.get_style(),
911 )
912 .append(commit.friendly_describe(glyphs)?)
913 .build(),
914
915 TestStatus::TerminatedBySignal => StyledStringBuilder::new()
916 .append_styled("Command terminated by signal: ", self.get_style())
917 .append(commit.friendly_describe(glyphs)?)
918 .build(),
919
920 TestStatus::AlreadyInProgress => StyledStringBuilder::new()
921 .append_styled("Command already in progress? ", self.get_style())
922 .append(commit.friendly_describe(glyphs)?)
923 .build(),
924
925 TestStatus::ReadCacheFailed(_) => StyledStringBuilder::new()
926 .append_styled("Could not read cached command result: ", self.get_style())
927 .append(commit.friendly_describe(glyphs)?)
928 .build(),
929
930 TestStatus::Indeterminate { exit_code } => StyledStringBuilder::new()
931 .append_styled(
932 format!("Exit code indicated to skip this commit (exit code {exit_code}): "),
933 self.get_style(),
934 )
935 .append(commit.friendly_describe(glyphs)?)
936 .build(),
937
938 TestStatus::Abort { exit_code } => StyledStringBuilder::new()
939 .append_styled(
940 format!("Exit code indicated to abort command (exit code {exit_code}): "),
941 self.get_style(),
942 )
943 .append(commit.friendly_describe(glyphs)?)
944 .build(),
945
946 TestStatus::Failed {
947 cached,
948 interactive,
949 exit_code,
950 } => {
951 let mut descriptors = Vec::new();
952 if *cached {
953 descriptors.push("cached".to_string());
954 }
955 descriptors.push(format!("exit code {exit_code}"));
956 if *interactive {
957 descriptors.push("interactive".to_string());
958 }
959 let descriptors = descriptors.join(", ");
960 StyledStringBuilder::new()
961 .append_styled(format!("Failed ({descriptors}): "), self.get_style())
962 .append(commit.friendly_describe(glyphs)?)
963 .build()
964 }
965
966 TestStatus::Passed {
967 cached,
968 interactive,
969 fix_info:
970 FixInfo {
971 head_commit_oid: _,
972 snapshot_tree_oid,
973 },
974 } => {
975 let mut descriptors = Vec::new();
976 if *cached {
977 descriptors.push("cached".to_string());
978 }
979 match (snapshot_tree_oid, commit.get_tree_oid()) {
980 (Some(snapshot_tree_oid), MaybeZeroOid::NonZero(original_tree_oid)) => {
981 if *snapshot_tree_oid != original_tree_oid {
982 descriptors.push(if apply_fixes {
983 "fixed".to_string()
984 } else {
985 "fixable".to_string()
986 });
987 }
988 }
989 (None, _) | (_, MaybeZeroOid::Zero) => {}
990 }
991 if *interactive {
992 descriptors.push("interactive".to_string());
993 }
994 let descriptors = if descriptors.is_empty() {
995 "".to_string()
996 } else {
997 format!(" ({})", descriptors.join(", "))
998 };
999 StyledStringBuilder::new()
1000 .append_styled(format!("Passed{descriptors}: "), self.get_style())
1001 .append(commit.friendly_describe(glyphs)?)
1002 .build()
1003 }
1004 };
1005 Ok(description)
1006 }
1007}
1008
1009#[derive(Debug)]
1011pub struct TestingAbortedError {
1012 pub commit_oid: NonZeroOid,
1014
1015 pub exit_code: i32,
1017}
1018
1019impl TestOutput {
1020 #[instrument]
1021 fn describe(
1022 &self,
1023 effects: &Effects,
1024 commit: &Commit,
1025 apply_fixes: bool,
1026 verbosity: Verbosity,
1027 ) -> eyre::Result<StyledString> {
1028 let description = StyledStringBuilder::new()
1029 .append_styled(self.test_status.get_icon(), self.test_status.get_style())
1030 .append_plain(" ")
1031 .append(
1032 self.test_status
1033 .describe(effects.get_glyphs(), commit, apply_fixes)?,
1034 )
1035 .build();
1036
1037 if verbosity == Verbosity::None {
1038 return Ok(StyledStringBuilder::from_lines(vec![description]));
1039 }
1040
1041 fn abbreviate_lines(path: &Path, verbosity: Verbosity) -> Vec<StyledString> {
1042 let should_show_all_lines = match verbosity {
1043 Verbosity::None => return Vec::new(),
1044 Verbosity::PartialOutput => false,
1045 Verbosity::FullOutput => true,
1046 };
1047
1048 let contents = match std::fs::read_to_string(path) {
1050 Ok(contents) => contents,
1051 Err(_) => {
1052 return vec![StyledStringBuilder::new()
1053 .append_plain("<failed to read file>")
1054 .build()]
1055 }
1056 };
1057
1058 const NUM_CONTEXT_LINES: usize = 5;
1059 let lines = contents.lines().collect_vec();
1060 let num_missing_lines = lines.len().saturating_sub(2 * NUM_CONTEXT_LINES);
1061 let num_missing_lines_message = format!("<{num_missing_lines} more lines>");
1062 let lines = if lines.is_empty() {
1063 vec!["<no output>"]
1064 } else if num_missing_lines == 0 || should_show_all_lines {
1065 lines
1066 } else {
1067 [
1068 &lines[..NUM_CONTEXT_LINES],
1069 &[num_missing_lines_message.as_str()],
1070 &lines[lines.len() - NUM_CONTEXT_LINES..],
1071 ]
1072 .concat()
1073 };
1074 lines
1075 .into_iter()
1076 .map(|line| StyledStringBuilder::new().append_plain(line).build())
1077 .collect()
1078 }
1079
1080 let interactive = match self.test_status {
1081 TestStatus::CheckoutFailed
1082 | TestStatus::SpawnTestFailed(_)
1083 | TestStatus::TerminatedBySignal
1084 | TestStatus::AlreadyInProgress
1085 | TestStatus::ReadCacheFailed(_)
1086 | TestStatus::Indeterminate { .. }
1087 | TestStatus::Abort { .. } => false,
1088 TestStatus::Failed { interactive, .. } | TestStatus::Passed { interactive, .. } => {
1089 interactive
1090 }
1091 };
1092
1093 let stdout_lines = {
1094 let mut lines = Vec::new();
1095 if !interactive {
1096 lines.push(
1097 StyledStringBuilder::new()
1098 .append_styled("Stdout: ", Effect::Bold)
1099 .append_plain(self.stdout_path.to_string_lossy())
1100 .build(),
1101 );
1102 lines.extend(abbreviate_lines(&self.stdout_path, verbosity));
1103 }
1104 lines
1105 };
1106 let stderr_lines = {
1107 let mut lines = Vec::new();
1108 if !interactive {
1109 lines.push(
1110 StyledStringBuilder::new()
1111 .append_styled("Stderr: ", Effect::Bold)
1112 .append_plain(self.stderr_path.to_string_lossy())
1113 .build(),
1114 );
1115 lines.extend(abbreviate_lines(&self.stderr_path, verbosity));
1116 }
1117 lines
1118 };
1119
1120 Ok(StyledStringBuilder::from_lines(
1121 [
1122 &[description],
1123 stdout_lines.as_slice(),
1124 stderr_lines.as_slice(),
1125 ]
1126 .concat(),
1127 ))
1128 }
1129}
1130
1131fn shell_escape(s: impl AsRef<str>) -> String {
1132 let s = s.as_ref();
1133 let mut escaped = String::new();
1134 escaped.push('"');
1135 for c in s.chars() {
1136 match c {
1137 '"' => escaped.push_str(r#"\""#),
1138 '\\' => escaped.push_str(r"\\\\"),
1139 c => escaped.push(c),
1140 }
1141 }
1142 escaped.push('"');
1143 escaped
1144}
1145
1146#[derive(Clone, Debug, Eq, Hash, PartialEq)]
1147struct TestJob {
1148 commit_oid: NonZeroOid,
1149 operation_type: OperationType,
1150}
1151
1152#[derive(Debug, Error)]
1153enum SearchGraphError {
1154 #[error(transparent)]
1155 Dag(#[from] eden_dag::Error),
1156
1157 #[error(transparent)]
1158 Other(#[from] eyre::Error),
1159}
1160
1161#[derive(Debug)]
1162struct SearchGraph<'a> {
1163 dag: &'a Dag,
1164 commit_set: CommitSet,
1165}
1166
1167impl<'a> BasicSourceControlGraph for SearchGraph<'a> {
1168 type Node = NonZeroOid;
1169 type Error = SearchGraphError;
1170
1171 #[instrument]
1172 fn ancestors(&self, node: Self::Node) -> Result<HashSet<Self::Node>, Self::Error> {
1173 let ancestors = self.dag.query_ancestors(CommitSet::from(node))?;
1174 let ancestors = ancestors.intersection(&self.commit_set);
1175 let ancestors = self.dag.commit_set_to_vec(&ancestors)?;
1176 Ok(ancestors.into_iter().collect())
1177 }
1178
1179 #[instrument]
1180 fn descendants(&self, node: Self::Node) -> Result<HashSet<Self::Node>, Self::Error> {
1181 let descendants = self.dag.query_descendants(CommitSet::from(node))?;
1182 let descendants = descendants.intersection(&self.commit_set);
1183 let descendants = self.dag.commit_set_to_vec(&descendants)?;
1184 Ok(descendants.into_iter().collect())
1185 }
1186}
1187
1188#[derive(Debug)]
1190pub struct TestResults {
1191 pub search_bounds: search::Bounds<NonZeroOid>,
1194
1195 pub test_outputs: IndexMap<NonZeroOid, TestOutput>,
1197
1198 pub testing_aborted_error: Option<TestingAbortedError>,
1200}
1201
1202#[instrument]
1204pub fn run_tests<'a>(
1205 now: SystemTime,
1206 effects: &Effects,
1207 git_run_info: &GitRunInfo,
1208 dag: &Dag,
1209 repo: &Repo,
1210 event_log_db: &EventLogDb,
1211 revset: &Revset,
1212 commits: &[Commit],
1213 options: &ResolvedTestOptions,
1214) -> EyreExitOr<TestResults> {
1215 let event_tx_id = EventTransactionId::Suppressed;
1216 let abort_trap = match set_abort_trap(
1217 now,
1218 effects,
1219 git_run_info,
1220 repo,
1221 event_log_db,
1222 event_tx_id,
1223 options.execution_strategy,
1224 )? {
1225 Ok(abort_trap) => abort_trap,
1226 Err(exit_code) => return Ok(Err(exit_code)),
1227 };
1228 let test_results: Result<_, _> = {
1229 let effects = if options.is_interactive {
1230 effects.suppress()
1231 } else {
1232 effects.clone()
1233 };
1234 run_tests_inner(
1235 &effects,
1236 git_run_info,
1237 dag,
1238 repo,
1239 event_log_db,
1240 event_tx_id,
1241 revset,
1242 commits,
1243 options,
1244 )
1245 };
1246
1247 try_exit_code!(clear_abort_trap(
1248 effects,
1249 git_run_info,
1250 event_tx_id,
1251 abort_trap
1252 )?);
1253 test_results
1254}
1255
1256#[instrument]
1257fn run_tests_inner<'a>(
1258 effects: &Effects,
1259 git_run_info: &GitRunInfo,
1260 dag: &Dag,
1261 repo: &Repo,
1262 event_log_db: &EventLogDb,
1263 event_tx_id: EventTransactionId,
1264 revset: &Revset,
1265 commits: &[Commit],
1266 options: &ResolvedTestOptions,
1267) -> EyreExitOr<TestResults> {
1268 let ResolvedTestOptions {
1269 command,
1270 execution_strategy,
1271 search_strategy,
1272 use_cache: _, is_dry_run: _, is_interactive: _, num_jobs,
1276 verbosity: _, fix_options: _, } = &options;
1279
1280 let shell_path = match get_sh() {
1281 Some(shell_path) => shell_path,
1282 None => {
1283 writeln!(
1284 effects.get_output_stream(),
1285 "{}",
1286 effects.get_glyphs().render(
1287 StyledStringBuilder::new()
1288 .append_styled(
1289 "Error: Could not determine path to shell.",
1290 BaseColor::Red.light()
1291 )
1292 .build()
1293 )?
1294 )?;
1295 return Ok(Err(ExitCode(1)));
1296 }
1297 };
1298
1299 if let Some(strategy_value) = execution_strategy.to_possible_value() {
1300 writeln!(
1301 effects.get_output_stream(),
1302 "Using command execution strategy: {}",
1303 effects.get_glyphs().render(
1304 StyledStringBuilder::new()
1305 .append_styled(strategy_value.get_name(), Effect::Bold)
1306 .build()
1307 )?,
1308 )?;
1309 }
1310
1311 if let Some(strategy_value) = search_strategy.and_then(|opt| opt.to_possible_value()) {
1312 writeln!(
1313 effects.get_output_stream(),
1314 "Using test search strategy: {}",
1315 effects.get_glyphs().render(
1316 StyledStringBuilder::new()
1317 .append_styled(strategy_value.get_name(), Effect::Bold)
1318 .build()
1319 )?,
1320 )?;
1321 }
1322 let search_strategy = match search_strategy {
1323 None => None,
1324 Some(TestSearchStrategy::Linear) => Some(BasicStrategyKind::Linear),
1325 Some(TestSearchStrategy::Reverse) => Some(BasicStrategyKind::LinearReverse),
1326 Some(TestSearchStrategy::Binary) => Some(BasicStrategyKind::Binary),
1327 };
1328 let search_strategy = search_strategy.map(BasicStrategy::new);
1329
1330 let latest_test_command_path = get_latest_test_command_path(repo)?;
1331 if let Some(parent) = latest_test_command_path.parent() {
1332 if let Err(err) = std::fs::create_dir_all(parent) {
1333 warn!(
1334 ?err,
1335 ?latest_test_command_path,
1336 "Failed to create containing directory for latest test command"
1337 );
1338 }
1339 }
1340 if let Err(err) = std::fs::write(&latest_test_command_path, command.to_string()) {
1341 warn!(
1342 ?err,
1343 ?latest_test_command_path,
1344 "Failed to write latest test command to disk"
1345 );
1346 }
1347
1348 let EventLoopOutput {
1349 search,
1350 test_outputs: test_outputs_unordered,
1351 testing_aborted_error,
1352 } = {
1353 let (effects, progress) =
1354 effects.start_operation(OperationType::RunTests(Arc::new(command.to_string())));
1355 progress.notify_progress(0, commits.len());
1356 let commit_jobs = {
1357 let mut results = IndexMap::new();
1358 for commit in commits {
1359 let commit_description = effects
1362 .get_glyphs()
1363 .render(commit.friendly_describe(effects.get_glyphs())?)?;
1364 let operation_type =
1365 OperationType::RunTestOnCommit(Arc::new(commit_description.clone()));
1366 let (_effects, progress) = effects.start_operation(operation_type.clone());
1367 progress.notify_status(
1368 OperationIcon::InProgress,
1369 format!("Waiting to run on {commit_description}"),
1370 );
1371 results.insert(
1372 commit.get_oid(),
1373 TestJob {
1374 commit_oid: commit.get_oid(),
1375 operation_type,
1376 },
1377 );
1378 }
1379 results
1380 };
1381
1382 let graph = SearchGraph {
1383 dag,
1384 commit_set: commits.iter().map(|c| c.get_oid()).collect(),
1385 };
1386 let search = search::Search::new(graph, commits.iter().map(|c| c.get_oid()));
1387
1388 let work_queue = WorkQueue::new();
1389 let repo_dir = repo.get_path();
1390 crossbeam::thread::scope(|scope| -> eyre::Result<_> {
1391 let (result_tx, result_rx) = crossbeam::channel::unbounded();
1392 let workers: HashMap<WorkerId, crossbeam::thread::ScopedJoinHandle<()>> = {
1393 let mut result = HashMap::new();
1394 for worker_id in 1..=*num_jobs {
1395 let effects = &effects;
1396 let progress = &progress;
1397 let shell_path = &shell_path;
1398 let work_queue = work_queue.clone();
1399 let result_tx = result_tx.clone();
1400 let setup = move || -> eyre::Result<Repo> {
1401 let repo = Repo::from_dir(repo_dir)?;
1402 Ok(repo)
1403 };
1404 let f = move |job: TestJob, repo: &Repo| -> eyre::Result<TestOutput> {
1405 let TestJob {
1406 commit_oid,
1407 operation_type,
1408 } = job;
1409 let commit = repo.find_commit_or_fail(commit_oid)?;
1410 run_test(
1411 effects,
1412 operation_type,
1413 git_run_info,
1414 shell_path,
1415 repo,
1416 event_tx_id,
1417 options,
1418 worker_id,
1419 &commit,
1420 )
1421 };
1422 result.insert(
1423 worker_id,
1424 scope.spawn(move |_scope| {
1425 worker(progress, worker_id, work_queue, result_tx, setup, f);
1426 debug!("Exiting spawned thread closure");
1427 }),
1428 );
1429 }
1430 result
1431 };
1432
1433 drop(result_tx);
1438
1439 let test_results = event_loop(
1440 commit_jobs,
1441 search,
1442 search_strategy.clone(),
1443 *num_jobs,
1444 work_queue.clone(),
1445 result_rx,
1446 );
1447 work_queue.close();
1448 let test_results = test_results?;
1449
1450 if test_results.testing_aborted_error.is_none() && search_strategy.is_none() {
1451 debug!("Waiting for workers");
1452 progress.notify_status(OperationIcon::InProgress, "Waiting for workers");
1453 for (worker_id, worker) in workers {
1454 worker
1455 .join()
1456 .map_err(|_err| eyre::eyre!("Waiting for worker {worker_id} to exit"))?;
1457 }
1458 }
1459
1460 debug!("About to return from thread scope");
1461 Ok(test_results)
1462 })
1463 .map_err(|_| eyre::eyre!("Could not spawn workers"))?
1464 .wrap_err("Failed waiting on workers")?
1465 };
1466 debug!("Returned from thread scope");
1467
1468 let test_outputs_ordered: IndexMap<NonZeroOid, TestOutput> = {
1471 let mut test_outputs_unordered = test_outputs_unordered;
1472 let mut test_outputs_ordered = IndexMap::new();
1473 for commit_oid in commits.iter().map(|commit| commit.get_oid()) {
1474 match test_outputs_unordered.remove(&commit_oid) {
1475 Some(result) => {
1476 test_outputs_ordered.insert(commit_oid, result);
1477 }
1478 None => {
1479 if search_strategy.is_none() && testing_aborted_error.is_none() {
1480 warn!(?commit_oid, "No result was returned for commit");
1481 }
1482 }
1483 }
1484 }
1485 if !test_outputs_unordered.is_empty() {
1486 warn!(
1487 ?test_outputs_unordered,
1488 ?commits,
1489 "There were extra results for commits not appearing in the input list"
1490 );
1491 }
1492 test_outputs_ordered
1493 };
1494
1495 Ok(Ok(TestResults {
1496 search_bounds: match search_strategy {
1497 None => Default::default(),
1498 Some(search_strategy) => search.search(&search_strategy)?.bounds,
1499 },
1500 test_outputs: test_outputs_ordered,
1501 testing_aborted_error,
1502 }))
1503}
1504
1505struct EventLoopOutput<'a> {
1506 search: search::Search<SearchGraph<'a>>,
1507 test_outputs: HashMap<NonZeroOid, TestOutput>,
1508 testing_aborted_error: Option<TestingAbortedError>,
1509}
1510
1511fn event_loop(
1512 commit_jobs: IndexMap<NonZeroOid, TestJob>,
1513 mut search: search::Search<SearchGraph>,
1514 search_strategy: Option<BasicStrategy>,
1515 num_jobs: usize,
1516 work_queue: WorkQueue<TestJob>,
1517 result_rx: Receiver<JobResult<TestJob, TestOutput>>,
1518) -> eyre::Result<EventLoopOutput> {
1519 #[derive(Debug)]
1520 enum ScheduledJob {
1521 Scheduled(TestJob),
1522 Complete(TestOutput),
1523 }
1524 let mut scheduled_jobs: HashMap<NonZeroOid, ScheduledJob> = Default::default();
1525 let mut testing_aborted_error = None;
1526
1527 if search_strategy.is_none() {
1528 let jobs_to_schedule = commit_jobs
1529 .keys()
1530 .map(|commit_oid| commit_jobs[commit_oid].clone())
1531 .collect_vec();
1532 debug!(
1533 ?jobs_to_schedule,
1534 "Scheduling all jobs (since no search strategy was specified)"
1535 );
1536 for job in &jobs_to_schedule {
1537 scheduled_jobs.insert(job.commit_oid, ScheduledJob::Scheduled(job.clone()));
1538 }
1539 work_queue.set(jobs_to_schedule);
1540 }
1541
1542 loop {
1543 if let Some(err) = &testing_aborted_error {
1544 debug!(?err, "Testing aborted");
1545 break;
1546 }
1547
1548 if let Some(search_strategy) = &search_strategy {
1549 scheduled_jobs = scheduled_jobs
1550 .into_iter()
1551 .filter_map(|(commit_oid, scheduled_job)| match scheduled_job {
1552 ScheduledJob::Scheduled(_) => None,
1553 scheduled_job @ ScheduledJob::Complete(_) => Some((commit_oid, scheduled_job)),
1554 })
1555 .collect();
1556
1557 let solution = search.search(search_strategy)?;
1558 let next_to_search: Vec<_> = solution
1559 .next_to_search
1560 .filter(|commit_oid| {
1561 let commit_oid = match commit_oid {
1562 Ok(commit_oid) => commit_oid,
1563 Err(_) => {
1564 return true;
1566 }
1567 };
1568
1569 match scheduled_jobs.get(commit_oid) {
1572 Some(ScheduledJob::Complete(_)) => false,
1573 Some(ScheduledJob::Scheduled(_)) => {
1574 warn!(
1575 ?commit_oid,
1576 "Left-over scheduled job; this should have already been filtered out."
1577 );
1578 true
1579 }
1580 None => true,
1581 }
1582 })
1583 .take(num_jobs)
1584 .try_collect()?;
1585 if next_to_search.is_empty() {
1586 debug!("Search completed, exiting.");
1587 break;
1588 }
1589 let jobs_to_schedule = next_to_search
1590 .into_iter()
1591 .map(|commit_oid| commit_jobs[&commit_oid].clone())
1592 .collect_vec();
1593 debug!(
1594 ?search_strategy,
1595 ?jobs_to_schedule,
1596 "Jobs to schedule for search"
1597 );
1598 for job in &jobs_to_schedule {
1599 if let Some(previous_job) =
1600 scheduled_jobs.insert(job.commit_oid, ScheduledJob::Scheduled(job.clone()))
1601 {
1602 warn!(?job, ?previous_job, "Overwriting previously-scheduled job");
1603 }
1604 }
1605 work_queue.set(jobs_to_schedule);
1606 }
1607
1608 let message = {
1609 let jobs_in_progress = scheduled_jobs
1610 .values()
1611 .filter_map(|scheduled_job| match scheduled_job {
1612 ScheduledJob::Scheduled(job) => Some(job),
1613 ScheduledJob::Complete(_) => None,
1614 })
1615 .collect_vec();
1616 if jobs_in_progress.is_empty() {
1617 debug!("No more in-progress jobs to wait on, exiting");
1618 break;
1619 }
1620
1621 debug!(?jobs_in_progress, "Event loop waiting for new job result");
1627 let result = result_rx.recv();
1628 debug!(?result, "Event loop got new job result");
1629 result
1630 };
1631 let (job, test_output) = match message {
1632 Err(RecvError) => {
1633 debug!("No more job results could be received because result_rx closed");
1634 break;
1635 }
1636
1637 Ok(JobResult::Error(worker_id, job, error_message)) => {
1638 let TestJob {
1639 commit_oid,
1640 operation_type: _,
1641 } = job;
1642 eyre::bail!("Worker {worker_id} failed when processing commit {commit_oid}: {error_message}");
1643 }
1644
1645 Ok(JobResult::Done(job, test_output)) => (job, test_output),
1646 };
1647
1648 let TestJob {
1649 commit_oid,
1650 operation_type: _,
1651 } = job;
1652 let (maybe_testing_aborted_error, search_status) = match &test_output.test_status {
1653 TestStatus::CheckoutFailed
1654 | TestStatus::SpawnTestFailed(_)
1655 | TestStatus::TerminatedBySignal
1656 | TestStatus::AlreadyInProgress
1657 | TestStatus::ReadCacheFailed(_)
1658 | TestStatus::Indeterminate { .. } => (None, search::Status::Indeterminate),
1659
1660 TestStatus::Abort { exit_code } => (
1661 Some(TestingAbortedError {
1662 commit_oid,
1663 exit_code: *exit_code,
1664 }),
1665 search::Status::Indeterminate,
1666 ),
1667
1668 TestStatus::Failed {
1669 cached: _,
1670 interactive: _,
1671 exit_code: _,
1672 } => (None, search::Status::Failure),
1673
1674 TestStatus::Passed {
1675 cached: _,
1676 fix_info: _,
1677 interactive: _,
1678 } => (None, search::Status::Success),
1679 };
1680 if search_strategy.is_some() {
1681 search.notify(commit_oid, search_status)?;
1682 }
1683 if scheduled_jobs
1684 .insert(commit_oid, ScheduledJob::Complete(test_output))
1685 .is_none()
1686 {
1687 warn!(
1688 ?commit_oid,
1689 "Received test result for commit that was not scheduled"
1690 );
1691 }
1692
1693 if let Some(err) = maybe_testing_aborted_error {
1694 testing_aborted_error = Some(err);
1695 }
1696 }
1697
1698 let test_outputs = scheduled_jobs
1699 .into_iter()
1700 .filter_map(|(commit_oid, scheduled_job)| match scheduled_job {
1701 ScheduledJob::Scheduled(_) => None,
1702 ScheduledJob::Complete(test_output) => Some((commit_oid, test_output)),
1703 })
1704 .collect();
1705 Ok(EventLoopOutput {
1706 search,
1707 test_outputs,
1708 testing_aborted_error,
1709 })
1710}
1711
1712#[instrument]
1713fn print_summary(
1714 effects: &Effects,
1715 dag: &Dag,
1716 repo: &Repo,
1717 revset: &Revset,
1718 command: &TestCommand,
1719 test_results: &TestResults,
1720 is_search: bool,
1721 apply_fixes: bool,
1722 verbosity: &Verbosity,
1723) -> EyreExitOr<()> {
1724 let mut num_passed = 0;
1725 let mut num_failed = 0;
1726 let mut num_skipped = 0;
1727 let mut num_cached_results = 0;
1728 for (commit_oid, test_output) in &test_results.test_outputs {
1729 let commit = repo.find_commit_or_fail(*commit_oid)?;
1730 write!(
1731 effects.get_output_stream(),
1732 "{}",
1733 effects.get_glyphs().render(test_output.describe(
1734 effects,
1735 &commit,
1736 apply_fixes,
1737 *verbosity,
1738 )?)?
1739 )?;
1740 match test_output.test_status {
1741 TestStatus::CheckoutFailed
1742 | TestStatus::SpawnTestFailed(_)
1743 | TestStatus::AlreadyInProgress
1744 | TestStatus::ReadCacheFailed(_)
1745 | TestStatus::TerminatedBySignal
1746 | TestStatus::Indeterminate { .. } => num_skipped += 1,
1747
1748 TestStatus::Abort { .. } => {
1749 num_failed += 1;
1750 }
1751 TestStatus::Failed {
1752 cached,
1753 exit_code: _,
1754 interactive: _,
1755 } => {
1756 num_failed += 1;
1757 if cached {
1758 num_cached_results += 1;
1759 }
1760 }
1761 TestStatus::Passed {
1762 cached,
1763 fix_info: _,
1764 interactive: _,
1765 } => {
1766 num_passed += 1;
1767 if cached {
1768 num_cached_results += 1;
1769 }
1770 }
1771 }
1772 }
1773
1774 writeln!(
1775 effects.get_output_stream(),
1776 "Ran command on {}: {}",
1777 Pluralize {
1778 determiner: None,
1779 amount: test_results.test_outputs.len(),
1780 unit: ("commit", "commits")
1781 },
1782 effects.get_glyphs().render(
1783 StyledStringBuilder::new()
1784 .append_styled(command.to_string(), Effect::Bold)
1785 .build()
1786 )?,
1787 )?;
1788
1789 let passed = effects.get_glyphs().render(
1790 StyledStringBuilder::new()
1791 .append_styled(format!("{num_passed} passed"), *STYLE_SUCCESS)
1792 .build(),
1793 )?;
1794 let failed = effects.get_glyphs().render(
1795 StyledStringBuilder::new()
1796 .append_styled(format!("{num_failed} failed"), *STYLE_FAILURE)
1797 .build(),
1798 )?;
1799 let skipped = effects.get_glyphs().render(
1800 StyledStringBuilder::new()
1801 .append_styled(format!("{num_skipped} skipped"), *STYLE_SKIPPED)
1802 .build(),
1803 )?;
1804 writeln!(effects.get_output_stream(), "{passed}, {failed}, {skipped}")?;
1805
1806 if is_search {
1807 let success_commits: CommitSet =
1808 test_results.search_bounds.success.iter().copied().collect();
1809 let success_commits = sorted_commit_set(repo, dag, &success_commits)?;
1810 if success_commits.is_empty() {
1811 writeln!(
1812 effects.get_output_stream(),
1813 "There were no passing commits in the provided set."
1814 )?;
1815 } else {
1816 writeln!(
1817 effects.get_output_stream(),
1818 "Last passing {commits}:",
1819 commits = if success_commits.len() == 1 {
1820 "commit"
1821 } else {
1822 "commits"
1823 },
1824 )?;
1825 for commit in success_commits {
1826 writeln!(
1827 effects.get_output_stream(),
1828 "{} {}",
1829 effects.get_glyphs().bullet_point,
1830 effects
1831 .get_glyphs()
1832 .render(commit.friendly_describe(effects.get_glyphs())?)?
1833 )?;
1834 }
1835 }
1836
1837 let failure_commits: CommitSet =
1838 test_results.search_bounds.failure.iter().copied().collect();
1839 let failure_commits = sorted_commit_set(repo, dag, &failure_commits)?;
1840 if failure_commits.is_empty() {
1841 writeln!(
1842 effects.get_output_stream(),
1843 "There were no failing commits in the provided set."
1844 )?;
1845 } else {
1846 writeln!(
1847 effects.get_output_stream(),
1848 "First failing {commits}:",
1849 commits = if failure_commits.len() == 1 {
1850 "commit"
1851 } else {
1852 "commits"
1853 },
1854 )?;
1855 for commit in failure_commits {
1856 writeln!(
1857 effects.get_output_stream(),
1858 "{} {}",
1859 effects.get_glyphs().bullet_point,
1860 effects
1861 .get_glyphs()
1862 .render(commit.friendly_describe(effects.get_glyphs())?)?
1863 )?;
1864 }
1865 }
1866 }
1867
1868 if num_cached_results > 0 && get_hint_enabled(repo, Hint::CleanCachedTestResults)? {
1869 writeln!(
1870 effects.get_output_stream(),
1871 "{}: there {}",
1872 effects.get_glyphs().render(get_hint_string())?,
1873 Pluralize {
1874 determiner: Some(("was", "were")),
1875 amount: num_cached_results,
1876 unit: ("cached test result", "cached test results")
1877 }
1878 )?;
1879 writeln!(
1880 effects.get_output_stream(),
1881 "{}: to clear these cached results, run: git test clean {}",
1882 effects.get_glyphs().render(get_hint_string())?,
1883 shell_escape(revset.to_string()),
1884 )?;
1885 print_hint_suppression_notice(effects, Hint::CleanCachedTestResults)?;
1886 }
1887
1888 if let Some(testing_aborted_error) = &test_results.testing_aborted_error {
1889 let TestingAbortedError {
1890 commit_oid,
1891 exit_code,
1892 } = testing_aborted_error;
1893 let commit = repo.find_commit_or_fail(*commit_oid)?;
1894 writeln!(
1895 effects.get_output_stream(),
1896 "Aborted running commands with exit code {} at commit: {}",
1897 exit_code,
1898 effects
1899 .get_glyphs()
1900 .render(commit.friendly_describe(effects.get_glyphs())?)?
1901 )?;
1902 return Ok(Err(ExitCode(1)));
1903 }
1904
1905 if is_search {
1906 Ok(Ok(()))
1907 } else if num_failed > 0 || num_skipped > 0 {
1908 Ok(Err(ExitCode(1)))
1909 } else {
1910 Ok(Ok(()))
1911 }
1912}
1913
1914#[instrument(skip(permissions))]
1915fn apply_fixes(
1916 effects: &Effects,
1917 git_run_info: &GitRunInfo,
1918 dag: &mut Dag,
1919 repo: &Repo,
1920 event_log_db: &EventLogDb,
1921 execute_options: &ExecuteRebasePlanOptions,
1922 permissions: RebasePlanPermissions,
1923 dry_run: bool,
1924 command: &TestCommand,
1925 test_results: &TestResults,
1926) -> EyreExitOr<()> {
1927 let fixed_tree_oids: Vec<(NonZeroOid, NonZeroOid)> = test_results
1928 .test_outputs
1929 .iter()
1930 .filter_map(|(commit_oid, test_output)| match test_output.test_status {
1931 TestStatus::Passed {
1932 cached: _,
1933 fix_info:
1934 FixInfo {
1935 head_commit_oid: _,
1936 snapshot_tree_oid: Some(snapshot_tree_oid),
1937 },
1938 interactive: _,
1939 } => Some((*commit_oid, snapshot_tree_oid)),
1940
1941 TestStatus::Passed {
1942 cached: _,
1943 fix_info:
1944 FixInfo {
1945 head_commit_oid: _,
1946 snapshot_tree_oid: None,
1947 },
1948 interactive: _,
1949 }
1950 | TestStatus::CheckoutFailed
1951 | TestStatus::SpawnTestFailed(_)
1952 | TestStatus::TerminatedBySignal
1953 | TestStatus::AlreadyInProgress
1954 | TestStatus::ReadCacheFailed(_)
1955 | TestStatus::Indeterminate { .. }
1956 | TestStatus::Failed { .. }
1957 | TestStatus::Abort { .. } => None,
1958 })
1959 .collect();
1960
1961 #[derive(Debug)]
1962 struct Fix {
1963 original_commit_oid: NonZeroOid,
1964 original_commit_parent_oids: Vec<NonZeroOid>,
1965 fixed_commit_oid: NonZeroOid,
1966 }
1967 let fixes: Vec<Fix> = {
1968 let mut fixes = Vec::new();
1969 for (original_commit_oid, fixed_tree_oid) in fixed_tree_oids {
1970 let original_commit = repo.find_commit_or_fail(original_commit_oid)?;
1971 let original_tree_oid = original_commit.get_tree_oid();
1972 let commit_message = original_commit.get_message_raw();
1973 let commit_message = commit_message.to_str().with_context(|| {
1974 eyre::eyre!(
1975 "Could not decode commit message for commit: {:?}",
1976 original_commit_oid
1977 )
1978 })?;
1979 let parents: Vec<Commit> = original_commit
1980 .get_parent_oids()
1981 .into_iter()
1982 .map(|parent_oid| repo.find_commit_or_fail(parent_oid))
1983 .try_collect()?;
1984 let fixed_tree = repo.find_tree_or_fail(fixed_tree_oid)?;
1985 let fixed_commit_oid = repo.create_commit(
1986 None,
1987 &original_commit.get_author(),
1988 &original_commit.get_committer(),
1989 commit_message,
1990 &fixed_tree,
1991 parents.iter().collect(),
1992 )?;
1993 if original_commit_oid == fixed_commit_oid {
1994 continue;
1995 }
1996
1997 let fix = Fix {
1998 original_commit_oid,
1999 original_commit_parent_oids: original_commit.get_parent_oids(),
2000 fixed_commit_oid,
2001 };
2002 debug!(
2003 ?fix,
2004 ?original_tree_oid,
2005 ?fixed_tree_oid,
2006 "Generated fix to apply"
2007 );
2008 fixes.push(fix);
2009 }
2010 fixes
2011 };
2012
2013 dag.sync_from_oids(
2014 effects,
2015 repo,
2016 CommitSet::empty(),
2017 fixes
2018 .iter()
2019 .map(|fix| {
2020 let Fix {
2021 original_commit_oid: _,
2022 original_commit_parent_oids: _,
2023 fixed_commit_oid,
2024 } = fix;
2025 fixed_commit_oid
2026 })
2027 .copied()
2028 .collect(),
2029 )?;
2030
2031 let rebase_plan = {
2032 let mut builder = RebasePlanBuilder::new(dag, permissions);
2033 for fix in &fixes {
2034 let Fix {
2035 original_commit_oid,
2036 original_commit_parent_oids,
2037 fixed_commit_oid,
2038 } = fix;
2039 builder.replace_commit(*original_commit_oid, *fixed_commit_oid)?;
2040 builder.move_subtree(*original_commit_oid, original_commit_parent_oids.clone())?;
2041 }
2042
2043 let original_oids: CommitSet = fixes
2044 .iter()
2045 .map(|fix| {
2046 let Fix {
2047 original_commit_oid,
2048 original_commit_parent_oids: _,
2049 fixed_commit_oid: _,
2050 } = fix;
2051 original_commit_oid
2052 })
2053 .copied()
2054 .collect();
2055 let descendant_oids = dag.query_descendants(original_oids.clone())?;
2056 let descendant_oids = dag
2057 .filter_visible_commits(descendant_oids)?
2058 .difference(&original_oids);
2059 for descendant_oid in dag.commit_set_to_vec(&descendant_oids)? {
2060 let descendant_commit = repo.find_commit_or_fail(descendant_oid)?;
2061 builder.replace_commit(descendant_oid, descendant_oid)?;
2062 builder.move_subtree(descendant_oid, descendant_commit.get_parent_oids())?;
2063 }
2064
2065 let thread_pool = ThreadPoolBuilder::new().build()?;
2066 let repo_pool = RepoResource::new_pool(repo)?;
2067 builder.build(effects, &thread_pool, &repo_pool)?
2068 };
2069
2070 let rebase_plan = match rebase_plan {
2071 Ok(Some(plan)) => plan,
2072 Ok(None) => {
2073 writeln!(effects.get_output_stream(), "No commits to fix.")?;
2074 return Ok(Ok(()));
2075 }
2076 Err(err) => {
2077 err.describe(effects, repo, dag)?;
2078 return Ok(Err(ExitCode(1)));
2079 }
2080 };
2081
2082 let rewritten_oids = if dry_run {
2083 Default::default()
2084 } else {
2085 match execute_rebase_plan(
2086 effects,
2087 git_run_info,
2088 repo,
2089 event_log_db,
2090 &rebase_plan,
2091 execute_options,
2092 )? {
2093 ExecuteRebasePlanResult::Succeeded { rewritten_oids } => rewritten_oids,
2094 ExecuteRebasePlanResult::DeclinedToMerge { failed_merge_info } => {
2095 writeln!(effects.get_output_stream(), "BUG: encountered merge conflicts during git test fix, but we should not be applying any patches: {failed_merge_info:?}")?;
2096 return Ok(Err(ExitCode(1)));
2097 }
2098 ExecuteRebasePlanResult::Failed { exit_code } => return Ok(Err(exit_code)),
2099 }
2100 };
2101 let rewritten_oids = match rewritten_oids {
2102 Some(rewritten_oids) => rewritten_oids,
2103
2104 None => fixes
2109 .iter()
2110 .map(|fix| {
2111 let Fix {
2112 original_commit_oid,
2113 original_commit_parent_oids: _,
2114 fixed_commit_oid,
2115 } = fix;
2116 (
2117 *original_commit_oid,
2118 MaybeZeroOid::NonZero(*fixed_commit_oid),
2119 )
2120 })
2121 .collect(),
2122 };
2123
2124 writeln!(
2125 effects.get_output_stream(),
2126 "Fixed {} with {}:",
2127 Pluralize {
2128 determiner: None,
2129 amount: fixes.len(),
2130 unit: ("commit", "commits")
2131 },
2132 effects.get_glyphs().render(
2133 StyledStringBuilder::new()
2134 .append_styled(command.to_string(), Effect::Bold)
2135 .build()
2136 )?,
2137 )?;
2138 for fix in fixes {
2139 let Fix {
2140 original_commit_oid,
2141 original_commit_parent_oids: _,
2142 fixed_commit_oid,
2143 } = fix;
2144 let original_commit = repo.find_commit_or_fail(original_commit_oid)?;
2145 let fixed_commit_oid = rewritten_oids
2146 .get(&original_commit_oid)
2147 .copied()
2148 .unwrap_or(MaybeZeroOid::NonZero(fixed_commit_oid));
2149 match fixed_commit_oid {
2150 MaybeZeroOid::NonZero(fixed_commit_oid) => {
2151 let fixed_commit = repo.find_commit_or_fail(fixed_commit_oid)?;
2152 writeln!(
2153 effects.get_output_stream(),
2154 "{} -> {}",
2155 effects
2156 .get_glyphs()
2157 .render(original_commit.friendly_describe_oid(effects.get_glyphs())?)?,
2158 effects
2159 .get_glyphs()
2160 .render(fixed_commit.friendly_describe(effects.get_glyphs())?)?
2161 )?;
2162 }
2163
2164 MaybeZeroOid::Zero => {
2165 writeln!(
2167 effects.get_output_stream(),
2168 "(deleted) {}",
2169 effects
2170 .get_glyphs()
2171 .render(original_commit.friendly_describe_oid(effects.get_glyphs())?)?,
2172 )?;
2173 }
2174 }
2175 }
2176
2177 if dry_run {
2178 writeln!(effects.get_output_stream(), "(This was a dry-run, so no commits were rewritten. Re-run without the --dry-run option to apply fixes.)")?;
2179 }
2180
2181 Ok(Ok(()))
2182}
2183
2184#[instrument]
2185fn run_test(
2186 effects: &Effects,
2187 operation_type: OperationType,
2188 git_run_info: &GitRunInfo,
2189 shell_path: &Path,
2190 repo: &Repo,
2191 event_tx_id: EventTransactionId,
2192 options: &ResolvedTestOptions,
2193 worker_id: WorkerId,
2194 commit: &Commit,
2195) -> eyre::Result<TestOutput> {
2196 let ResolvedTestOptions {
2197 command: _, execution_strategy,
2199 search_strategy: _, use_cache: _, is_dry_run: _, is_interactive: _, num_jobs: _, verbosity: _,
2205 fix_options,
2206 } = options;
2207 let (effects, progress) = effects.start_operation(operation_type);
2208 progress.notify_status(
2209 OperationIcon::InProgress,
2210 format!(
2211 "Preparing {}",
2212 effects
2213 .get_glyphs()
2214 .render(commit.friendly_describe(effects.get_glyphs())?)?
2215 ),
2216 );
2217
2218 let test_output = match make_test_files(repo, commit, options)? {
2219 TestFilesResult::Cached(test_output) => test_output,
2220 TestFilesResult::NotCached(test_files) => {
2221 match prepare_working_directory(
2222 git_run_info,
2223 repo,
2224 event_tx_id,
2225 commit,
2226 *execution_strategy,
2227 worker_id,
2228 )? {
2229 Err(err) => {
2230 info!(?err, "Failed to prepare working directory for testing");
2231 let TestFiles {
2232 temp_dir,
2233 lock_file: _, result_path,
2235 result_file: _,
2236 stdout_path,
2237 stdout_file: _,
2238 stderr_path,
2239 stderr_file: _,
2240 } = test_files;
2241 TestOutput {
2242 temp_dir,
2243 result_path,
2244 stdout_path,
2245 stderr_path,
2246 test_status: TestStatus::CheckoutFailed,
2247 }
2248 }
2249 Ok(PreparedWorkingDirectory {
2250 lock_file: mut working_directory_lock_file,
2251 path,
2252 }) => {
2253 progress.notify_status(
2254 OperationIcon::InProgress,
2255 format!(
2256 "Running on {}",
2257 effects
2258 .get_glyphs()
2259 .render(commit.friendly_describe(effects.get_glyphs())?)?
2260 ),
2261 );
2262
2263 let result = test_commit(
2264 &effects,
2265 git_run_info,
2266 repo,
2267 event_tx_id,
2268 test_files,
2269 &path,
2270 shell_path,
2271 options,
2272 commit,
2273 )?;
2274 working_directory_lock_file
2275 .unlock()
2276 .wrap_err_with(|| format!("Unlocking working directory at {path:?}"))?;
2277 drop(working_directory_lock_file);
2278 result
2279 }
2280 }
2281 }
2282 };
2283
2284 let description = StyledStringBuilder::new()
2285 .append(test_output.test_status.describe(
2286 effects.get_glyphs(),
2287 commit,
2288 fix_options.is_some(),
2289 )?)
2290 .build();
2291 progress.notify_status(
2292 match test_output.test_status {
2293 TestStatus::CheckoutFailed
2294 | TestStatus::SpawnTestFailed(_)
2295 | TestStatus::AlreadyInProgress
2296 | TestStatus::ReadCacheFailed(_)
2297 | TestStatus::Indeterminate { .. } => OperationIcon::Warning,
2298
2299 TestStatus::TerminatedBySignal
2300 | TestStatus::Failed { .. }
2301 | TestStatus::Abort { .. } => OperationIcon::Failure,
2302
2303 TestStatus::Passed { .. } => OperationIcon::Success,
2304 },
2305 effects.get_glyphs().render(description)?,
2306 );
2307 Ok(test_output)
2308}
2309
2310#[derive(Debug)]
2311struct TestFiles {
2312 temp_dir: Option<TempDir>,
2313 lock_file: LockFile,
2314 result_path: PathBuf,
2315 result_file: File,
2316 stdout_path: PathBuf,
2317 stdout_file: File,
2318 stderr_path: PathBuf,
2319 stderr_file: File,
2320}
2321
2322#[derive(Debug)]
2323enum TestFilesResult {
2324 Cached(TestOutput),
2325 NotCached(TestFiles),
2326}
2327
2328#[instrument]
2329fn make_test_files(
2330 repo: &Repo,
2331 commit: &Commit,
2332 options: &ResolvedTestOptions,
2333) -> eyre::Result<TestFilesResult> {
2334 if !options.use_cache {
2335 let temp_dir = tempfile::tempdir().context("Creating temporary directory")?;
2336 let lock_path = temp_dir.path().join("pid.lock");
2337 let mut lock_file = LockFile::open(&lock_path)
2338 .wrap_err_with(|| format!("Opening lock file {lock_path:?}"))?;
2339 if !lock_file.try_lock_with_pid()? {
2340 warn!(
2341 ?temp_dir,
2342 ?lock_file,
2343 "Could not acquire lock despite being in a temporary directory"
2344 );
2345 }
2346
2347 let result_path = temp_dir.path().join("result");
2348 let stdout_path = temp_dir.path().join("stdout");
2349 let stderr_path = temp_dir.path().join("stderr");
2350 let result_file = File::create(&result_path)
2351 .wrap_err_with(|| format!("Opening result file {result_path:?}"))?;
2352 let stdout_file = File::create(&stdout_path)
2353 .wrap_err_with(|| format!("Opening stdout file {stdout_path:?}"))?;
2354 let stderr_file = File::create(&stderr_path)
2355 .wrap_err_with(|| format!("Opening stderr file {stderr_path:?}"))?;
2356 return Ok(TestFilesResult::NotCached(TestFiles {
2357 temp_dir: Some(temp_dir),
2358 lock_file,
2359 result_path,
2360 result_file,
2361 stdout_path,
2362 stdout_file,
2363 stderr_path,
2364 stderr_file,
2365 }));
2366 }
2367
2368 let tree_dir = get_test_tree_dir(repo, commit)?;
2369 std::fs::create_dir_all(&tree_dir)
2370 .wrap_err_with(|| format!("Creating tree directory {tree_dir:?}"))?;
2371
2372 let command_dir = tree_dir.join(options.make_command_slug());
2373 std::fs::create_dir_all(&command_dir)
2374 .wrap_err_with(|| format!("Creating command directory {command_dir:?}"))?;
2375
2376 let result_path = command_dir.join("result");
2377 let stdout_path = command_dir.join("stdout");
2378 let stderr_path = command_dir.join("stderr");
2379 let lock_path = command_dir.join("pid.lock");
2380
2381 let mut lock_file =
2382 LockFile::open(&lock_path).wrap_err_with(|| format!("Opening lock file {lock_path:?}"))?;
2383 if !lock_file
2384 .try_lock_with_pid()
2385 .wrap_err_with(|| format!("Locking file {lock_path:?}"))?
2386 {
2387 return Ok(TestFilesResult::Cached(TestOutput {
2388 temp_dir: None,
2389 result_path,
2390 stdout_path,
2391 stderr_path,
2392 test_status: TestStatus::AlreadyInProgress,
2393 }));
2394 }
2395
2396 if let Ok(contents) = std::fs::read_to_string(&result_path) {
2397 if !contents.is_empty() {
2403 let serialized_result: Result<SerializedTestResult, _> =
2404 serde_json::from_str(&contents);
2405 let test_status = match serialized_result {
2406 Ok(SerializedTestResult {
2407 command: _,
2408 exit_code: 0,
2409 head_commit_oid,
2410 snapshot_tree_oid,
2411 interactive,
2412 }) => TestStatus::Passed {
2413 cached: true,
2414 fix_info: FixInfo {
2415 head_commit_oid: head_commit_oid.map(|SerializedNonZeroOid(oid)| oid),
2416 snapshot_tree_oid: snapshot_tree_oid.map(|SerializedNonZeroOid(oid)| oid),
2417 },
2418
2419 interactive,
2420 },
2421
2422 Ok(SerializedTestResult {
2423 command: _,
2424 exit_code,
2425 head_commit_oid: _,
2426 snapshot_tree_oid: _,
2427 interactive: _,
2428 }) if exit_code == TEST_INDETERMINATE_EXIT_CODE => {
2429 TestStatus::Indeterminate { exit_code }
2430 }
2431
2432 Ok(SerializedTestResult {
2433 command: _,
2434 exit_code,
2435 head_commit_oid: _,
2436 snapshot_tree_oid: _,
2437 interactive: _,
2438 }) if exit_code == TEST_ABORT_EXIT_CODE => TestStatus::Abort { exit_code },
2439
2440 Ok(SerializedTestResult {
2441 command: _,
2442 exit_code,
2443 head_commit_oid: _,
2444 snapshot_tree_oid: _,
2445 interactive,
2446 }) => TestStatus::Failed {
2447 cached: true,
2448 exit_code,
2449 interactive,
2450 },
2451 Err(err) => TestStatus::ReadCacheFailed(err.to_string()),
2452 };
2453 return Ok(TestFilesResult::Cached(TestOutput {
2454 temp_dir: None,
2455 result_path,
2456 stdout_path,
2457 stderr_path,
2458 test_status,
2459 }));
2460 }
2461 }
2462
2463 let result_file = File::create(&result_path)
2464 .wrap_err_with(|| format!("Opening result file {result_path:?}"))?;
2465 let stdout_file = File::create(&stdout_path)
2466 .wrap_err_with(|| format!("Opening stdout file {stdout_path:?}"))?;
2467 let stderr_file = File::create(&stderr_path)
2468 .wrap_err_with(|| format!("Opening stderr file {stderr_path:?}"))?;
2469 Ok(TestFilesResult::NotCached(TestFiles {
2470 temp_dir: None,
2471 lock_file,
2472 result_path,
2473 result_file,
2474 stdout_path,
2475 stdout_file,
2476 stderr_path,
2477 stderr_file,
2478 }))
2479}
2480
2481#[derive(Debug)]
2482struct PreparedWorkingDirectory {
2483 lock_file: LockFile,
2484 path: PathBuf,
2485}
2486
2487#[allow(dead_code)] #[derive(Debug)]
2489enum PrepareWorkingDirectoryError {
2490 LockFailed(PathBuf),
2491 NoWorkingCopy,
2492 CheckoutFailed(NonZeroOid),
2493 CreateWorktreeFailed(PathBuf),
2494}
2495
2496#[instrument]
2497fn prepare_working_directory(
2498 git_run_info: &GitRunInfo,
2499 repo: &Repo,
2500 event_tx_id: EventTransactionId,
2501 commit: &Commit,
2502 strategy: TestExecutionStrategy,
2503 worker_id: WorkerId,
2504) -> eyre::Result<Result<PreparedWorkingDirectory, PrepareWorkingDirectoryError>> {
2505 let test_lock_dir_path = get_test_locks_dir(repo)?;
2506 std::fs::create_dir_all(&test_lock_dir_path)
2507 .wrap_err_with(|| format!("Creating test lock dir path: {test_lock_dir_path:?}"))?;
2508
2509 let lock_file_name = match strategy {
2510 TestExecutionStrategy::WorkingCopy => "working-copy.lock".to_string(),
2511 TestExecutionStrategy::Worktree => {
2512 format!("worktree-{worker_id}.lock")
2513 }
2514 };
2515 let lock_path = test_lock_dir_path.join(lock_file_name);
2516 let mut lock_file = LockFile::open(&lock_path)
2517 .wrap_err_with(|| format!("Opening working copy lock at {lock_path:?}"))?;
2518 if !lock_file
2519 .try_lock_with_pid()
2520 .wrap_err_with(|| format!("Locking working copy with {lock_path:?}"))?
2521 {
2522 return Ok(Err(PrepareWorkingDirectoryError::LockFailed(lock_path)));
2523 }
2524
2525 match strategy {
2526 TestExecutionStrategy::WorkingCopy => {
2527 let working_copy_path = match repo.get_working_copy_path() {
2528 None => return Ok(Err(PrepareWorkingDirectoryError::NoWorkingCopy)),
2529 Some(working_copy_path) => working_copy_path.to_owned(),
2530 };
2531
2532 let GitRunResult { exit_code, stdout: _, stderr: _ } =
2533 git_run_info.run_silent(
2536 repo,
2537 Some(event_tx_id),
2538 &["reset", "--hard", &commit.get_oid().to_string()],
2539 Default::default()
2540 ).context("Checking out commit to prepare working directory")?;
2541 if exit_code.is_success() {
2542 Ok(Ok(PreparedWorkingDirectory {
2543 lock_file,
2544 path: working_copy_path,
2545 }))
2546 } else {
2547 Ok(Err(PrepareWorkingDirectoryError::CheckoutFailed(
2548 commit.get_oid(),
2549 )))
2550 }
2551 }
2552
2553 TestExecutionStrategy::Worktree => {
2554 let parent_dir = get_test_worktrees_dir(repo)?;
2555 std::fs::create_dir_all(&parent_dir)
2556 .wrap_err_with(|| format!("Creating worktree parent dir at {parent_dir:?}"))?;
2557
2558 let worktree_dir_name = format!("testing-worktree-{worker_id}");
2559 let worktree_dir = parent_dir.join(worktree_dir_name);
2560 let worktree_dir_str = match worktree_dir.to_str() {
2561 Some(worktree_dir) => worktree_dir,
2562 None => {
2563 return Ok(Err(PrepareWorkingDirectoryError::CreateWorktreeFailed(
2564 worktree_dir,
2565 )));
2566 }
2567 };
2568
2569 if !worktree_dir.exists() {
2570 let GitRunResult {
2571 exit_code,
2572 stdout: _,
2573 stderr: _,
2574 } = git_run_info.run_silent(
2575 repo,
2576 Some(event_tx_id),
2577 &["worktree", "add", worktree_dir_str, "--force", "--detach"],
2578 Default::default(),
2579 )?;
2580 if !exit_code.is_success() {
2581 return Ok(Err(PrepareWorkingDirectoryError::CreateWorktreeFailed(
2582 worktree_dir,
2583 )));
2584 }
2585 }
2586
2587 let GitRunResult {
2588 exit_code,
2589 stdout: _,
2590 stderr: _,
2591 } = git_run_info.run_silent(
2592 repo,
2593 Some(event_tx_id),
2594 &[
2595 "-C",
2596 worktree_dir_str,
2597 "checkout",
2598 "--force",
2599 &commit.get_oid().to_string(),
2600 ],
2601 Default::default(),
2602 )?;
2603 if !exit_code.is_success() {
2604 return Ok(Err(PrepareWorkingDirectoryError::CheckoutFailed(
2605 commit.get_oid(),
2606 )));
2607 }
2608 Ok(Ok(PreparedWorkingDirectory {
2609 lock_file,
2610 path: worktree_dir,
2611 }))
2612 }
2613 }
2614}
2615
2616#[instrument]
2617fn test_commit(
2618 effects: &Effects,
2619 git_run_info: &GitRunInfo,
2620 repo: &Repo,
2621 event_tx_id: EventTransactionId,
2622 test_files: TestFiles,
2623 working_directory: &Path,
2624 shell_path: &Path,
2625 options: &ResolvedTestOptions,
2626 commit: &Commit,
2627) -> eyre::Result<TestOutput> {
2628 let TestFiles {
2629 temp_dir,
2630 lock_file: _lock_file, result_path,
2632 result_file,
2633 stdout_path,
2634 stdout_file,
2635 stderr_path,
2636 stderr_file,
2637 } = test_files;
2638
2639 let mut command = Command::new(shell_path);
2640 command
2641 .arg("-c")
2642 .arg(options.command.to_string())
2643 .current_dir(working_directory)
2644 .env(BRANCHLESS_TRANSACTION_ID_ENV_VAR, event_tx_id.to_string())
2645 .env("BRANCHLESS_TEST_COMMIT", commit.get_oid().to_string())
2646 .env("BRANCHLESS_TEST_COMMAND", options.command.to_string());
2647
2648 if options.is_interactive {
2649 let commit_desc = effects
2650 .get_glyphs()
2651 .render(commit.friendly_describe(effects.get_glyphs())?)?;
2652 let passed = "passed";
2653 let exit0 = effects
2654 .get_glyphs()
2655 .render(StyledString::styled("exit 0", *STYLE_SUCCESS))?;
2656 let failed = "failed";
2657 let exit1 = effects
2658 .get_glyphs()
2659 .render(StyledString::styled("exit 1", *STYLE_FAILURE))?;
2660 let skipped = "skipped";
2661 let exit125 = effects
2662 .get_glyphs()
2663 .render(StyledString::styled("exit 125", *STYLE_SKIPPED))?;
2664 let exit127 = effects
2665 .get_glyphs()
2666 .render(StyledString::styled("exit 127", *STYLE_FAILURE))?;
2667
2668 println!(
2672 "\
2673You are now at: {commit_desc}
2674To mark this commit as {passed},run: {exit0}
2675To mark this commit as {failed}, run: {exit1}
2676To mark this commit as {skipped}, run: {exit125}
2677To abort testing entirely, run: {exit127}",
2678 );
2679 match options.execution_strategy {
2680 TestExecutionStrategy::WorkingCopy => {}
2681 TestExecutionStrategy::Worktree => {
2682 let warning = effects
2683 .get_glyphs()
2684 .render(StyledString::styled(
2685 "Warning: You are in a worktree. Your changes will not be propagated between the worktree and the main repository.",
2686 *STYLE_SKIPPED
2687 ))?;
2688 println!("{warning}");
2689 println!("To save your changes, create a new branch or note the commit hash.");
2690 println!("To incorporate the changes from the main repository, switch to the main repository's current commit or branch.");
2691 }
2692 }
2693 } else {
2694 command
2695 .stdin(Stdio::null())
2696 .stdout(stdout_file)
2697 .stderr(stderr_file);
2698 }
2699
2700 let exit_code = match command.status() {
2701 Ok(status) => status.code(),
2702 Err(err) => {
2703 return Ok(TestOutput {
2704 temp_dir,
2705 result_path,
2706 stdout_path,
2707 stderr_path,
2708 test_status: TestStatus::SpawnTestFailed(err.to_string()),
2709 });
2710 }
2711 };
2712 let exit_code = match exit_code {
2713 Some(exit_code) => exit_code,
2714 None => {
2715 return Ok(TestOutput {
2716 temp_dir,
2717 result_path,
2718 stdout_path,
2719 stderr_path,
2720 test_status: TestStatus::TerminatedBySignal,
2721 });
2722 }
2723 };
2724 let test_status = match exit_code {
2725 TEST_SUCCESS_EXIT_CODE => {
2726 let fix_info = {
2727 let repo = Repo::from_dir(working_directory)?;
2728 let (head_commit_oid, snapshot) = {
2729 let index = repo.get_index()?;
2730 let head_info = repo.get_head_info()?;
2731 let (snapshot, _status) = repo.get_status(
2732 &effects.suppress(),
2733 git_run_info,
2734 &index,
2735 &head_info,
2736 Some(event_tx_id),
2737 )?;
2738 (head_info.oid, snapshot)
2739 };
2740 let snapshot_tree_oid = match snapshot.get_working_copy_changes_type()? {
2741 WorkingCopyChangesType::None | WorkingCopyChangesType::Unstaged => {
2742 let fixed_tree_oid: MaybeZeroOid = snapshot.commit_unstaged.get_tree_oid();
2743 fixed_tree_oid.into()
2744 }
2745 changes_type @ (WorkingCopyChangesType::Staged
2746 | WorkingCopyChangesType::Conflicts) => {
2747 warn!(
2749 ?changes_type,
2750 "There were staged changes or conflicts in the resulting working copy"
2751 );
2752 None
2753 }
2754 };
2755 FixInfo {
2756 head_commit_oid,
2757 snapshot_tree_oid,
2758 }
2759 };
2760 TestStatus::Passed {
2761 cached: false,
2762 fix_info,
2763 interactive: options.is_interactive,
2764 }
2765 }
2766
2767 exit_code @ TEST_INDETERMINATE_EXIT_CODE => TestStatus::Indeterminate { exit_code },
2768 exit_code @ TEST_ABORT_EXIT_CODE => TestStatus::Abort { exit_code },
2769
2770 exit_code => TestStatus::Failed {
2771 cached: false,
2772 exit_code,
2773 interactive: options.is_interactive,
2774 },
2775 };
2776
2777 let fix_info = match &test_status {
2778 TestStatus::Passed {
2779 cached: _,
2780 fix_info,
2781 interactive: _,
2782 } => Some(fix_info),
2783 TestStatus::CheckoutFailed
2784 | TestStatus::SpawnTestFailed(_)
2785 | TestStatus::TerminatedBySignal
2786 | TestStatus::AlreadyInProgress
2787 | TestStatus::ReadCacheFailed(_)
2788 | TestStatus::Failed { .. }
2789 | TestStatus::Abort { .. }
2790 | TestStatus::Indeterminate { .. } => None,
2791 };
2792 let serialized_test_result = SerializedTestResult {
2793 command: options.command.clone(),
2794 exit_code,
2795 head_commit_oid: fix_info
2796 .and_then(|fix_info| fix_info.head_commit_oid.map(SerializedNonZeroOid)),
2797 snapshot_tree_oid: fix_info
2798 .and_then(|fix_info| fix_info.snapshot_tree_oid.map(SerializedNonZeroOid)),
2799 interactive: options.is_interactive,
2800 };
2801 serde_json::to_writer_pretty(result_file, &serialized_test_result)
2802 .wrap_err_with(|| format!("Writing test status {test_status:?} to {result_path:?}"))?;
2803
2804 Ok(TestOutput {
2805 temp_dir,
2806 result_path,
2807 stdout_path,
2808 stderr_path,
2809 test_status,
2810 })
2811}
2812
2813#[instrument]
2816fn subcommand_show(
2817 effects: &Effects,
2818 options: &RawTestOptions,
2819 revset: Revset,
2820 resolve_revset_options: &ResolveRevsetOptions,
2821) -> EyreExitOr<()> {
2822 let now = SystemTime::now();
2823 let repo = Repo::from_current_dir()?;
2824 let conn = repo.get_db_conn()?;
2825 let event_log_db = EventLogDb::new(&conn)?;
2826 let event_tx_id = event_log_db.make_transaction_id(now, "test show")?;
2827 let event_replayer = EventReplayer::from_event_log_db(effects, &repo, &event_log_db)?;
2828 let event_cursor = event_replayer.make_default_cursor();
2829 let references_snapshot = repo.get_references_snapshot()?;
2830 let mut dag = Dag::open_and_sync(
2831 effects,
2832 &repo,
2833 &event_replayer,
2834 event_cursor,
2835 &references_snapshot,
2836 )?;
2837
2838 let commit_set =
2839 match resolve_commits(effects, &repo, &mut dag, &[revset], resolve_revset_options) {
2840 Ok(mut commit_sets) => commit_sets.pop().unwrap(),
2841 Err(err) => {
2842 err.describe(effects)?;
2843 return Ok(Err(ExitCode(1)));
2844 }
2845 };
2846
2847 let options = try_exit_code!(ResolvedTestOptions::resolve(
2848 now,
2849 effects,
2850 &dag,
2851 &repo,
2852 event_tx_id,
2853 &commit_set,
2854 None,
2855 options,
2856 )?);
2857
2858 let commits = sorted_commit_set(&repo, &dag, &commit_set)?;
2859 for commit in commits {
2860 let test_files = make_test_files(&repo, &commit, &options)?;
2861 match test_files {
2862 TestFilesResult::NotCached(_) => {
2863 writeln!(
2864 effects.get_output_stream(),
2865 "No cached test data for {}",
2866 effects
2867 .get_glyphs()
2868 .render(commit.friendly_describe(effects.get_glyphs())?)?
2869 )?;
2870 }
2871 TestFilesResult::Cached(test_output) => {
2872 write!(
2873 effects.get_output_stream(),
2874 "{}",
2875 effects.get_glyphs().render(test_output.describe(
2876 effects,
2877 &commit,
2878 false,
2879 options.verbosity
2880 )?)?,
2881 )?;
2882 }
2883 }
2884 }
2885
2886 if get_hint_enabled(&repo, Hint::TestShowVerbose)? {
2887 match options.verbosity {
2888 Verbosity::None => {
2889 writeln!(
2890 effects.get_output_stream(),
2891 "{}: to see more detailed output, re-run with -v/--verbose",
2892 effects.get_glyphs().render(get_hint_string())?,
2893 )?;
2894 print_hint_suppression_notice(effects, Hint::TestShowVerbose)?;
2895 }
2896 Verbosity::PartialOutput => {
2897 writeln!(
2898 effects.get_output_stream(),
2899 "{}: to see more detailed output, re-run with -vv/--verbose --verbose",
2900 effects.get_glyphs().render(get_hint_string())?,
2901 )?;
2902 print_hint_suppression_notice(effects, Hint::TestShowVerbose)?;
2903 }
2904 Verbosity::FullOutput => {}
2905 }
2906 }
2907
2908 Ok(Ok(()))
2909}
2910
2911#[instrument]
2913pub fn subcommand_clean(
2914 effects: &Effects,
2915 revset: Revset,
2916 resolve_revset_options: &ResolveRevsetOptions,
2917) -> EyreExitOr<()> {
2918 let repo = Repo::from_current_dir()?;
2919 let conn = repo.get_db_conn()?;
2920 let event_log_db = EventLogDb::new(&conn)?;
2921 let event_replayer = EventReplayer::from_event_log_db(effects, &repo, &event_log_db)?;
2922 let event_cursor = event_replayer.make_default_cursor();
2923 let references_snapshot = repo.get_references_snapshot()?;
2924 let mut dag = Dag::open_and_sync(
2925 effects,
2926 &repo,
2927 &event_replayer,
2928 event_cursor,
2929 &references_snapshot,
2930 )?;
2931
2932 let commit_set =
2933 match resolve_commits(effects, &repo, &mut dag, &[revset], resolve_revset_options) {
2934 Ok(mut commit_sets) => commit_sets.pop().unwrap(),
2935 Err(err) => {
2936 err.describe(effects)?;
2937 return Ok(Err(ExitCode(1)));
2938 }
2939 };
2940
2941 let mut num_cleaned_commits = 0;
2942 for commit in sorted_commit_set(&repo, &dag, &commit_set)? {
2943 let tree_dir = get_test_tree_dir(&repo, &commit)?;
2944 if tree_dir.exists() {
2945 writeln!(
2946 effects.get_output_stream(),
2947 "Cleaning results for {}",
2948 effects
2949 .get_glyphs()
2950 .render(commit.friendly_describe(effects.get_glyphs())?)?,
2951 )?;
2952 std::fs::remove_dir_all(&tree_dir)
2953 .with_context(|| format!("Cleaning test dir: {tree_dir:?}"))?;
2954 num_cleaned_commits += 1;
2955 } else {
2956 writeln!(
2957 effects.get_output_stream(),
2958 "Nothing to clean for {}",
2959 effects
2960 .get_glyphs()
2961 .render(commit.friendly_describe(effects.get_glyphs())?)?,
2962 )?;
2963 }
2964 }
2965 writeln!(
2966 effects.get_output_stream(),
2967 "Cleaned {}.",
2968 Pluralize {
2969 determiner: None,
2970 amount: num_cleaned_commits,
2971 unit: ("cached test result", "cached test results")
2972 }
2973 )?;
2974 Ok(Ok(()))
2975}
2976
2977#[cfg(test)]
2978mod tests {
2979 use lib::testing::make_git;
2980
2981 use super::*;
2982
2983 #[test]
2984 fn test_lock_prepared_working_directory() -> eyre::Result<()> {
2985 let git = make_git()?;
2986 git.init_repo()?;
2987
2988 let git_run_info = git.get_git_run_info();
2989 let repo = git.get_repo()?;
2990 let conn = repo.get_db_conn()?;
2991 let event_log_db = EventLogDb::new(&conn)?;
2992 let event_tx_id = event_log_db.make_transaction_id(SystemTime::now(), "test")?;
2993 let head_oid = repo.get_head_info()?.oid.unwrap();
2994 let head_commit = repo.find_commit_or_fail(head_oid)?;
2995 let worker_id = 1;
2996
2997 let _prepared_working_copy = prepare_working_directory(
2998 &git_run_info,
2999 &repo,
3000 event_tx_id,
3001 &head_commit,
3002 TestExecutionStrategy::WorkingCopy,
3003 worker_id,
3004 )?
3005 .unwrap();
3006 assert!(matches!(
3007 prepare_working_directory(
3008 &git_run_info,
3009 &repo,
3010 event_tx_id,
3011 &head_commit,
3012 TestExecutionStrategy::WorkingCopy,
3013 worker_id
3014 )?,
3015 Err(PrepareWorkingDirectoryError::LockFailed(_))
3016 ));
3017
3018 let _prepared_worktree = prepare_working_directory(
3019 &git_run_info,
3020 &repo,
3021 event_tx_id,
3022 &head_commit,
3023 TestExecutionStrategy::Worktree,
3024 worker_id,
3025 )?
3026 .unwrap();
3027 assert!(matches!(
3028 prepare_working_directory(
3029 &git_run_info,
3030 &repo,
3031 event_tx_id,
3032 &head_commit,
3033 TestExecutionStrategy::Worktree,
3034 worker_id
3035 )?,
3036 Err(PrepareWorkingDirectoryError::LockFailed(_))
3037 ));
3038
3039 Ok(())
3040 }
3041}