Skip to main content

supercov_engine/
rust_compiler_test_runner.rs

1//! Private execution and attribution for compiler-instrumented Rust artifacts.
2//!
3//! The compiler frontend freezes the complete denominator before this module
4//! launches anything. Ordinary Cargo artifacts run once under the selected
5//! toolchain's exact libtest companion and share one authenticated mmap whose
6//! dynamic contexts are partitioned by test. Nextest attempts and opaque
7//! custom harnesses retain their intrinsic process boundary. Context-zero
8//! records are always published as invocation background, never as test work.
9
10use std::{
11    collections::{BTreeMap, BTreeSet},
12    ffi::OsString,
13    fs::{self, OpenOptions},
14    io::{self, Write},
15    path::{Component, Path, PathBuf},
16    sync::{
17        Mutex,
18        atomic::{AtomicUsize, Ordering},
19    },
20    time::{SystemTime, UNIX_EPOCH},
21};
22
23use nextest_metadata::{BuildPlatform, FilterMatch, NextestExitCode, RustTestSuiteStatusSummary};
24use serde::{Deserialize, Serialize};
25use sha2::{Digest, Sha256};
26use supercov_contracts::{
27    AttributionPrecision, ExecutionModel, FrontendAttribution, FrontendLimitation,
28    FrontendLimitationScope, FrontendRunDeclaration, FrontendRunnerDeclaration,
29    LANGUAGE_FRONTEND_PROTOCOL_VERSION, StructuralSource,
30};
31
32use crate::{
33    coverage_report::{
34        CoverageModelDeclaration, CoveragePhase, CoverageReportRequest, ExecutionScope,
35        ExitCodeInput, PersistedCoverageModel, RawTestResult, RuntimeSnapshot, TestProvenance,
36    },
37    evidence_archive::EvidenceArchiveEntry,
38    process_supervision::{
39        CommandSpec, ProcessSupervisor, SupervisedOutput, SupervisedResult, SupervisionOptions,
40    },
41    rust_cargo_configuration::{
42        RustCargoResolvedRunner, RustCargoResolvedTargetRunner, RustCargoRunnerPlan,
43    },
44    rust_compiler_ctfe::RustCompilerCtfeUnit,
45    rust_compiler_evidence::{
46        RustCompilerEvidenceProjection, RustCompilerTransportHealth, project_rust_compiler_evidence,
47    },
48    rust_compiler_manifest::NormalizedRustCompilerManifest,
49    rust_compiler_orchestration::{
50        RustCompilerBuild, RustCompilerBuildRequest, RustCompilerTestArtifact,
51    },
52    rust_doctest::{RustdocJoinedOutcomeState, RustdocOutcomeResolution, RustdocOutcomeStatus},
53    rust_libtest_events::{
54        RUST_LIBTEST_EVENTS_ENV, RUST_LIBTEST_TOKEN_ENV, RustLibtestEvent, RustLibtestRunEvents,
55        RustLibtestTerminalResult, create_rust_libtest_event_file, read_rust_libtest_events,
56        validate_rust_libtest_run_events,
57    },
58    rust_probe_transport::{
59        DEFAULT_DESCRIPTOR_CAPACITY, DEFAULT_PAYLOAD_CAPACITY, RUST_CONTEXT_ENV,
60        RUST_TRANSPORT_ENV, RUST_TRANSPORT_TOKEN_ENV, RustTransportPartition, RustTransportRead,
61        create_rust_transport, partition_rust_transport_by_test_contexts, read_rust_transport,
62    },
63    rust_runner_attempt::{
64        NextestAttemptIdentity, RustRunnerInvocationIdentity, classify_rust_runner_environment,
65    },
66    rust_test_context::preflight_rust_test_contexts,
67    rust_test_runner::rust_libtest_selection,
68};
69
70const TOKEN_BYTES: usize = supercov_contracts::RUST_PROBE_TRANSPORT_TOKEN_SIZE;
71pub const RUST_CARGO_RUNNER_CONFIG_ENV: &str = "SUPERCOV_RUST_CARGO_RUNNER_CONFIG";
72pub const RUST_CARGO_RUNNER_VERSION: u32 = 6;
73
74#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
75#[serde(rename_all = "kebab-case")]
76pub enum RustCargoRunnerKind {
77    CargoTest,
78    CargoCustomHarness,
79    Nextest,
80}
81
82#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
83#[serde(rename_all = "camelCase", deny_unknown_fields)]
84pub struct RustCargoRunnerArtifact {
85    pub executable: PathBuf,
86    pub test_harness: bool,
87}
88
89#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
90#[serde(rename_all = "camelCase", deny_unknown_fields)]
91pub struct RustCargoRunnerConfig {
92    pub version: u32,
93    pub run_id: String,
94    pub target_directory: PathBuf,
95    pub output_directory: PathBuf,
96    pub target_runners: Vec<RustCargoResolvedTargetRunner>,
97    pub artifacts: Vec<RustCargoRunnerArtifact>,
98}
99
100#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
101#[serde(rename_all = "camelCase", deny_unknown_fields)]
102pub struct RustCargoRunnerAttempt {
103    pub test: String,
104    pub context_id: u64,
105    pub retry: usize,
106    pub total_attempts: usize,
107    pub runner_attempt_id: String,
108    pub outcome: RustCargoRunnerAttemptOutcome,
109    pub transport: RustTransportRead,
110}
111
112#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
113#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)]
114pub enum RustCargoRunnerAttemptOutcome {
115    Libtest {
116        result: RustLibtestTerminalResult,
117        timed_out: bool,
118    },
119    Unstarted,
120    OpaqueProcess,
121}
122
123fn attempt_outcome_succeeded(outcome: &RustCargoRunnerAttemptOutcome) -> bool {
124    matches!(
125        outcome,
126        RustCargoRunnerAttemptOutcome::Libtest {
127            result: RustLibtestTerminalResult::Passed
128                | RustLibtestTerminalResult::Ignored
129                | RustLibtestTerminalResult::Benchmarked,
130            ..
131        }
132    )
133}
134
135#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
136#[serde(rename_all = "camelCase", deny_unknown_fields)]
137pub struct RustCargoRunnerInvocation {
138    pub result: SupervisedResult,
139    pub started_at_ms: i64,
140    pub ended_at_ms: i64,
141    pub stdout: Vec<u8>,
142    pub stderr: Vec<u8>,
143    pub background_transport: RustTransportRead,
144}
145
146#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
147#[serde(rename_all = "camelCase", deny_unknown_fields)]
148pub struct RustCargoRunnerUnit {
149    pub version: u32,
150    pub run_id: String,
151    pub invocation_ordinal: u64,
152    pub runner: RustCargoRunnerKind,
153    #[serde(skip_serializing_if = "Option::is_none")]
154    pub runner_run_id: Option<String>,
155    #[serde(skip_serializing_if = "Option::is_none")]
156    pub runner_version: Option<String>,
157    #[serde(skip_serializing_if = "Option::is_none")]
158    pub runner_binary_id: Option<String>,
159    pub target: String,
160    pub artifact: PathBuf,
161    pub arguments: Vec<String>,
162    pub invocation: RustCargoRunnerInvocation,
163    pub attempts: Vec<RustCargoRunnerAttempt>,
164    /// Deterministic join-bounded quarantine notes: one per thread phase whose
165    /// lifetime escaped its creating test in this invocation's transport.
166    pub thread_scope_limitations: BTreeSet<String>,
167}
168
169fn validate_persisted_runner_transport(
170    unit: &RustCargoRunnerUnit,
171) -> Result<(), RustCompilerTestError> {
172    let mut roots = BTreeSet::new();
173    let mut combined = unit.invocation.background_transport.clone();
174    for attempt in &unit.attempts {
175        if matches!(attempt.context_id, 0 | u64::MAX) || !roots.insert(attempt.context_id) {
176            return Err(RustCompilerTestError::Context(format!(
177                "Cargo runner unit {} has a reserved or duplicate test context",
178                unit.invocation_ordinal
179            )));
180        }
181        combined
182            .observations
183            .extend(attempt.transport.observations.iter().cloned());
184        combined
185            .ordinal_hits
186            .extend(attempt.transport.ordinal_hits.iter().copied());
187        combined
188            .phases
189            .extend(attempt.transport.phases.iter().cloned());
190        combined
191            .thread_phases
192            .extend(attempt.transport.thread_phases.iter().copied());
193        combined
194            .thread_ends
195            .extend(attempt.transport.thread_ends.iter().copied());
196        combined
197            .test_boundaries
198            .extend(attempt.transport.test_boundaries.iter().copied());
199        combined.committed = combined
200            .committed
201            .checked_add(attempt.transport.committed)
202            .ok_or_else(|| {
203                RustCompilerTestError::Context(
204                    "Cargo runner committed transport count overflowed u64".into(),
205                )
206            })?;
207        combined.incomplete = combined
208            .incomplete
209            .checked_add(attempt.transport.incomplete)
210            .ok_or_else(|| {
211                RustCompilerTestError::Context(
212                    "Cargo runner incomplete transport count overflowed u64".into(),
213                )
214            })?;
215        combined.dropped = combined
216            .dropped
217            .checked_add(attempt.transport.dropped)
218            .ok_or_else(|| {
219                RustCompilerTestError::Context(
220                    "Cargo runner dropped transport count overflowed u64".into(),
221                )
222            })?;
223        combined.attachments = combined
224            .attachments
225            .checked_add(attempt.transport.attachments)
226            .ok_or_else(|| {
227                RustCompilerTestError::Context(
228                    "Cargo runner transport attachment count overflowed u64".into(),
229                )
230            })?;
231    }
232    let repartitioned =
233        partition_rust_transport_by_test_contexts(&combined, &roots).map_err(|error| {
234            RustCompilerTestError::Context(format!(
235                "Cargo runner unit {} has invalid persisted attribution: {error}",
236                unit.invocation_ordinal
237            ))
238        })?;
239    if repartitioned.background != unit.invocation.background_transport
240        || repartitioned.thread_scope_limitations != unit.thread_scope_limitations
241        || unit.attempts.iter().any(|attempt| {
242            repartitioned.attributed.get(&attempt.context_id) != Some(&attempt.transport)
243        })
244    {
245        return Err(RustCompilerTestError::Context(format!(
246            "Cargo runner unit {} does not preserve its exact transport partition",
247            unit.invocation_ordinal
248        )));
249    }
250    Ok(())
251}
252
253#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
254#[serde(rename_all = "camelCase", deny_unknown_fields)]
255struct RustCargoRunnerFailure {
256    version: u32,
257    run_id: String,
258    invocation_ordinal: u64,
259    target: Option<String>,
260    artifact: Option<PathBuf>,
261    error: String,
262}
263
264#[derive(Debug, Clone, PartialEq, Eq)]
265pub struct RustCargoRunnerExecution {
266    pub exit_code: i32,
267    pub unit_path: Option<PathBuf>,
268}
269
270#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
271#[serde(rename_all = "camelCase", deny_unknown_fields)]
272pub struct RustCompilerRunRequest {
273    pub project_root: PathBuf,
274    pub command: Vec<String>,
275    pub run_id: String,
276    pub generated_at: String,
277    pub wrapper_path: PathBuf,
278    pub companion_candidates: Vec<PathBuf>,
279    pub require_public_capabilities: bool,
280    pub cargo_runner_plan: RustCargoRunnerPlan,
281    pub watchdog_program: Option<PathBuf>,
282}
283
284impl RustCompilerRunRequest {
285    fn build_request(&self) -> RustCompilerBuildRequest {
286        RustCompilerBuildRequest {
287            project_root: self.project_root.clone(),
288            command: self.command.clone(),
289            run_id: self.run_id.clone(),
290            wrapper_path: self.wrapper_path.clone(),
291            companion_candidates: self.companion_candidates.clone(),
292            require_public_capabilities: self.require_public_capabilities,
293            cargo_runner_plan: self.cargo_runner_plan.clone(),
294        }
295    }
296}
297
298#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
299#[serde(rename_all = "camelCase")]
300pub struct RustCompilerTransportHealthRecord {
301    pub scope_id: String,
302    pub scope_kind: String,
303    pub status: String,
304    pub transport: RustCompilerTransportHealth,
305    /// Join-bounded thread quarantine notes for this scope's transport. Work
306    /// under an escaped thread phase is deterministic background evidence.
307    #[serde(skip_serializing_if = "BTreeSet::is_empty")]
308    pub thread_scope_limitations: BTreeSet<String>,
309}
310
311#[derive(Debug, Clone, PartialEq)]
312pub struct RustCompilerFrontendRun {
313    pub selection: crate::rust_compiler_selection::SelectedRustCompilerCompanion,
314    pub declaration: FrontendRunDeclaration,
315    pub request: CoverageReportRequest,
316    pub exit_code: i32,
317    pub artifacts: usize,
318    pub artifact_files: Vec<PathBuf>,
319    pub transport_health: Vec<RustCompilerTransportHealthRecord>,
320    pub build_ms: f64,
321    pub execution_ms: f64,
322}
323
324impl RustCompilerFrontendRun {
325    pub fn archive_entries(&self) -> Result<Vec<EvidenceArchiveEntry>, serde_json::Error> {
326        let model = PersistedCoverageModel::from_declaration(
327            self.request
328                .coverage_model
329                .as_ref()
330                .expect("Rust compiler frontend always declares a coverage model"),
331        )
332        .expect("Rust compiler coverage model is contract-valid");
333        let mut entries = vec![
334            EvidenceArchiveEntry {
335                path: "coverage-model.json".into(),
336                contents: serde_json::to_vec(&model)?,
337            },
338            EvidenceArchiveEntry {
339                path: "frontend.json".into(),
340                contents: serde_json::to_vec(&self.declaration)?,
341            },
342            EvidenceArchiveEntry {
343                path: "manifest.json".into(),
344                contents: serde_json::to_vec(&self.request.manifest)?,
345            },
346        ];
347        for (index, result) in self.request.raw_results.iter().enumerate() {
348            entries.push(EvidenceArchiveEntry {
349                path: format!("results/{index:08}/mcdc.json"),
350                contents: serde_json::to_vec(result)?,
351            });
352        }
353        entries.push(EvidenceArchiveEntry {
354            path: "rust/transport-health.json".into(),
355            contents: serde_json::to_vec(&self.transport_health)?,
356        });
357        Ok(entries)
358    }
359}
360
361#[derive(Debug)]
362pub enum RustCompilerTestError {
363    Build(String),
364    Io { path: PathBuf, reason: String },
365    UnsafeArtifact(String),
366    List { artifact: PathBuf, reason: String },
367    Context(String),
368    DuplicateTest(String),
369    Random(String),
370    Launch { test: String, reason: String },
371    Transport { test: String, reason: String },
372    DroppedEvidence { test: String, dropped: u64 },
373    Projection { test: String, reason: String },
374    UnsupportedCommand(String),
375    UnverifiedExecution { code: i32, reason: String },
376    Interrupted { code: i32, signal: String },
377}
378
379impl std::fmt::Display for RustCompilerTestError {
380    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
381        match self {
382            Self::Build(reason) => write!(formatter, "Rust compiler build failed: {reason}"),
383            Self::Io { path, reason } => write!(formatter, "{}: {reason}", path.display()),
384            Self::UnsafeArtifact(reason) => {
385                write!(formatter, "unsafe Rust test artifact: {reason}")
386            }
387            Self::List { artifact, reason } => write!(
388                formatter,
389                "could not enumerate tests in {}: {reason}",
390                artifact.display()
391            ),
392            Self::Context(reason) => write!(formatter, "invalid Rust test context: {reason}"),
393            Self::DuplicateTest(test) => write!(formatter, "duplicate Rust test identity: {test}"),
394            Self::Random(reason) => {
395                write!(formatter, "could not authenticate Rust evidence: {reason}")
396            }
397            Self::Launch { test, reason } => {
398                write!(formatter, "could not launch Rust test {test}: {reason}")
399            }
400            Self::Transport { test, reason } => {
401                write!(formatter, "invalid Rust transport for {test}: {reason}")
402            }
403            Self::DroppedEvidence { test, dropped } => write!(
404                formatter,
405                "Rust transport dropped {dropped} record(s) for {test}; refusing partial coverage"
406            ),
407            Self::Projection { test, reason } => {
408                write!(formatter, "invalid Rust evidence for {test}: {reason}")
409            }
410            Self::UnsupportedCommand(reason) => formatter.write_str(reason),
411            Self::UnverifiedExecution { code, reason } => write!(
412                formatter,
413                "Rust test command exited {code}, but Supercov could not authenticate complete coverage evidence: {reason}"
414            ),
415            Self::Interrupted { signal, .. } => {
416                write!(formatter, "Rust test run was interrupted by {signal}")
417            }
418        }
419    }
420}
421
422impl std::error::Error for RustCompilerTestError {}
423
424fn io_error(path: &Path, error: impl std::fmt::Display) -> RustCompilerTestError {
425    RustCompilerTestError::Io {
426        path: path.to_path_buf(),
427        reason: error.to_string(),
428    }
429}
430
431#[derive(Debug, Clone)]
432struct TestArtifact {
433    executable: PathBuf,
434    runner_argument: Option<OsString>,
435    package: String,
436    target_key: String,
437    kind: String,
438    source: String,
439    test_harness: bool,
440}
441
442#[derive(Debug, Clone)]
443struct ProcessTask {
444    ordinal: usize,
445    artifact_index: usize,
446    artifact: TestArtifact,
447    test: String,
448    test_id: String,
449    context_id: u64,
450    retry: usize,
451    total_attempts: usize,
452    runner_attempt_id: String,
453    runner: RustCargoRunnerKind,
454    transport: PathBuf,
455    libtest_events: PathBuf,
456    test_arguments: Vec<OsString>,
457    underlying_runner: Option<RustCargoResolvedRunner>,
458}
459
460#[derive(Debug)]
461struct ProcessOutcome {
462    task: ProcessTask,
463    output: SupervisedOutput,
464    read: RustTransportRead,
465    attempt_outcome: RustCargoRunnerAttemptOutcome,
466    started_at_ms: i64,
467    ended_at_ms: i64,
468}
469
470struct StockLibtestExecution {
471    output: SupervisedOutput,
472    events: RustLibtestRunEvents,
473    partition: RustTransportPartition,
474    started_at_ms: i64,
475    ended_at_ms: i64,
476}
477
478struct RemoveFileOnDrop(Option<PathBuf>);
479
480impl Drop for RemoveFileOnDrop {
481    fn drop(&mut self) {
482        if let Some(path) = self.0.take() {
483            let _ = fs::remove_file(path);
484        }
485    }
486}
487
488fn stock_libtest_transport_reason(
489    reason: impl std::fmt::Display,
490    output: &SupervisedOutput,
491) -> String {
492    const LIMIT: usize = 16 * 1024;
493    fn tail(bytes: &[u8]) -> String {
494        let start = bytes.len().saturating_sub(LIMIT);
495        String::from_utf8_lossy(&bytes[start..]).into_owned()
496    }
497
498    let mut message = format!(
499        "{reason}; stock libtest process exit={} ",
500        output.result.exit_code()
501    );
502    if !output.stdout.is_empty() {
503        message.push_str("\nstdout tail:\n");
504        message.push_str(&tail(&output.stdout));
505    }
506    if !output.stderr.is_empty() {
507        message.push_str("\nstderr tail:\n");
508        message.push_str(&tail(&output.stderr));
509    }
510    message
511}
512
513fn epoch_ms() -> Result<i64, RustCompilerTestError> {
514    let millis = SystemTime::now()
515        .duration_since(UNIX_EPOCH)
516        .map_err(|error| RustCompilerTestError::Random(error.to_string()))?
517        .as_millis();
518    i64::try_from(millis).map_err(|error| RustCompilerTestError::Random(error.to_string()))
519}
520
521fn relative_source(root: &Path, source: &Path) -> Result<String, RustCompilerTestError> {
522    let source = fs::canonicalize(source).map_err(|error| io_error(source, error))?;
523    let relative = source
524        .strip_prefix(root)
525        .map_err(|_| RustCompilerTestError::UnsafeArtifact(source.display().to_string()))?;
526    if relative.as_os_str().is_empty()
527        || relative
528            .components()
529            .any(|component| !matches!(component, Component::Normal(_)))
530    {
531        return Err(RustCompilerTestError::UnsafeArtifact(
532            source.display().to_string(),
533        ));
534    }
535    Ok(relative.to_string_lossy().replace('\\', "/"))
536}
537
538fn normalize_artifacts(
539    project_root: &Path,
540    target_directory: &Path,
541    artifacts: &[RustCompilerTestArtifact],
542) -> Result<Vec<TestArtifact>, RustCompilerTestError> {
543    let target_directory =
544        fs::canonicalize(target_directory).map_err(|error| io_error(target_directory, error))?;
545    artifacts
546        .iter()
547        .map(|artifact| {
548            let executable = fs::canonicalize(&artifact.executable)
549                .map_err(|error| io_error(&artifact.executable, error))?;
550            if !executable.starts_with(&target_directory) {
551                return Err(RustCompilerTestError::UnsafeArtifact(
552                    executable.display().to_string(),
553                ));
554            }
555            let mut target_kinds = artifact.target_kinds.clone();
556            target_kinds.sort();
557            target_kinds.dedup();
558            Ok(TestArtifact {
559                executable,
560                runner_argument: None,
561                package: artifact.package.clone(),
562                target_key: format!("{}:{}", target_kinds.join("+"), artifact.target_name),
563                kind: if artifact.target_kinds.iter().any(|kind| kind == "test") {
564                    "integration".into()
565                } else {
566                    "unit".into()
567                },
568                source: relative_source(project_root, &artifact.source_path)?,
569                test_harness: artifact.test_harness,
570            })
571        })
572        .collect()
573}
574
575fn libtest_id(compilation_target: &str, artifact: &TestArtifact, test: &str) -> String {
576    format!(
577        "rust:libtest:{compilation_target}:{}:{}:{}::{test}",
578        artifact.package, artifact.target_key, artifact.source,
579    )
580}
581
582fn custom_harness_id(compilation_target: &str, artifact: &TestArtifact) -> String {
583    format!(
584        "rust:custom-harness:{compilation_target}:{}:{}:{}",
585        artifact.package, artifact.target_key, artifact.source,
586    )
587}
588
589fn list_tests(
590    project_root: &Path,
591    artifact: &TestArtifact,
592    selection_arguments: &[String],
593    underlying_runner: Option<&RustCargoResolvedRunner>,
594    supervisor: &ProcessSupervisor,
595    options: SupervisionOptions,
596    event_path: &Path,
597) -> Result<Vec<String>, RustCompilerTestError> {
598    let mut test_arguments = selection_arguments
599        .iter()
600        .map(OsString::from)
601        .collect::<Vec<_>>();
602    test_arguments.extend(["--list".into(), "--format".into(), "terse".into()]);
603    let (program, arguments) = artifact_command(artifact, underlying_runner, test_arguments);
604    let mut event_token = [0_u8; supercov_contracts::RUST_LIBTEST_EVENT_TOKEN_SIZE];
605    getrandom::fill(&mut event_token).map_err(|error| {
606        RustCompilerTestError::Random(format!("libtest listing token: {error}"))
607    })?;
608    create_rust_libtest_event_file(event_path, event_token).map_err(|error| {
609        RustCompilerTestError::List {
610            artifact: artifact.executable.clone(),
611            reason: error.to_string(),
612        }
613    })?;
614    let mut event_cleanup = RemoveFileOnDrop(Some(event_path.to_owned()));
615    let output = supervisor
616        .supervise_captured(
617            &CommandSpec {
618                program,
619                arguments,
620                cwd: project_root.to_owned(),
621                environment: Some(inherited_environment([
622                    (
623                        OsString::from(RUST_LIBTEST_EVENTS_ENV),
624                        event_path.as_os_str().to_owned(),
625                    ),
626                    (
627                        OsString::from(RUST_LIBTEST_TOKEN_ENV),
628                        OsString::from(token_hex(&event_token)),
629                    ),
630                ])),
631                captured_output: None,
632            },
633            options,
634            &mut io::sink(),
635        )
636        .map_err(|error| RustCompilerTestError::List {
637            artifact: artifact.executable.clone(),
638            reason: error.to_string(),
639        })?;
640    if output.result.exit_code() != 0 {
641        return Err(RustCompilerTestError::List {
642            artifact: artifact.executable.clone(),
643            reason: format!(
644                "{}{}",
645                String::from_utf8_lossy(&output.stderr),
646                String::from_utf8_lossy(&output.stdout)
647            )
648            .trim()
649            .to_owned(),
650        });
651    }
652    let mut tests = String::from_utf8_lossy(&output.stdout)
653        .lines()
654        .filter_map(|line| {
655            line.strip_suffix(": test")
656                .or_else(|| line.strip_suffix(": benchmark"))
657        })
658        .map(str::to_owned)
659        .collect::<Vec<_>>();
660    tests.sort();
661    tests.dedup();
662    let events = read_rust_libtest_events(event_path, &event_token).map_err(|error| {
663        RustCompilerTestError::List {
664            artifact: artifact.executable.clone(),
665            reason: error.to_string(),
666        }
667    })?;
668    if !matches!(
669        events.as_slice(),
670        [RustLibtestEvent::FilteredOut { .. }, RustLibtestEvent::Filtered { count, .. }]
671            if *count == tests.len() as u64
672    ) {
673        return Err(RustCompilerTestError::List {
674            artifact: artifact.executable.clone(),
675            reason: "authenticated libtest listing events disagree with terse output".into(),
676        });
677    }
678    fs::remove_file(event_path).map_err(|error| io_error(event_path, error))?;
679    event_cleanup.0 = None;
680    Ok(tests)
681}
682
683fn artifact_command(
684    artifact: &TestArtifact,
685    underlying_runner: Option<&RustCargoResolvedRunner>,
686    test_arguments: Vec<OsString>,
687) -> (OsString, Vec<OsString>) {
688    match underlying_runner {
689        Some(runner) => {
690            let mut arguments = runner
691                .arguments
692                .iter()
693                .map(OsString::from)
694                .collect::<Vec<_>>();
695            arguments.push(
696                artifact
697                    .runner_argument
698                    .clone()
699                    .unwrap_or_else(|| artifact.executable.clone().into_os_string()),
700            );
701            arguments.extend(test_arguments);
702            (runner.program.clone().into_os_string(), arguments)
703        }
704        None => (artifact.executable.clone().into_os_string(), test_arguments),
705    }
706}
707
708fn token_hex<const N: usize>(token: &[u8; N]) -> String {
709    token.iter().map(|byte| format!("{byte:02x}")).collect()
710}
711
712fn phase_id(run_id: &str, attempt_id: &str) -> String {
713    let mut digest = Sha256::new();
714    digest.update((run_id.len() as u64).to_be_bytes());
715    digest.update(run_id.as_bytes());
716    digest.update((attempt_id.len() as u64).to_be_bytes());
717    digest.update(attempt_id.as_bytes());
718    let hex = format!("{:x}", digest.finalize());
719    format!("rust-test:{}", &hex[..40])
720}
721
722fn snapshot_has_evidence(snapshot: &RuntimeSnapshot) -> bool {
723    !snapshot.hits.is_empty() || !snapshot.decisions.is_empty() || !snapshot.events.is_empty()
724}
725
726fn inherited_environment(
727    overrides: impl IntoIterator<Item = (OsString, OsString)>,
728) -> Vec<(OsString, OsString)> {
729    let mut environment = std::env::vars_os().collect::<BTreeMap<_, _>>();
730    environment.extend(overrides);
731    environment.into_iter().collect()
732}
733
734fn run_process(
735    project_root: &Path,
736    task: &ProcessTask,
737    supervisor: &ProcessSupervisor,
738    options: SupervisionOptions,
739) -> Result<ProcessOutcome, String> {
740    let mut token = [0_u8; TOKEN_BYTES];
741    getrandom::fill(&mut token).map_err(|error| error.to_string())?;
742    create_rust_transport(
743        &task.transport,
744        token,
745        DEFAULT_DESCRIPTOR_CAPACITY,
746        DEFAULT_PAYLOAD_CAPACITY,
747    )
748    .map_err(|error| error.to_string())?;
749    let mut transport_cleanup = RemoveFileOnDrop(Some(task.transport.clone()));
750    let event_token = if task.runner == RustCargoRunnerKind::CargoCustomHarness {
751        None
752    } else {
753        let mut token = [0_u8; supercov_contracts::RUST_LIBTEST_EVENT_TOKEN_SIZE];
754        getrandom::fill(&mut token).map_err(|error| error.to_string())?;
755        create_rust_libtest_event_file(&task.libtest_events, token)
756            .map_err(|error| error.to_string())?;
757        Some(token)
758    };
759    let mut event_cleanup =
760        RemoveFileOnDrop(event_token.is_some().then(|| task.libtest_events.clone()));
761    let started_at_ms = epoch_ms().map_err(|error| error.to_string())?;
762    let (program, arguments) = artifact_command(
763        &task.artifact,
764        task.underlying_runner.as_ref(),
765        task.test_arguments.clone(),
766    );
767    let mut environment = vec![
768        (
769            OsString::from(RUST_TRANSPORT_ENV),
770            task.transport.clone().into_os_string(),
771        ),
772        (
773            OsString::from(RUST_TRANSPORT_TOKEN_ENV),
774            OsString::from(token_hex(&token)),
775        ),
776        (
777            OsString::from(RUST_CONTEXT_ENV),
778            OsString::from(format!("{:016x}", task.context_id)),
779        ),
780    ];
781    if let Some(event_token) = &event_token {
782        environment.extend([
783            (
784                OsString::from(RUST_LIBTEST_EVENTS_ENV),
785                task.libtest_events.clone().into_os_string(),
786            ),
787            (
788                OsString::from(RUST_LIBTEST_TOKEN_ENV),
789                OsString::from(token_hex(event_token)),
790            ),
791        ]);
792    }
793    let output = supervisor
794        .supervise_captured(
795            &CommandSpec {
796                program,
797                arguments,
798                cwd: project_root.to_owned(),
799                environment: Some(inherited_environment(environment)),
800                captured_output: None,
801            },
802            options,
803            &mut io::sink(),
804        )
805        .map_err(|error| error.to_string())?;
806    let ended_at_ms = epoch_ms().map_err(|error| error.to_string())?;
807    let read = read_rust_transport(&task.transport, &token).map_err(|error| error.to_string())?;
808    let attempt_outcome = if let Some(event_token) = &event_token {
809        let events = read_rust_libtest_events(&task.libtest_events, event_token)
810            .map_err(|error| error.to_string())?;
811        let joined = validate_rust_libtest_run_events(&events, [task.test.clone()])
812            .map_err(|error| error.to_string())?;
813        let [attempt] = joined.attempts.as_slice() else {
814            return Err("exact libtest attempt did not produce one terminal event".into());
815        };
816        if !joined.unstarted.is_empty()
817            || (matches!(
818                attempt.result,
819                RustLibtestTerminalResult::Passed
820                    | RustLibtestTerminalResult::Ignored
821                    | RustLibtestTerminalResult::Benchmarked
822            ) != (output.result.exit_code() == 0))
823        {
824            return Err("exact libtest terminal event disagrees with process status".into());
825        }
826        RustCargoRunnerAttemptOutcome::Libtest {
827            result: attempt.result,
828            timed_out: attempt.timed_out,
829        }
830    } else {
831        RustCargoRunnerAttemptOutcome::OpaqueProcess
832    };
833    fs::remove_file(&task.transport).map_err(|error| error.to_string())?;
834    transport_cleanup.0 = None;
835    if event_token.is_some() {
836        fs::remove_file(&task.libtest_events).map_err(|error| error.to_string())?;
837        event_cleanup.0 = None;
838    }
839    Ok(ProcessOutcome {
840        task: task.clone(),
841        output,
842        read,
843        attempt_outcome,
844        started_at_ms,
845        ended_at_ms,
846    })
847}
848
849#[allow(clippy::too_many_arguments)]
850fn run_stock_libtest_artifact(
851    project_root: &Path,
852    artifact: &TestArtifact,
853    underlying_runner: Option<&RustCargoResolvedRunner>,
854    arguments: Vec<OsString>,
855    selected_tests: &[String],
856    contexts: &BTreeMap<String, u64>,
857    transport_path: &Path,
858    event_path: &Path,
859    supervisor: &ProcessSupervisor,
860    options: SupervisionOptions,
861) -> Result<StockLibtestExecution, RustCompilerTestError> {
862    let mut token = [0_u8; TOKEN_BYTES];
863    getrandom::fill(&mut token)
864        .map_err(|error| RustCompilerTestError::Random(error.to_string()))?;
865    create_rust_transport(
866        transport_path,
867        token,
868        DEFAULT_DESCRIPTOR_CAPACITY,
869        DEFAULT_PAYLOAD_CAPACITY,
870    )
871    .map_err(|error| RustCompilerTestError::Transport {
872        test: artifact.target_key.clone(),
873        reason: error.to_string(),
874    })?;
875    let mut transport_cleanup = RemoveFileOnDrop(Some(transport_path.to_owned()));
876    let mut event_token = [0_u8; supercov_contracts::RUST_LIBTEST_EVENT_TOKEN_SIZE];
877    getrandom::fill(&mut event_token)
878        .map_err(|error| RustCompilerTestError::Random(error.to_string()))?;
879    create_rust_libtest_event_file(event_path, event_token).map_err(|error| {
880        RustCompilerTestError::Transport {
881            test: artifact.target_key.clone(),
882            reason: error.to_string(),
883        }
884    })?;
885    let mut event_cleanup = RemoveFileOnDrop(Some(event_path.to_owned()));
886    let (program, arguments) = artifact_command(artifact, underlying_runner, arguments);
887    let started_at_ms = epoch_ms()?;
888    let output = supervisor
889        .supervise_captured(
890            &CommandSpec {
891                program,
892                arguments,
893                cwd: project_root.to_owned(),
894                environment: Some(inherited_environment([
895                    (
896                        OsString::from(RUST_TRANSPORT_ENV),
897                        transport_path.as_os_str().to_owned(),
898                    ),
899                    (
900                        OsString::from(RUST_TRANSPORT_TOKEN_ENV),
901                        OsString::from(token_hex(&token)),
902                    ),
903                    (
904                        OsString::from(RUST_CONTEXT_ENV),
905                        OsString::from("0000000000000000"),
906                    ),
907                    (
908                        OsString::from(RUST_LIBTEST_EVENTS_ENV),
909                        event_path.as_os_str().to_owned(),
910                    ),
911                    (
912                        OsString::from(RUST_LIBTEST_TOKEN_ENV),
913                        OsString::from(token_hex(&event_token)),
914                    ),
915                ])),
916                captured_output: None,
917            },
918            options,
919            &mut io::sink(),
920        )
921        .map_err(|error| RustCompilerTestError::Launch {
922            test: artifact.target_key.clone(),
923            reason: error.to_string(),
924        })?;
925    let ended_at_ms = epoch_ms()?;
926    let read = read_rust_transport(transport_path, &token).map_err(|error| {
927        RustCompilerTestError::Transport {
928            test: artifact.target_key.clone(),
929            reason: error.to_string(),
930        }
931    })?;
932    let events = read_rust_libtest_events(event_path, &event_token).map_err(|error| {
933        RustCompilerTestError::Transport {
934            test: artifact.target_key.clone(),
935            reason: stock_libtest_transport_reason(error, &output),
936        }
937    })?;
938    let events = validate_rust_libtest_run_events(&events, selected_tests.iter().cloned())
939        .map_err(|error| RustCompilerTestError::Transport {
940            test: artifact.target_key.clone(),
941            reason: stock_libtest_transport_reason(error, &output),
942        })?;
943    let terminal_failure = events
944        .attempts
945        .iter()
946        .any(|attempt| attempt.result == RustLibtestTerminalResult::Failed);
947    let expected_success = !terminal_failure && events.unstarted.is_empty();
948    if expected_success != (output.result.exit_code() == 0) {
949        return Err(RustCompilerTestError::UnverifiedExecution {
950            code: output.result.exit_code(),
951            reason: "stock libtest process status disagrees with authenticated terminal and fail-fast events"
952                .into(),
953        });
954    }
955    let roots = contexts.values().copied().collect::<BTreeSet<_>>();
956    let partition = partition_rust_transport_by_test_contexts(&read, &roots).map_err(|error| {
957        RustCompilerTestError::Transport {
958            test: artifact.target_key.clone(),
959            reason: error.to_string(),
960        }
961    })?;
962    fs::remove_file(transport_path).map_err(|error| io_error(transport_path, error))?;
963    transport_cleanup.0 = None;
964    fs::remove_file(event_path).map_err(|error| io_error(event_path, error))?;
965    event_cleanup.0 = None;
966    Ok(StockLibtestExecution {
967        output,
968        events,
969        partition,
970        started_at_ms,
971        ended_at_ms,
972    })
973}
974
975fn execute_process_tasks(
976    project_root: &Path,
977    tasks: &[ProcessTask],
978    requested_workers: usize,
979    supervisor: &ProcessSupervisor,
980    options: SupervisionOptions,
981) -> Result<Vec<ProcessOutcome>, RustCompilerTestError> {
982    let workers = requested_workers.min(tasks.len());
983    let next = AtomicUsize::new(0);
984    let outcomes = Mutex::new(Vec::<Result<ProcessOutcome, String>>::with_capacity(
985        tasks.len(),
986    ));
987    std::thread::scope(|scope| {
988        for _ in 0..workers {
989            scope.spawn(|| {
990                loop {
991                    let index = next.fetch_add(1, Ordering::Relaxed);
992                    let Some(task) = tasks.get(index) else { break };
993                    outcomes
994                        .lock()
995                        .expect("Rust compiler result lock poisoned")
996                        .push(run_process(project_root, task, supervisor, options));
997                }
998            });
999        }
1000    });
1001    let mut outcomes = outcomes
1002        .into_inner()
1003        .map_err(|_| RustCompilerTestError::Context("Rust compiler result lock poisoned".into()))?
1004        .into_iter()
1005        .collect::<Result<Vec<_>, _>>()
1006        .map_err(|reason| RustCompilerTestError::Launch {
1007            test: "unknown attempt".into(),
1008            reason,
1009        })?;
1010    outcomes.sort_by_key(|outcome| outcome.task.ordinal);
1011    Ok(outcomes)
1012}
1013
1014fn regular_directory(path: &Path) -> Result<PathBuf, RustCompilerTestError> {
1015    let metadata = fs::symlink_metadata(path).map_err(|error| io_error(path, error))?;
1016    if !metadata.file_type().is_dir() {
1017        return Err(RustCompilerTestError::UnsafeArtifact(
1018            path.display().to_string(),
1019        ));
1020    }
1021    fs::canonicalize(path).map_err(|error| io_error(path, error))
1022}
1023
1024#[cfg(unix)]
1025fn sync_directory(path: &Path) -> Result<(), RustCompilerTestError> {
1026    fs::File::open(path)
1027        .and_then(|directory| directory.sync_all())
1028        .map_err(|error| io_error(path, error))
1029}
1030
1031#[cfg(not(unix))]
1032fn sync_directory(_path: &Path) -> Result<(), RustCompilerTestError> {
1033    Ok(())
1034}
1035
1036fn write_cargo_runner_unit(
1037    output_directory: &Path,
1038    unit: &RustCargoRunnerUnit,
1039) -> Result<PathBuf, RustCompilerTestError> {
1040    let artifact = unit.artifact.to_str().ok_or_else(|| {
1041        RustCompilerTestError::Context("Cargo test artifact path is not UTF-8".into())
1042    })?;
1043    let mut identity = Sha256::new();
1044    identity.update((unit.target.len() as u64).to_be_bytes());
1045    identity.update(unit.target.as_bytes());
1046    identity.update((artifact.len() as u64).to_be_bytes());
1047    identity.update(artifact.as_bytes());
1048    let digest = format!("{:x}", identity.finalize());
1049    let destination = output_directory.join(format!(
1050        "libtest-{:016}-{}.json",
1051        unit.invocation_ordinal,
1052        &digest[..24]
1053    ));
1054    let partial = output_directory.join(format!(
1055        ".libtest-{:016}-{}-{}.partial",
1056        unit.invocation_ordinal,
1057        &digest[..24],
1058        std::process::id()
1059    ));
1060    let bytes = serde_json::to_vec(unit)
1061        .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1062    let mut options = OpenOptions::new();
1063    options.write(true).create_new(true);
1064    #[cfg(unix)]
1065    {
1066        use std::os::unix::fs::OpenOptionsExt as _;
1067        options.mode(0o600);
1068    }
1069    let mut file = options
1070        .open(&partial)
1071        .map_err(|error| io_error(&partial, error))?;
1072    let write_result = (|| {
1073        file.write_all(&bytes)
1074            .map_err(|error| io_error(&partial, error))?;
1075        file.sync_all().map_err(|error| io_error(&partial, error))?;
1076        drop(file);
1077        fs::rename(&partial, &destination).map_err(|error| io_error(&destination, error))?;
1078        sync_directory(output_directory)
1079    })();
1080    if write_result.is_err() {
1081        let _ = fs::remove_file(&partial);
1082    }
1083    write_result.map(|()| destination)
1084}
1085
1086fn write_cargo_runner_failure(
1087    output_directory: &Path,
1088    failure: &RustCargoRunnerFailure,
1089) -> Result<PathBuf, RustCompilerTestError> {
1090    let destination =
1091        output_directory.join(format!("failure-{:016}.json", failure.invocation_ordinal));
1092    let partial = output_directory.join(format!(
1093        ".failure-{:016}-{}.partial",
1094        failure.invocation_ordinal,
1095        std::process::id()
1096    ));
1097    let bytes = serde_json::to_vec(failure)
1098        .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1099    let mut options = OpenOptions::new();
1100    options.write(true).create_new(true);
1101    #[cfg(unix)]
1102    {
1103        use std::os::unix::fs::OpenOptionsExt as _;
1104        options.mode(0o600);
1105    }
1106    let mut file = options
1107        .open(&partial)
1108        .map_err(|error| io_error(&partial, error))?;
1109    let write_result = (|| {
1110        file.write_all(&bytes)
1111            .map_err(|error| io_error(&partial, error))?;
1112        file.sync_all().map_err(|error| io_error(&partial, error))?;
1113        drop(file);
1114        fs::rename(&partial, &destination).map_err(|error| io_error(&destination, error))?;
1115        sync_directory(output_directory)
1116    })();
1117    if write_result.is_err() {
1118        let _ = fs::remove_file(&partial);
1119    }
1120    write_result.map(|()| destination)
1121}
1122
1123fn reserve_cargo_runner_ordinal(output_directory: &Path) -> Result<u64, RustCompilerTestError> {
1124    for ordinal in 0..1_000_000_u64 {
1125        let reservation = output_directory.join(format!(".sequence-{ordinal:016}.reserved"));
1126        let mut options = OpenOptions::new();
1127        options.write(true).create_new(true);
1128        #[cfg(unix)]
1129        {
1130            use std::os::unix::fs::OpenOptionsExt as _;
1131            options.mode(0o600);
1132        }
1133        match options.open(&reservation) {
1134            Ok(file) => {
1135                file.sync_all()
1136                    .map_err(|error| io_error(&reservation, error))?;
1137                sync_directory(output_directory)?;
1138                return Ok(ordinal);
1139            }
1140            Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}
1141            Err(error) => return Err(io_error(&reservation, error)),
1142        }
1143    }
1144    Err(RustCompilerTestError::Context(
1145        "Cargo runner invocation ordinal space is exhausted".into(),
1146    ))
1147}
1148
1149fn run_nextest_list_passthrough(
1150    current_directory: &Path,
1151    artifact: &TestArtifact,
1152    underlying_runner: Option<&RustCargoResolvedRunner>,
1153    arguments: Vec<OsString>,
1154    watchdog_program: Option<&Path>,
1155    stdout: &mut dyn Write,
1156    stderr: &mut dyn Write,
1157) -> Result<RustCargoRunnerExecution, RustCompilerTestError> {
1158    let supervisor = watchdog_program
1159        .map_or_else(ProcessSupervisor::new, ProcessSupervisor::new_crash_safe)
1160        .map_err(|error| RustCompilerTestError::Launch {
1161            test: "nextest list".into(),
1162            reason: error.to_string(),
1163        })?;
1164    let options =
1165        SupervisionOptions::from_environment().map_err(|error| RustCompilerTestError::Launch {
1166            test: "nextest list".into(),
1167            reason: error.to_string(),
1168        })?;
1169    let (program, arguments) = artifact_command(artifact, underlying_runner, arguments);
1170    let output = supervisor
1171        .supervise_captured(
1172            &CommandSpec {
1173                program,
1174                arguments,
1175                cwd: current_directory.to_owned(),
1176                environment: Some(inherited_environment([])),
1177                captured_output: None,
1178            },
1179            options,
1180            &mut io::sink(),
1181        )
1182        .map_err(|error| RustCompilerTestError::Launch {
1183            test: "nextest list".into(),
1184            reason: error.to_string(),
1185        })?;
1186    stdout
1187        .write_all(&output.stdout)
1188        .map_err(|error| io_error(current_directory, error))?;
1189    stderr
1190        .write_all(&output.stderr)
1191        .map_err(|error| io_error(current_directory, error))?;
1192    Ok(RustCargoRunnerExecution {
1193        exit_code: output.result.exit_code(),
1194        unit_path: None,
1195    })
1196}
1197
1198pub fn run_cargo_libtest_runner(
1199    config_path: &Path,
1200    arguments: Vec<OsString>,
1201    watchdog_program: Option<PathBuf>,
1202    stdout: &mut dyn Write,
1203    stderr: &mut dyn Write,
1204) -> Result<RustCargoRunnerExecution, RustCompilerTestError> {
1205    let config_metadata =
1206        fs::symlink_metadata(config_path).map_err(|error| io_error(config_path, error))?;
1207    if !config_metadata.file_type().is_file() {
1208        return Err(RustCompilerTestError::UnsafeArtifact(
1209            config_path.display().to_string(),
1210        ));
1211    }
1212    let config: RustCargoRunnerConfig = serde_json::from_slice(
1213        &fs::read(config_path).map_err(|error| io_error(config_path, error))?,
1214    )
1215    .map_err(|error| {
1216        RustCompilerTestError::Context(format!("invalid Cargo runner config: {error}"))
1217    })?;
1218    if config.version != RUST_CARGO_RUNNER_VERSION
1219        || !config.run_id.starts_with("run_")
1220        || config.run_id.len() != 20
1221        || !config.run_id[4..]
1222            .bytes()
1223            .all(|byte| byte.is_ascii_hexdigit())
1224    {
1225        return Err(RustCompilerTestError::Context(
1226            "Cargo runner config has an unsupported version or invalid run ID".into(),
1227        ));
1228    }
1229    let target_directory = regular_directory(&config.target_directory)?;
1230    let output_directory = regular_directory(&config.output_directory)?;
1231    let run_root = target_directory
1232        .parent()
1233        .ok_or_else(|| RustCompilerTestError::Context("Cargo target has no run root".into()))?;
1234    if !output_directory.starts_with(run_root) || output_directory == target_directory {
1235        return Err(RustCompilerTestError::UnsafeArtifact(
1236            output_directory.display().to_string(),
1237        ));
1238    }
1239    let mut configured_targets = BTreeSet::new();
1240    if config.target_runners.is_empty()
1241        || config
1242            .target_runners
1243            .iter()
1244            .any(|target| target.target.is_empty() || !configured_targets.insert(&target.target))
1245    {
1246        return Err(RustCompilerTestError::Context(
1247            "Cargo runner config has empty or duplicate target identities".into(),
1248        ));
1249    }
1250    let mut configured_artifacts = BTreeSet::new();
1251    if config.artifacts.iter().any(|artifact| {
1252        !configured_artifacts.insert(&artifact.executable)
1253            || !artifact.executable.starts_with(&target_directory)
1254    }) {
1255        return Err(RustCompilerTestError::Context(
1256            "Cargo runner config has duplicate or out-of-target artifacts".into(),
1257        ));
1258    }
1259    let runner_identity = classify_rust_runner_environment()
1260        .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1261    let run_id = config.run_id.clone();
1262    let failure_target = arguments
1263        .first()
1264        .and_then(|target| target.clone().into_string().ok());
1265    let failure_artifact = arguments.get(1).map(PathBuf::from);
1266    let mut runner_arguments = arguments.into_iter();
1267    let target = runner_arguments
1268        .next()
1269        .ok_or_else(|| {
1270            RustCompilerTestError::Context("Cargo runner received no target identity".into())
1271        })?
1272        .into_string()
1273        .map_err(|_| {
1274            RustCompilerTestError::Context(
1275                "Cargo runner received a non-UTF-8 target identity".into(),
1276            )
1277        })?;
1278    let target_runner = config
1279        .target_runners
1280        .iter()
1281        .find(|candidate| candidate.target == target)
1282        .ok_or_else(|| {
1283            RustCompilerTestError::Context(format!(
1284                "Cargo runner received an unconfigured target identity: {target}"
1285            ))
1286        })?;
1287    let artifact_argument = runner_arguments.next().ok_or_else(|| {
1288        RustCompilerTestError::Context("Cargo runner received no artifact".into())
1289    })?;
1290    let artifact_path = PathBuf::from(&artifact_argument);
1291    let artifact =
1292        fs::canonicalize(&artifact_path).map_err(|error| io_error(&artifact_path, error))?;
1293    if !artifact.starts_with(&target_directory)
1294        || !fs::symlink_metadata(&artifact).is_ok_and(|metadata| metadata.file_type().is_file())
1295    {
1296        return Err(RustCompilerTestError::UnsafeArtifact(
1297            artifact.display().to_string(),
1298        ));
1299    }
1300    let test_harness = config
1301        .artifacts
1302        .iter()
1303        .find(|candidate| candidate.executable == artifact)
1304        .map(|candidate| candidate.test_harness);
1305    if test_harness.is_none()
1306        && !matches!(
1307            runner_identity,
1308            RustRunnerInvocationIdentity::NextestList(_)
1309        )
1310    {
1311        return Err(RustCompilerTestError::Context(format!(
1312            "Cargo runner executed an unclassified artifact: {}",
1313            artifact.display()
1314        )));
1315    }
1316    let arguments = runner_arguments.collect::<Vec<_>>();
1317    let current_directory = std::env::current_dir()
1318        .and_then(fs::canonicalize)
1319        .map_err(|error| io_error(Path::new("."), error))?;
1320    let test_artifact = TestArtifact {
1321        executable: artifact.clone(),
1322        runner_argument: Some(artifact_argument),
1323        package: "cargo-pending".into(),
1324        target_key: "cargo-pending".into(),
1325        kind: "cargo-pending".into(),
1326        source: "cargo-pending".into(),
1327        test_harness: test_harness.unwrap_or(true),
1328    };
1329    let underlying_runner = target_runner.underlying_runner.clone();
1330    if matches!(
1331        runner_identity,
1332        RustRunnerInvocationIdentity::NextestList(_)
1333    ) {
1334        return run_nextest_list_passthrough(
1335            &current_directory,
1336            &test_artifact,
1337            underlying_runner.as_ref(),
1338            arguments,
1339            watchdog_program.as_deref(),
1340            stdout,
1341            stderr,
1342        );
1343    }
1344
1345    let invocation_ordinal = reserve_cargo_runner_ordinal(&output_directory)?;
1346    let result = (|| {
1347        let utf8_arguments = arguments
1348            .iter()
1349            .cloned()
1350            .map(|argument| {
1351                argument.into_string().map_err(|_| {
1352                    RustCompilerTestError::Context(
1353                        "Cargo runner received a non-UTF-8 libtest argument".into(),
1354                    )
1355                })
1356            })
1357            .collect::<Result<Vec<_>, _>>()?;
1358        let invocation = crate::rust_test_runner::CargoTestInvocation {
1359            program: "cargo".into(),
1360            kind: crate::rust_test_runner::RustCargoCommandKind::CargoTest,
1361            arguments: vec!["test".into()],
1362            runner_arguments: utf8_arguments.clone(),
1363        };
1364        let artifact_digest = format!(
1365            "{:x}",
1366            Sha256::digest(artifact.as_os_str().as_encoded_bytes())
1367        );
1368        let transport_directory = output_directory.join("attempts").join(format!(
1369            "{invocation_ordinal:016}-{}",
1370            &artifact_digest[..24]
1371        ));
1372        fs::create_dir_all(&transport_directory)
1373            .map_err(|error| io_error(&transport_directory, error))?;
1374        let transport_directory = regular_directory(&transport_directory)?;
1375        let supervisor = watchdog_program
1376            .as_deref()
1377            .map_or_else(ProcessSupervisor::new, ProcessSupervisor::new_crash_safe)
1378            .map_err(|error| RustCompilerTestError::Launch {
1379                test: "Cargo runner".into(),
1380                reason: error.to_string(),
1381            })?;
1382        let options = SupervisionOptions::from_environment().map_err(|error| {
1383            RustCompilerTestError::Launch {
1384                test: "Cargo runner".into(),
1385                reason: error.to_string(),
1386            }
1387        })?;
1388        let (runner, runner_run_id, runner_version, runner_binary_id, requested_workers, attempts) =
1389            match &runner_identity {
1390                RustRunnerInvocationIdentity::CargoSingleAttempt if test_artifact.test_harness => {
1391                    let selection = rust_libtest_selection(&invocation).map_err(|error| {
1392                        RustCompilerTestError::UnsupportedCommand(error.to_string())
1393                    })?;
1394                    let tests = list_tests(
1395                        &current_directory,
1396                        &test_artifact,
1397                        &selection.list_arguments,
1398                        underlying_runner.as_ref(),
1399                        &supervisor,
1400                        options,
1401                        &transport_directory.join("list.events"),
1402                    )?;
1403                    let contexts = preflight_rust_test_contexts(tests.clone())
1404                        .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1405                    let stock = run_stock_libtest_artifact(
1406                        &current_directory,
1407                        &test_artifact,
1408                        underlying_runner.as_ref(),
1409                        arguments.clone(),
1410                        &tests,
1411                        &contexts,
1412                        &transport_directory.join("artifact.mmap"),
1413                        &transport_directory.join("artifact.libtest-events"),
1414                        &supervisor,
1415                        options,
1416                    )?;
1417                    let StockLibtestExecution {
1418                        output,
1419                        events,
1420                        mut partition,
1421                        started_at_ms,
1422                        ended_at_ms,
1423                    } = stock;
1424                    let exit_code = output.result.exit_code();
1425                    stdout
1426                        .write_all(&output.stdout)
1427                        .and_then(|()| stderr.write_all(&output.stderr))
1428                        .map_err(|error| io_error(&output_directory, error))?;
1429                    let mut attempts = Vec::with_capacity(tests.len());
1430                    for attempt in events.attempts {
1431                        let context_id = contexts[&attempt.name];
1432                        let index = attempts.len();
1433                        attempts.push(RustCargoRunnerAttempt {
1434                            test: attempt.name,
1435                            context_id,
1436                            retry: 0,
1437                            total_attempts: 1,
1438                            runner_attempt_id: format!(
1439                                "{}:cargo:{invocation_ordinal:016}:{index:08}",
1440                                config.run_id
1441                            ),
1442                            outcome: RustCargoRunnerAttemptOutcome::Libtest {
1443                                result: attempt.result,
1444                                timed_out: attempt.timed_out,
1445                            },
1446                            transport: partition
1447                                .attributed
1448                                .remove(&context_id)
1449                                .expect("every selected test context was partitioned"),
1450                        });
1451                    }
1452                    for test in events.unstarted {
1453                        let context_id = contexts[&test];
1454                        let index = attempts.len();
1455                        attempts.push(RustCargoRunnerAttempt {
1456                            test,
1457                            context_id,
1458                            retry: 0,
1459                            total_attempts: 1,
1460                            runner_attempt_id: format!(
1461                                "{}:cargo:{invocation_ordinal:016}:{index:08}",
1462                                config.run_id
1463                            ),
1464                            outcome: RustCargoRunnerAttemptOutcome::Unstarted,
1465                            transport: partition
1466                                .attributed
1467                                .remove(&context_id)
1468                                .expect("every unstarted test context was partitioned"),
1469                        });
1470                    }
1471                    if !partition.attributed.is_empty() {
1472                        return Err(RustCompilerTestError::Context(
1473                            "stock libtest event join left selected test contexts unclaimed".into(),
1474                        ));
1475                    }
1476                    let unit = RustCargoRunnerUnit {
1477                        version: RUST_CARGO_RUNNER_VERSION,
1478                        run_id: run_id.clone(),
1479                        invocation_ordinal,
1480                        runner: RustCargoRunnerKind::CargoTest,
1481                        runner_run_id: None,
1482                        runner_version: None,
1483                        runner_binary_id: None,
1484                        target,
1485                        artifact,
1486                        arguments: utf8_arguments,
1487                        invocation: RustCargoRunnerInvocation {
1488                            result: output.result,
1489                            started_at_ms,
1490                            ended_at_ms,
1491                            stdout: output.stdout,
1492                            stderr: output.stderr,
1493                            background_transport: partition.background,
1494                        },
1495                        attempts,
1496                        thread_scope_limitations: partition.thread_scope_limitations,
1497                    };
1498                    let unit_path = write_cargo_runner_unit(&output_directory, &unit)?;
1499                    fs::remove_dir(&transport_directory)
1500                        .map_err(|error| io_error(&transport_directory, error))?;
1501                    return Ok(RustCargoRunnerExecution {
1502                        exit_code,
1503                        unit_path: Some(unit_path),
1504                    });
1505                }
1506                RustRunnerInvocationIdentity::CargoSingleAttempt => (
1507                    RustCargoRunnerKind::CargoCustomHarness,
1508                    None,
1509                    None,
1510                    None,
1511                    1,
1512                    vec![(
1513                        "custom-harness".into(),
1514                        0,
1515                        1,
1516                        format!("{}:cargo:{invocation_ordinal:016}:00000000", config.run_id),
1517                        arguments.clone(),
1518                    )],
1519                ),
1520                RustRunnerInvocationIdentity::NextestAttempt(NextestAttemptIdentity {
1521                    invocation,
1522                    test_name,
1523                    retry,
1524                    total_attempts,
1525                    runner_attempt_id,
1526                }) => {
1527                    if !test_artifact.test_harness {
1528                        return Err(RustCompilerTestError::Context(
1529                            "nextest attempted to execute a custom Cargo harness".into(),
1530                        ));
1531                    }
1532                    if !utf8_arguments
1533                        .windows(2)
1534                        .any(|pair| pair == ["--exact", test_name])
1535                    {
1536                        return Err(RustCompilerTestError::Context(
1537                            "nextest target-runner arguments do not select NEXTEST_TEST_NAME exactly"
1538                                .into(),
1539                        ));
1540                    }
1541                    (
1542                        RustCargoRunnerKind::Nextest,
1543                        Some(invocation.run_id.clone()),
1544                        Some(invocation.version.clone()),
1545                        Some(invocation.binary_id.clone()),
1546                        1,
1547                        vec![(
1548                            test_name.clone(),
1549                            *retry,
1550                            *total_attempts,
1551                            runner_attempt_id.clone(),
1552                            arguments.clone(),
1553                        )],
1554                    )
1555                }
1556                RustRunnerInvocationIdentity::NextestList(_) => unreachable!("handled above"),
1557            };
1558        let tests = attempts
1559            .iter()
1560            .map(|(test, _, _, _, _)| test.clone())
1561            .collect::<Vec<_>>();
1562        let contexts = preflight_rust_test_contexts(tests.clone())
1563            .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1564        let tasks = attempts
1565            .into_iter()
1566            .enumerate()
1567            .map(
1568                |(index, (test, retry, total_attempts, runner_attempt_id, test_arguments))| {
1569                    ProcessTask {
1570                        ordinal: index,
1571                        artifact_index: 0,
1572                        artifact: test_artifact.clone(),
1573                        test: test.clone(),
1574                        test_id: format!("rust:cargo-runner:{}::{test}", &artifact_digest[..24]),
1575                        context_id: contexts[&test],
1576                        retry,
1577                        total_attempts,
1578                        runner_attempt_id,
1579                        runner,
1580                        transport: transport_directory.join(format!("{index:08}.mmap")),
1581                        libtest_events: transport_directory
1582                            .join(format!("{index:08}.libtest-events")),
1583                        test_arguments,
1584                        underlying_runner: underlying_runner.clone(),
1585                    }
1586                },
1587            )
1588            .collect::<Vec<_>>();
1589        let mut outcomes = execute_process_tasks(
1590            &current_directory,
1591            &tasks,
1592            requested_workers,
1593            &supervisor,
1594            options,
1595        )?;
1596        let [outcome] = outcomes.as_mut_slice() else {
1597            return Err(RustCompilerTestError::Context(
1598                "custom-harness and nextest runner invocations must own exactly one process".into(),
1599            ));
1600        };
1601        stdout
1602            .write_all(&outcome.output.stdout)
1603            .and_then(|()| stderr.write_all(&outcome.output.stderr))
1604            .map_err(|error| io_error(&output_directory, error))?;
1605        let exit_code = outcome.output.result.exit_code();
1606        let roots = BTreeSet::from([outcome.task.context_id]);
1607        let mut partition = partition_rust_transport_by_test_contexts(&outcome.read, &roots)
1608            .map_err(|error| RustCompilerTestError::Transport {
1609                test: outcome.task.test.clone(),
1610                reason: error.to_string(),
1611            })?;
1612        let attempt_outcome = match runner {
1613            RustCargoRunnerKind::CargoCustomHarness => RustCargoRunnerAttemptOutcome::OpaqueProcess,
1614            RustCargoRunnerKind::Nextest => match outcome.attempt_outcome {
1615                RustCargoRunnerAttemptOutcome::Libtest { result, timed_out } => {
1616                    RustCargoRunnerAttemptOutcome::Libtest { result, timed_out }
1617                }
1618                _ => {
1619                    return Err(RustCompilerTestError::Context(
1620                        "nextest process has no authenticated libtest terminal event".into(),
1621                    ));
1622                }
1623            },
1624            RustCargoRunnerKind::CargoTest => unreachable!("stock libtest returned above"),
1625        };
1626        let attempts = vec![RustCargoRunnerAttempt {
1627            test: outcome.task.test.clone(),
1628            context_id: outcome.task.context_id,
1629            retry: outcome.task.retry,
1630            total_attempts: outcome.task.total_attempts,
1631            runner_attempt_id: outcome.task.runner_attempt_id.clone(),
1632            outcome: attempt_outcome,
1633            transport: partition
1634                .attributed
1635                .remove(&outcome.task.context_id)
1636                .expect("the one attempt context was partitioned"),
1637        }];
1638        let unit = RustCargoRunnerUnit {
1639            version: RUST_CARGO_RUNNER_VERSION,
1640            run_id: run_id.clone(),
1641            invocation_ordinal,
1642            runner,
1643            runner_run_id,
1644            runner_version,
1645            runner_binary_id,
1646            target,
1647            artifact,
1648            arguments: utf8_arguments,
1649            invocation: RustCargoRunnerInvocation {
1650                result: outcome.output.result.clone(),
1651                started_at_ms: outcome.started_at_ms,
1652                ended_at_ms: outcome.ended_at_ms,
1653                stdout: outcome.output.stdout.clone(),
1654                stderr: outcome.output.stderr.clone(),
1655                background_transport: partition.background,
1656            },
1657            attempts,
1658            thread_scope_limitations: partition.thread_scope_limitations,
1659        };
1660        let unit_path = write_cargo_runner_unit(&output_directory, &unit)?;
1661        fs::remove_dir(&transport_directory)
1662            .map_err(|error| io_error(&transport_directory, error))?;
1663        Ok(RustCargoRunnerExecution {
1664            exit_code,
1665            unit_path: Some(unit_path),
1666        })
1667    })();
1668    if let Err(error) = &result {
1669        let failure = RustCargoRunnerFailure {
1670            version: RUST_CARGO_RUNNER_VERSION,
1671            run_id,
1672            invocation_ordinal,
1673            target: failure_target,
1674            artifact: failure_artifact,
1675            error: error.to_string(),
1676        };
1677        if let Err(publication_error) = write_cargo_runner_failure(&output_directory, &failure) {
1678            return Err(RustCompilerTestError::Context(format!(
1679                "{error}; Cargo runner also could not publish its failure: {publication_error}"
1680            )));
1681        }
1682    }
1683    result
1684}
1685
1686pub fn read_cargo_runner_units(
1687    output_directory: &Path,
1688    run_id: &str,
1689    expected_targets: &[String],
1690) -> Result<Vec<RustCargoRunnerUnit>, RustCompilerTestError> {
1691    let output_directory = regular_directory(output_directory)?;
1692    let expected_target_count = expected_targets.len();
1693    let expected_targets = expected_targets.iter().collect::<BTreeSet<_>>();
1694    if expected_targets.is_empty() || expected_targets.len() != expected_target_count {
1695        return Err(RustCompilerTestError::Context(
1696            "Cargo runner expected-target set is empty or duplicated".into(),
1697        ));
1698    }
1699    let mut reservations = BTreeSet::new();
1700    let mut units = Vec::new();
1701    let mut failures = Vec::new();
1702    let mut retained_attempt_state = false;
1703    for entry in
1704        fs::read_dir(&output_directory).map_err(|error| io_error(&output_directory, error))?
1705    {
1706        let entry = entry.map_err(|error| io_error(&output_directory, error))?;
1707        let path = entry.path();
1708        let metadata = fs::symlink_metadata(&path).map_err(|error| io_error(&path, error))?;
1709        let name = entry.file_name();
1710        let name = name.to_str().ok_or_else(|| {
1711            RustCompilerTestError::Context("Cargo runner output name is not UTF-8".into())
1712        })?;
1713        if name == "attempts" && metadata.file_type().is_dir() {
1714            retained_attempt_state = fs::read_dir(&path)
1715                .map_err(|error| io_error(&path, error))?
1716                .next()
1717                .transpose()
1718                .map_err(|error| io_error(&path, error))?
1719                .is_some();
1720            continue;
1721        }
1722        if name.starts_with(".sequence-") && name.ends_with(".reserved") {
1723            if !metadata.file_type().is_file() || metadata.len() != 0 {
1724                return Err(RustCompilerTestError::UnsafeArtifact(
1725                    path.display().to_string(),
1726                ));
1727            }
1728            let ordinal = name
1729                .strip_prefix(".sequence-")
1730                .and_then(|name| name.strip_suffix(".reserved"))
1731                .filter(|value| {
1732                    value.len() == 16 && value.bytes().all(|byte| byte.is_ascii_digit())
1733                })
1734                .and_then(|value| value.parse::<u64>().ok())
1735                .ok_or_else(|| {
1736                    RustCompilerTestError::Context("malformed Cargo runner reservation".into())
1737                })?;
1738            if !reservations.insert(ordinal) {
1739                return Err(RustCompilerTestError::Context(
1740                    "duplicate Cargo runner reservation".into(),
1741                ));
1742            }
1743            continue;
1744        }
1745        if name.starts_with("failure-") && name.ends_with(".json") {
1746            if !metadata.file_type().is_file() {
1747                return Err(RustCompilerTestError::UnsafeArtifact(
1748                    path.display().to_string(),
1749                ));
1750            }
1751            let failure: RustCargoRunnerFailure =
1752                serde_json::from_slice(&fs::read(&path).map_err(|error| io_error(&path, error))?)
1753                    .map_err(|error| {
1754                    RustCompilerTestError::Context(format!(
1755                        "invalid Cargo runner failure unit: {error}"
1756                    ))
1757                })?;
1758            if failure.version != RUST_CARGO_RUNNER_VERSION || failure.run_id != run_id {
1759                return Err(RustCompilerTestError::Context(
1760                    "Cargo runner failure unit has incompatible identity".into(),
1761                ));
1762            }
1763            failures.push(failure);
1764            continue;
1765        }
1766        if !name.starts_with("libtest-")
1767            || !name.ends_with(".json")
1768            || !metadata.file_type().is_file()
1769        {
1770            return Err(RustCompilerTestError::UnsafeArtifact(
1771                path.display().to_string(),
1772            ));
1773        }
1774        let unit: RustCargoRunnerUnit =
1775            serde_json::from_slice(&fs::read(&path).map_err(|error| io_error(&path, error))?)
1776                .map_err(|error| {
1777                    RustCompilerTestError::Context(format!("invalid Cargo runner unit: {error}"))
1778                })?;
1779        if unit.version != RUST_CARGO_RUNNER_VERSION || unit.run_id != run_id {
1780            return Err(RustCompilerTestError::Context(
1781                "Cargo runner unit has incompatible identity".into(),
1782            ));
1783        }
1784        if !expected_targets.contains(&unit.target) {
1785            return Err(RustCompilerTestError::Context(format!(
1786                "Cargo runner unit has an unselected target identity: {}",
1787                unit.target
1788            )));
1789        }
1790        validate_persisted_runner_transport(&unit)?;
1791        units.push(unit);
1792    }
1793    units.sort_by_key(|unit| unit.invocation_ordinal);
1794    failures.sort_by_key(|failure| failure.invocation_ordinal);
1795    let mut publications = BTreeSet::new();
1796    for ordinal in units
1797        .iter()
1798        .map(|unit| unit.invocation_ordinal)
1799        .chain(failures.iter().map(|failure| failure.invocation_ordinal))
1800    {
1801        if !reservations.contains(&ordinal) || !publications.insert(ordinal) {
1802            return Err(RustCompilerTestError::Context(
1803                "Cargo runner invocation publications are malformed or duplicated".into(),
1804            ));
1805        }
1806    }
1807    if reservations.len() != publications.len()
1808        || reservations
1809            .iter()
1810            .enumerate()
1811            .any(|(expected, ordinal)| *ordinal != expected as u64)
1812    {
1813        return Err(RustCompilerTestError::Context(
1814            "Cargo runner reserved an invocation without publishing its unit".into(),
1815        ));
1816    }
1817    if let Some(failure) = failures.first() {
1818        return Err(RustCompilerTestError::Context(format!(
1819            "Cargo runner invocation {} failed for {}: {}",
1820            failure.invocation_ordinal,
1821            failure.artifact.as_ref().map_or_else(
1822                || "an unknown artifact".into(),
1823                |path| path.display().to_string()
1824            ),
1825            failure.error
1826        )));
1827    }
1828    if retained_attempt_state {
1829        return Err(RustCompilerTestError::Context(
1830            "Cargo runner retained attempt transport state".into(),
1831        ));
1832    }
1833    let runner_kinds = units
1834        .iter()
1835        .map(|unit| unit.runner)
1836        .collect::<BTreeSet<_>>();
1837    if runner_kinds.contains(&RustCargoRunnerKind::Nextest) && runner_kinds.len() > 1 {
1838        return Err(RustCompilerTestError::Context(
1839            "Cargo runner mixed standard Cargo and nextest units".into(),
1840        ));
1841    }
1842    let mut attempt_ids = BTreeSet::new();
1843    if units.iter().flat_map(|unit| &unit.attempts).any(|attempt| {
1844        attempt.runner_attempt_id.is_empty()
1845            || !attempt_ids.insert(attempt.runner_attempt_id.clone())
1846    }) {
1847        return Err(RustCompilerTestError::Context(
1848            "Cargo runner attempt identity is empty or duplicated".into(),
1849        ));
1850    }
1851    if runner_kinds.contains(&RustCargoRunnerKind::Nextest) {
1852        let mut nextest_identity = None;
1853        let mut logical_attempts =
1854            BTreeMap::<(String, PathBuf, String), Vec<&RustCargoRunnerAttempt>>::new();
1855        for unit in &units {
1856            let identity = (unit.runner_run_id.as_ref(), unit.runner_version.as_ref());
1857            if identity.0.is_none()
1858                || identity.1.is_none()
1859                || unit.runner_binary_id.is_none()
1860                || unit.attempts.len() != 1
1861            {
1862                return Err(RustCompilerTestError::Context(
1863                    "nextest runner unit lacks exact invocation or attempt identity".into(),
1864                ));
1865            }
1866            match &nextest_identity {
1867                Some(expected) if *expected != identity => {
1868                    return Err(RustCompilerTestError::Context(
1869                        "nextest runner units belong to different runs or versions".into(),
1870                    ));
1871                }
1872                None => nextest_identity = Some(identity),
1873                _ => {}
1874            }
1875            let attempt = &unit.attempts[0];
1876            logical_attempts
1877                .entry((
1878                    unit.target.clone(),
1879                    unit.artifact.clone(),
1880                    attempt.test.clone(),
1881                ))
1882                .or_default()
1883                .push(attempt);
1884        }
1885        for attempts in logical_attempts.values_mut() {
1886            attempts.sort_by_key(|attempt| attempt.retry);
1887            let total_attempts = attempts[0].total_attempts;
1888            for (expected_retry, attempt) in attempts.iter().enumerate() {
1889                if attempt.retry != expected_retry
1890                    || attempt.total_attempts != total_attempts
1891                    || attempt.retry >= total_attempts
1892                    || (expected_retry + 1 < attempts.len()
1893                        && attempt_outcome_succeeded(&attempt.outcome))
1894                {
1895                    return Err(RustCompilerTestError::Context(
1896                        "nextest retry sequence is noncontiguous, inconsistent, or continues after success"
1897                            .into(),
1898                    ));
1899                }
1900            }
1901        }
1902    } else {
1903        let mut artifacts = BTreeSet::new();
1904        if units.iter().any(|unit| {
1905            unit.runner_run_id.is_some()
1906                || unit.runner_version.is_some()
1907                || unit.runner_binary_id.is_some()
1908                || !artifacts.insert((unit.target.clone(), unit.artifact.clone()))
1909                || unit
1910                    .attempts
1911                    .iter()
1912                    .any(|attempt| attempt.retry != 0 || attempt.total_attempts != 1)
1913                || (unit.runner == RustCargoRunnerKind::CargoCustomHarness
1914                    && (unit.attempts.len() != 1 || unit.attempts[0].test != "custom-harness"))
1915        }) {
1916            return Err(RustCompilerTestError::Context(
1917                "Cargo runner units violate the single-attempt artifact identity contract".into(),
1918            ));
1919        }
1920    }
1921    Ok(units)
1922}
1923
1924fn rust_compiler_coverage_model() -> CoverageModelDeclaration {
1925    CoverageModelDeclaration {
1926        language: "rust".into(),
1927        variant: "rustc-mir-owned-v1".into(),
1928        name: "supercov-rust-compiler-v1".into(),
1929        completeness_meaning: "Every compiler-derived obligation in the frozen owned-source denominator was observed; explicit compiler limitations identify Rust surfaces not yet measured.".into(),
1930        measured: vec![
1931            "compiler-derived owned Rust statement and function-entry obligations".into(),
1932            "compiler-derived control-flow alternatives and atomic decision vectors".into(),
1933            "macro-expanded and generated owned code with exact compiler provenance".into(),
1934            "exact stock-libtest in-process context attribution".into(),
1935            "exact process-per-custom-harness-invocation attribution".into(),
1936            "exact assertion-phase attribution for supported assertion macros".into(),
1937        ],
1938        not_measured: vec![
1939            "capabilities explicitly listed in the compiler manifest limitations".into(),
1940            "all input values, semantic partitions, paths, or concurrency interleavings".into(),
1941            "mutation score or assertion fault-detection strength".into(),
1942        ],
1943    }
1944}
1945
1946fn runner_declaration() -> FrontendRunnerDeclaration {
1947    FrontendRunnerDeclaration {
1948        runner: "rust-libtest".into(),
1949        execution_model: ExecutionModel::ParallelContextPropagated,
1950        attribution: FrontendAttribution {
1951            run: AttributionPrecision::Exact,
1952            worker: AttributionPrecision::Exact,
1953            test: AttributionPrecision::Exact,
1954            retry: AttributionPrecision::Exact,
1955            phase: AttributionPrecision::Exact,
1956            action: AttributionPrecision::Unavailable,
1957            assertion: AttributionPrecision::Exact,
1958        },
1959        limitations: vec![FrontendLimitation {
1960            id: "rust-action-linkage-unavailable".into(),
1961            scopes: vec![FrontendLimitationScope::Action],
1962            reason: "Rust libtest exposes no general application-action lifecycle".into(),
1963        }],
1964    }
1965}
1966
1967fn nextest_runner_declaration() -> FrontendRunnerDeclaration {
1968    let mut declaration = runner_declaration();
1969    declaration.runner = "rust-nextest".into();
1970    declaration.execution_model = ExecutionModel::ProcessPerTest;
1971    declaration.limitations[0].reason =
1972        "nextest exposes no general application-action lifecycle".into();
1973    declaration
1974}
1975
1976fn custom_harness_runner_declaration() -> FrontendRunnerDeclaration {
1977    let mut declaration = runner_declaration();
1978    declaration.runner = "rust-custom-harness".into();
1979    declaration.execution_model = ExecutionModel::ProcessPerTest;
1980    declaration.limitations[0].id = "rust-custom-harness-action-linkage-unavailable".into();
1981    declaration.limitations[0].reason =
1982        "A custom Cargo harness exposes no general application-action lifecycle".into();
1983    declaration.limitations.push(FrontendLimitation {
1984        id: "rust-custom-harness-internal-tests-opaque".into(),
1985        scopes: vec![FrontendLimitationScope::Test],
1986        reason: "Cargo exposes a custom harness as one target invocation; Supercov attributes that invocation exactly without inventing internal test-case identities".into(),
1987    });
1988    declaration
1989}
1990
1991fn compiler_runner_declaration() -> FrontendRunnerDeclaration {
1992    FrontendRunnerDeclaration {
1993        runner: "rustc".into(),
1994        execution_model: ExecutionModel::ProcessPerTest,
1995        attribution: FrontendAttribution {
1996            run: AttributionPrecision::Exact,
1997            worker: AttributionPrecision::Exact,
1998            test: AttributionPrecision::Exact,
1999            retry: AttributionPrecision::Exact,
2000            phase: AttributionPrecision::Exact,
2001            action: AttributionPrecision::Unavailable,
2002            assertion: AttributionPrecision::Unavailable,
2003        },
2004        limitations: vec![
2005            FrontendLimitation {
2006                id: "rust-ctfe-action-linkage-unavailable".into(),
2007                scopes: vec![FrontendLimitationScope::Action],
2008                reason: "Compile-time evaluation is build execution, not an application action"
2009                    .into(),
2010            },
2011            FrontendLimitation {
2012                id: "rust-ctfe-assertion-linkage-unavailable".into(),
2013                scopes: vec![FrontendLimitationScope::Assertion],
2014                reason: "Compile-time execution has no user-test assertion lifecycle".into(),
2015            },
2016        ],
2017    }
2018}
2019
2020fn rustdoc_runner_declaration() -> FrontendRunnerDeclaration {
2021    FrontendRunnerDeclaration {
2022        runner: "rustdoc".into(),
2023        execution_model: ExecutionModel::ParallelContextPropagated,
2024        attribution: FrontendAttribution {
2025            run: AttributionPrecision::Exact,
2026            worker: AttributionPrecision::Exact,
2027            test: AttributionPrecision::Exact,
2028            retry: AttributionPrecision::Exact,
2029            phase: AttributionPrecision::Exact,
2030            action: AttributionPrecision::Unavailable,
2031            assertion: AttributionPrecision::Exact,
2032        },
2033        limitations: vec![FrontendLimitation {
2034            id: "rustdoc-action-linkage-unavailable".into(),
2035            scopes: vec![FrontendLimitationScope::Action],
2036            reason: "Rust doctests expose assertions but no general application-action lifecycle"
2037                .into(),
2038        }],
2039    }
2040}
2041
2042fn doctest_raw_results(
2043    run_id: &str,
2044    resolution: &RustdocOutcomeResolution,
2045    started_at_ms: i64,
2046    ended_at_ms: i64,
2047    normalized: &NormalizedRustCompilerManifest,
2048) -> Result<(Vec<RawTestResult>, Vec<RustCompilerTransportHealthRecord>), RustCompilerTestError> {
2049    let mut results = Vec::new();
2050    let mut health = Vec::new();
2051    for group in &resolution.groups {
2052        if group.transport.dropped != 0 {
2053            return Err(RustCompilerTestError::DroppedEvidence {
2054                test: format!("rustdoc:{}", group.group),
2055                dropped: group.transport.dropped,
2056            });
2057        }
2058        let worker_id = format!("rustdoc-{}", &group.invocation_id[..16]);
2059        let mut accounted_committed = 0_u64;
2060        for joined in &group.entries {
2061            let entry = &joined.catalog;
2062            let test_id = format!("rust:doctest:{}:{}:{}", group.group, entry.file, entry.line);
2063            let attempt_id = format!(
2064                "{run_id}:doctest:{}:{}",
2065                group.invocation_id, joined.catalog_index
2066            );
2067            let (status, error, started, completed) = match &joined.state {
2068                RustdocJoinedOutcomeState::Completed { outcome } => (
2069                    match outcome.status {
2070                        RustdocOutcomeStatus::Passed => "passed",
2071                        RustdocOutcomeStatus::Failed => "failed",
2072                        RustdocOutcomeStatus::Ignored => "skipped",
2073                    },
2074                    (outcome.status == RustdocOutcomeStatus::Failed).then(|| {
2075                        outcome
2076                            .message
2077                            .as_deref()
2078                            .or(outcome.reason.as_deref())
2079                            .unwrap_or("rustdoc reported a failed doctest")
2080                            .to_owned()
2081                    }),
2082                    true,
2083                    true,
2084                ),
2085                RustdocJoinedOutcomeState::UnfinishedStarted => (
2086                    "unknown",
2087                    Some("rustdoc fail-fast ended after this doctest started".into()),
2088                    true,
2089                    false,
2090                ),
2091                RustdocJoinedOutcomeState::Unstarted => (
2092                    "unknown",
2093                    Some("rustdoc fail-fast ended before this doctest started".into()),
2094                    false,
2095                    false,
2096                ),
2097                RustdocJoinedOutcomeState::FilteredOut => ("skipped", None, false, false),
2098                RustdocJoinedOutcomeState::NotRunAmbiguous => (
2099                    "unknown",
2100                    Some(
2101                        "rustdoc did not identify whether this doctest was filtered or left unstarted by fail-fast"
2102                            .into(),
2103                    ),
2104                    false,
2105                    false,
2106                ),
2107            };
2108            let mut phases = started
2109                .then(|| CoveragePhase {
2110                    id: phase_id(run_id, &attempt_id),
2111                    kind: "test".into(),
2112                    operation: format!("Rust doctest {}", entry.name),
2113                    source: Some(entry.file.clone()),
2114                    caused_by_phase_id: None,
2115                    // Pinned libtest reports duration but no wall-clock
2116                    // boundaries. The authenticated rustdoc invocation is the
2117                    // narrowest non-invented interval available; an unfinished
2118                    // test deliberately has no terminal timestamp.
2119                    started_at_ms,
2120                    ended_at_ms: completed.then_some(ended_at_ms),
2121                    status: Some(status.into()),
2122                    error: error.clone(),
2123                })
2124                .into_iter()
2125                .collect::<Vec<_>>();
2126            let (base_context, transport) =
2127                group.attributed_transport(joined).map_err(|error| {
2128                    RustCompilerTestError::Projection {
2129                        test: test_id.clone(),
2130                        reason: error.to_string(),
2131                    }
2132                })?;
2133            accounted_committed = accounted_committed
2134                .checked_add(transport.committed)
2135                .ok_or_else(|| RustCompilerTestError::Projection {
2136                    test: test_id.clone(),
2137                    reason: "rustdoc committed evidence count overflow".into(),
2138                })?;
2139            if !started && transport.committed != 0 {
2140                return Err(RustCompilerTestError::Projection {
2141                    test: test_id,
2142                    reason: "a filtered or unstarted doctest emitted runtime evidence".into(),
2143                });
2144            }
2145            let runtime = if let Some(base_phase) = phases.first() {
2146                let projection = project_rust_compiler_evidence(
2147                    base_context,
2148                    base_phase,
2149                    &transport,
2150                    normalized,
2151                )
2152                .map_err(|error| RustCompilerTestError::Projection {
2153                    test: test_id.clone(),
2154                    reason: error.to_string(),
2155                })?;
2156                if snapshot_has_evidence(&projection.background) {
2157                    return Err(RustCompilerTestError::Projection {
2158                        test: test_id.clone(),
2159                        reason: "doctest context partition retained background evidence".into(),
2160                    });
2161                }
2162                phases.extend(projection.assertion_phases);
2163                vec![projection.attributed]
2164            } else {
2165                Vec::new()
2166            };
2167            results.push(RawTestResult {
2168                test_id: Some(test_id.clone()),
2169                scope: Some(ExecutionScope {
2170                    version: 1,
2171                    run_id: run_id.into(),
2172                    worker_id: worker_id.clone(),
2173                    test_id: test_id.clone(),
2174                    test_key: test_id.clone(),
2175                    retry: 0,
2176                    attempt_id,
2177                }),
2178                test: test_id,
2179                test_file: Some(entry.file.clone()),
2180                title: Some(entry.name.clone()),
2181                retry: Some(0),
2182                status: Some(status.into()),
2183                expected_status: Some("passed".into()),
2184                flaky: false,
2185                provenance: TestProvenance {
2186                    runner: "rustdoc".into(),
2187                    kind: "doctest".into(),
2188                    project: Some(group.group.clone()),
2189                    source: "supercov-rustdoc-outcome".into(),
2190                },
2191                role: "test".into(),
2192                phases,
2193                runtime,
2194                browser: Vec::new(),
2195                server: Vec::new(),
2196            });
2197        }
2198
2199        let background =
2200            group
2201                .background_transport()
2202                .map_err(|error| RustCompilerTestError::Projection {
2203                    test: format!("rustdoc:{}", group.group),
2204                    reason: error.to_string(),
2205                })?;
2206        accounted_committed = accounted_committed
2207            .checked_add(background.committed)
2208            .ok_or_else(|| RustCompilerTestError::Projection {
2209                test: format!("rustdoc:{}", group.group),
2210                reason: "rustdoc background evidence count overflow".into(),
2211            })?;
2212        if accounted_committed != group.transport.committed {
2213            return Err(RustCompilerTestError::Projection {
2214                test: format!("rustdoc:{}", group.group),
2215                reason: format!(
2216                    "rustdoc context partition accounted for {accounted_committed} of {} committed records",
2217                    group.transport.committed
2218                ),
2219            });
2220        }
2221        if background.committed != 0 {
2222            let background_id = format!("background:rustdoc:{}", group.invocation_id);
2223            let background_phase = CoveragePhase {
2224                id: phase_id(run_id, &background_id),
2225                kind: "setup".into(),
2226                operation: format!("Background while running Rust doctests for {}", group.group),
2227                source: None,
2228                caused_by_phase_id: None,
2229                started_at_ms,
2230                ended_at_ms: Some(ended_at_ms),
2231                status: Some("passed".into()),
2232                error: None,
2233            };
2234            // The rustdoc background partition may hold join-bounded
2235            // quarantined thread evidence with real contexts; flatten every
2236            // record to context zero so escaped-thread work can never become
2237            // test or phase attributed downstream.
2238            let mut flattened = background.clone();
2239            for observation in &mut flattened.observations {
2240                observation.context_id = 0;
2241            }
2242            for hit in &mut flattened.ordinal_hits {
2243                hit.context_id = 0;
2244            }
2245            flattened.phases.clear();
2246            flattened.thread_phases.clear();
2247            flattened.thread_ends.clear();
2248            flattened.test_boundaries.clear();
2249            let projection =
2250                project_rust_compiler_evidence(1, &background_phase, &flattened, normalized)
2251                    .map_err(|error| RustCompilerTestError::Projection {
2252                        test: background_id.clone(),
2253                        reason: error.to_string(),
2254                    })?;
2255            if snapshot_has_evidence(&projection.attributed)
2256                || !projection.assertion_phases.is_empty()
2257            {
2258                return Err(RustCompilerTestError::Projection {
2259                    test: background_id,
2260                    reason: "context-zero doctest evidence became test-attributed".into(),
2261                });
2262            }
2263            results.push(RawTestResult {
2264                test_id: Some(background_id.clone()),
2265                scope: Some(ExecutionScope {
2266                    version: 1,
2267                    run_id: run_id.into(),
2268                    worker_id: worker_id.clone(),
2269                    test_id: background_id.clone(),
2270                    test_key: background_id.clone(),
2271                    retry: 0,
2272                    attempt_id: format!("{run_id}:doctest:{}:background", group.invocation_id),
2273                }),
2274                test: background_id,
2275                test_file: None,
2276                title: Some(format!("Background Rust doctest work for {}", group.group)),
2277                retry: Some(0),
2278                status: Some("passed".into()),
2279                expected_status: Some("passed".into()),
2280                flaky: false,
2281                provenance: TestProvenance {
2282                    runner: "rustdoc".into(),
2283                    kind: "doctest".into(),
2284                    project: Some(group.group.clone()),
2285                    source: "supercov-rustdoc-context-zero".into(),
2286                },
2287                role: "background".into(),
2288                phases: Vec::new(),
2289                runtime: vec![projection.background],
2290                browser: Vec::new(),
2291                server: Vec::new(),
2292            });
2293        }
2294        health.push(RustCompilerTransportHealthRecord {
2295            scope_id: format!("rustdoc:{}", group.invocation_id),
2296            scope_kind: "runner-invocation".into(),
2297            status: if group.transport.dropped == 0 && group.transport.incomplete == 0 {
2298                "passed".into()
2299            } else {
2300                "unknown".into()
2301            },
2302            transport: RustCompilerTransportHealth {
2303                committed: group.transport.committed,
2304                incomplete: group.transport.incomplete,
2305                dropped: group.transport.dropped,
2306                attachments: group.transport.attachments,
2307            },
2308            thread_scope_limitations: group.thread_scope_limitations().map_err(|error| {
2309                RustCompilerTestError::Projection {
2310                    test: format!("rustdoc:{}", group.group),
2311                    reason: error.to_string(),
2312                }
2313            })?,
2314        });
2315    }
2316    Ok((results, health))
2317}
2318
2319fn doctest_command_failed(resolution: &RustdocOutcomeResolution) -> bool {
2320    resolution.groups.iter().any(|group| {
2321        group.entries.iter().any(|joined| match &joined.state {
2322            RustdocJoinedOutcomeState::Completed { outcome } => {
2323                outcome.status == RustdocOutcomeStatus::Failed
2324            }
2325            RustdocJoinedOutcomeState::UnfinishedStarted | RustdocJoinedOutcomeState::Unstarted => {
2326                true
2327            }
2328            RustdocJoinedOutcomeState::FilteredOut => false,
2329            RustdocJoinedOutcomeState::NotRunAmbiguous => true,
2330        }) || group.ambiguous_unstarted_tests != 0
2331    })
2332}
2333
2334fn ctfe_raw_results(
2335    run_id: &str,
2336    units: Vec<RustCompilerCtfeUnit>,
2337    started_at_ms: i64,
2338    ended_at_ms: i64,
2339) -> Vec<RawTestResult> {
2340    units
2341        .into_iter()
2342        .enumerate()
2343        .map(|(index, unit)| {
2344            let worker_id = format!("rustc-{index:04}");
2345            let test_id = format!("rust:build:ctfe:{}:{index:04}", unit.crate_name);
2346            let attempt_id = format!("{run_id}:ctfe:{index:04}");
2347            let phase_id = phase_id(run_id, &attempt_id);
2348            let mut snapshot = unit.snapshot;
2349            for event in &mut snapshot.events {
2350                event.phase_id = Some(phase_id.clone());
2351            }
2352            RawTestResult {
2353                test_id: Some(test_id.clone()),
2354                scope: Some(ExecutionScope {
2355                    version: 1,
2356                    run_id: run_id.into(),
2357                    worker_id,
2358                    test_id: test_id.clone(),
2359                    test_key: test_id.clone(),
2360                    retry: 0,
2361                    attempt_id,
2362                }),
2363                test: test_id.clone(),
2364                test_file: None,
2365                title: Some(format!(
2366                    "Compile-time evaluation for {} ({})",
2367                    unit.crate_name, unit.identity
2368                )),
2369                retry: Some(0),
2370                status: Some("passed".into()),
2371                expected_status: Some("passed".into()),
2372                flaky: false,
2373                provenance: TestProvenance {
2374                    runner: "rustc".into(),
2375                    kind: "build".into(),
2376                    project: Some(unit.crate_name),
2377                    source: "supercov-rustc-ctfe".into(),
2378                },
2379                role: "setup".into(),
2380                phases: vec![CoveragePhase {
2381                    id: phase_id,
2382                    kind: "setup".into(),
2383                    operation: "Rust constant evaluation".into(),
2384                    source: None,
2385                    caused_by_phase_id: None,
2386                    started_at_ms,
2387                    ended_at_ms: Some(ended_at_ms),
2388                    status: Some("passed".into()),
2389                    error: None,
2390                }],
2391                runtime: vec![snapshot],
2392                browser: Vec::new(),
2393                server: Vec::new(),
2394            }
2395        })
2396        .collect()
2397}
2398
2399fn status(
2400    outcome: &RustCargoRunnerAttemptOutcome,
2401    output: &SupervisedOutput,
2402) -> (&'static str, i32) {
2403    match outcome {
2404        RustCargoRunnerAttemptOutcome::Libtest {
2405            result: RustLibtestTerminalResult::Passed | RustLibtestTerminalResult::Benchmarked,
2406            ..
2407        } => ("passed", 0),
2408        RustCargoRunnerAttemptOutcome::Libtest {
2409            result: RustLibtestTerminalResult::Ignored,
2410            ..
2411        } => ("skipped", 0),
2412        RustCargoRunnerAttemptOutcome::Libtest {
2413            result: RustLibtestTerminalResult::Failed,
2414            ..
2415        } => ("failed", 101),
2416        RustCargoRunnerAttemptOutcome::Unstarted => ("unstarted", 0),
2417        RustCargoRunnerAttemptOutcome::OpaqueProcess => {
2418            let exit = output.result.exit_code();
2419            (if exit == 0 { "passed" } else { "failed" }, exit)
2420        }
2421    }
2422}
2423
2424fn raw_result(
2425    run_id: &str,
2426    task: &ProcessTask,
2427    status: &str,
2428    base_phase: CoveragePhase,
2429    projection: RustCompilerEvidenceProjection,
2430) -> (RawTestResult, RustCompilerTransportHealthRecord) {
2431    let worker_id = format!("artifact-{:04}", task.artifact_index);
2432    let attempt_id = task.runner_attempt_id.clone();
2433    let runner = match task.runner {
2434        RustCargoRunnerKind::CargoTest => "rust-libtest",
2435        RustCargoRunnerKind::CargoCustomHarness => "rust-custom-harness",
2436        RustCargoRunnerKind::Nextest => "rust-nextest",
2437    };
2438    let scope = ExecutionScope {
2439        version: 1,
2440        run_id: run_id.into(),
2441        worker_id: worker_id.clone(),
2442        test_id: task.test_id.clone(),
2443        test_key: task.test_id.clone(),
2444        retry: task.retry,
2445        attempt_id: attempt_id.clone(),
2446    };
2447    let mut phases = vec![base_phase];
2448    phases.extend(projection.assertion_phases);
2449    let result = RawTestResult {
2450        test_id: Some(task.test_id.clone()),
2451        scope: Some(scope),
2452        test: task.test_id.clone(),
2453        test_file: Some(task.artifact.source.clone()),
2454        title: Some(if task.runner == RustCargoRunnerKind::CargoCustomHarness {
2455            task.artifact.target_key.clone()
2456        } else {
2457            task.test.clone()
2458        }),
2459        retry: Some(task.retry),
2460        status: Some(status.into()),
2461        expected_status: Some("passed".into()),
2462        flaky: false,
2463        provenance: TestProvenance {
2464            runner: runner.into(),
2465            kind: task.artifact.kind.clone(),
2466            project: Some(task.artifact.package.clone()),
2467            source: match task.runner {
2468                RustCargoRunnerKind::CargoTest => "supercov-rustc-stock-libtest-context",
2469                RustCargoRunnerKind::CargoCustomHarness => "supercov-rustc-custom-harness-process",
2470                RustCargoRunnerKind::Nextest => "supercov-rustc-nextest-process",
2471            }
2472            .into(),
2473        },
2474        role: "test".into(),
2475        phases,
2476        runtime: vec![projection.attributed],
2477        browser: Vec::new(),
2478        server: Vec::new(),
2479    };
2480    let health = RustCompilerTransportHealthRecord {
2481        scope_id: task.test_id.clone(),
2482        scope_kind: "test-attempt".into(),
2483        status: status.into(),
2484        transport: projection.health,
2485        thread_scope_limitations: BTreeSet::new(),
2486    };
2487    (result, health)
2488}
2489
2490fn cargo_runner_background_result(
2491    run_id: &str,
2492    artifact_index: usize,
2493    artifact: &TestArtifact,
2494    unit: &RustCargoRunnerUnit,
2495    normalized: &NormalizedRustCompilerManifest,
2496) -> Result<(Option<RawTestResult>, RustCompilerTransportHealthRecord), RustCompilerTestError> {
2497    let transport = &unit.invocation.background_transport;
2498    let background_id = format!("background:rust-runner:{:016}", unit.invocation_ordinal);
2499    let invocation_status = if unit.invocation.result.exit_code() == 0 {
2500        "passed"
2501    } else {
2502        "failed"
2503    };
2504    let base_phase = CoveragePhase {
2505        id: phase_id(run_id, &background_id),
2506        kind: "setup".into(),
2507        operation: format!("Background while running {}", artifact.target_key),
2508        source: Some(artifact.source.clone()),
2509        caused_by_phase_id: None,
2510        started_at_ms: unit.invocation.started_at_ms,
2511        ended_at_ms: Some(unit.invocation.ended_at_ms),
2512        status: Some(invocation_status.into()),
2513        error: None,
2514    };
2515    // The persisted background partition holds context-zero records plus
2516    // join-bounded quarantined thread evidence that keeps its real contexts.
2517    // Projection deliberately flattens every record to context zero under a
2518    // non-reserved synthetic base: the shared projector still validates probe
2519    // identities while escaped-thread work can never become test or phase
2520    // attributed downstream.
2521    let mut flattened = transport.clone();
2522    for observation in &mut flattened.observations {
2523        observation.context_id = 0;
2524    }
2525    for hit in &mut flattened.ordinal_hits {
2526        hit.context_id = 0;
2527    }
2528    flattened.phases.clear();
2529    flattened.thread_phases.clear();
2530    flattened.thread_ends.clear();
2531    flattened.test_boundaries.clear();
2532    let projection = project_rust_compiler_evidence(1, &base_phase, &flattened, normalized)
2533        .map_err(|error| RustCompilerTestError::Projection {
2534            test: background_id.clone(),
2535            reason: error.to_string(),
2536        })?;
2537    if snapshot_has_evidence(&projection.attributed) || !projection.assertion_phases.is_empty() {
2538        return Err(RustCompilerTestError::Projection {
2539            test: background_id,
2540            reason: "context-zero Cargo evidence became test-attributed".into(),
2541        });
2542    }
2543    let runner = match unit.runner {
2544        RustCargoRunnerKind::CargoTest => "rust-libtest",
2545        RustCargoRunnerKind::CargoCustomHarness => "rust-custom-harness",
2546        RustCargoRunnerKind::Nextest => "rust-nextest",
2547    };
2548    let result = snapshot_has_evidence(&projection.background).then(|| RawTestResult {
2549        test_id: Some(background_id.clone()),
2550        scope: Some(ExecutionScope {
2551            version: 1,
2552            run_id: run_id.into(),
2553            worker_id: format!("artifact-{artifact_index:04}"),
2554            test_id: background_id.clone(),
2555            test_key: background_id.clone(),
2556            retry: 0,
2557            attempt_id: format!("{run_id}:cargo:{:016}:background", unit.invocation_ordinal),
2558        }),
2559        test: background_id.clone(),
2560        test_file: Some(artifact.source.clone()),
2561        title: Some(format!("Background Rust work for {}", artifact.target_key)),
2562        retry: Some(0),
2563        status: Some(invocation_status.into()),
2564        expected_status: Some("passed".into()),
2565        flaky: false,
2566        provenance: TestProvenance {
2567            runner: runner.into(),
2568            kind: artifact.kind.clone(),
2569            project: Some(artifact.package.clone()),
2570            source: "supercov-rustc-context-zero".into(),
2571        },
2572        role: "background".into(),
2573        phases: Vec::new(),
2574        runtime: vec![projection.background],
2575        browser: Vec::new(),
2576        server: Vec::new(),
2577    });
2578    Ok((
2579        result,
2580        RustCompilerTransportHealthRecord {
2581            scope_id: background_id,
2582            scope_kind: "runner-invocation".into(),
2583            status: invocation_status.into(),
2584            transport: projection.health,
2585            thread_scope_limitations: unit.thread_scope_limitations.clone(),
2586        },
2587    ))
2588}
2589
2590pub fn run_rust_compiler_frontend(
2591    request: &RustCompilerRunRequest,
2592    diagnostics: &mut dyn Write,
2593) -> Result<RustCompilerFrontendRun, RustCompilerTestError> {
2594    let supervisor = request
2595        .watchdog_program
2596        .as_deref()
2597        .map_or_else(ProcessSupervisor::new, ProcessSupervisor::new_crash_safe)
2598        .map_err(|error| RustCompilerTestError::Build(error.to_string()))?;
2599    let options = SupervisionOptions::from_environment()
2600        .map_err(|error| RustCompilerTestError::Build(error.to_string()))?;
2601    let build = crate::rust_compiler_orchestration::build_with_rust_compiler_companion_supervised(
2602        &request.build_request(),
2603        &supervisor,
2604        options,
2605        diagnostics,
2606    )
2607    .map_err(|error| {
2608        match error {
2609        crate::rust_compiler_orchestration::RustCompilerOrchestrationError::Interrupted {
2610            code,
2611            signal,
2612        } => RustCompilerTestError::Interrupted { code, signal },
2613        crate::rust_compiler_orchestration::RustCompilerOrchestrationError::UnverifiedExecution {
2614            code,
2615            reason,
2616        } => RustCompilerTestError::UnverifiedExecution { code, reason },
2617        error => RustCompilerTestError::Build(error.to_string()),
2618    }
2619    })?;
2620    execute_compiler_build(request, build, diagnostics)
2621}
2622
2623fn execute_compiler_build(
2624    request: &RustCompilerRunRequest,
2625    build: RustCompilerBuild,
2626    diagnostics: &mut dyn Write,
2627) -> Result<RustCompilerFrontendRun, RustCompilerTestError> {
2628    let project_root = fs::canonicalize(&request.project_root)
2629        .map_err(|error| io_error(&request.project_root, error))?;
2630    let artifacts = normalize_artifacts(&project_root, &build.target_directory, &build.artifacts)?;
2631    let artifact_by_path = artifacts
2632        .iter()
2633        .enumerate()
2634        .map(|(index, artifact)| (artifact.executable.clone(), (index, artifact.clone())))
2635        .collect::<BTreeMap<_, _>>();
2636    let nextest_version = match build.command_kind {
2637        crate::rust_test_runner::RustCargoCommandKind::NextestRun => {
2638            Some(build.nextest_version.as_deref().ok_or_else(|| {
2639                RustCompilerTestError::Context(
2640                    "nextest execution lacks its authenticated version handshake".into(),
2641                )
2642            })?)
2643        }
2644        crate::rust_test_runner::RustCargoCommandKind::CargoTest => {
2645            if build.nextest_version.is_some() || build.nextest_catalog.is_some() {
2646                return Err(RustCompilerTestError::Context(
2647                    "standard Cargo execution carries foreign nextest preflight state".into(),
2648                ));
2649            }
2650            None
2651        }
2652    };
2653    let mut nextest_selected =
2654        BTreeMap::<(PathBuf, String), (String, usize, TestArtifact, String)>::new();
2655    let mut nextest_selected_ids = BTreeSet::new();
2656    let mut nextest_binary_by_artifact = BTreeMap::<PathBuf, String>::new();
2657    if let Some(catalog) = &build.nextest_catalog {
2658        if build.command_kind != crate::rust_test_runner::RustCargoCommandKind::NextestRun {
2659            return Err(RustCompilerTestError::Context(
2660                "a nextest catalog was attached to a non-nextest build".into(),
2661            ));
2662        }
2663        for (binary_id, suite) in &catalog.rust_suites {
2664            if suite.status != RustTestSuiteStatusSummary::LISTED && !suite.test_cases.is_empty() {
2665                return Err(RustCompilerTestError::Context(format!(
2666                    "nextest skipped suite {binary_id} contains test cases"
2667                )));
2668            }
2669            let executable = fs::canonicalize(suite.binary.binary_path.as_std_path())
2670                .map_err(|error| io_error(suite.binary.binary_path.as_std_path(), error))?;
2671            let (artifact_index, artifact) =
2672                artifact_by_path.get(&executable).ok_or_else(|| {
2673                    RustCompilerTestError::Context(format!(
2674                        "nextest catalog contains an unknown artifact: {}",
2675                        executable.display()
2676                    ))
2677                })?;
2678            if nextest_binary_by_artifact
2679                .insert(executable.clone(), binary_id.to_string())
2680                .is_some()
2681            {
2682                return Err(RustCompilerTestError::Context(
2683                    "nextest catalog aliases two binary identities to one artifact".into(),
2684                ));
2685            }
2686            let compilation_target = match suite.binary.build_platform {
2687                BuildPlatform::Host => catalog
2688                    .rust_build_meta
2689                    .platforms
2690                    .as_ref()
2691                    .map(|platforms| platforms.host.platform.triple.as_str()),
2692                BuildPlatform::Target => {
2693                    catalog
2694                        .rust_build_meta
2695                        .platforms
2696                        .as_ref()
2697                        .and_then(|platforms| {
2698                            if platforms.targets.len() > 1 {
2699                                None
2700                            } else {
2701                                platforms
2702                                    .targets
2703                                    .first()
2704                                    .map(|target| target.platform.triple.as_str())
2705                                    .or(Some(platforms.host.platform.triple.as_str()))
2706                            }
2707                        })
2708                }
2709            }
2710            .ok_or_else(|| {
2711                RustCompilerTestError::Context(format!(
2712                    "nextest binary {binary_id} lacks one exact compilation target"
2713                ))
2714            })?
2715            .to_owned();
2716            if !request
2717                .cargo_runner_plan
2718                .targets
2719                .iter()
2720                .any(|target| target.target == compilation_target)
2721            {
2722                return Err(RustCompilerTestError::Context(format!(
2723                    "nextest binary {binary_id} uses unselected target {compilation_target}"
2724                )));
2725            }
2726            for (test, summary) in &suite.test_cases {
2727                if summary.kind.is_none() {
2728                    return Err(RustCompilerTestError::Context(format!(
2729                        "nextest catalog test {binary_id}::{test} lacks a test kind"
2730                    )));
2731                }
2732                if summary.filter_match == FilterMatch::Matches {
2733                    let test = test.to_string();
2734                    let test_id = libtest_id(&compilation_target, artifact, &test);
2735                    if !nextest_selected_ids.insert(test_id.clone()) {
2736                        return Err(RustCompilerTestError::DuplicateTest(test_id));
2737                    }
2738                    if nextest_selected
2739                        .insert(
2740                            (executable.clone(), test.clone()),
2741                            (
2742                                compilation_target.clone(),
2743                                *artifact_index,
2744                                artifact.clone(),
2745                                test,
2746                            ),
2747                        )
2748                        .is_some()
2749                    {
2750                        return Err(RustCompilerTestError::DuplicateTest(test_id));
2751                    }
2752                }
2753            }
2754        }
2755    } else if build.command_kind == crate::rust_test_runner::RustCargoCommandKind::NextestRun {
2756        return Err(RustCompilerTestError::Context(
2757            "nextest execution lacks its exact selected-test catalog".into(),
2758        ));
2759    }
2760    let mut outcomes = Vec::new();
2761    let mut identities = BTreeSet::new();
2762    let mut attempt_ids = BTreeSet::new();
2763    let mut nextest_attempted = BTreeSet::new();
2764    for unit in &build.cargo_runner_units {
2765        let (artifact_index, artifact) = artifact_by_path.get(&unit.artifact).ok_or_else(|| {
2766            RustCompilerTestError::Context(format!(
2767                "Cargo runner executed an unknown artifact: {}",
2768                unit.artifact.display()
2769            ))
2770        })?;
2771        let tests = unit
2772            .attempts
2773            .iter()
2774            .map(|attempt| attempt.test.clone())
2775            .collect::<Vec<_>>();
2776        let contexts = preflight_rust_test_contexts(tests)
2777            .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
2778        for (test_index, attempt) in unit.attempts.iter().enumerate() {
2779            if contexts[&attempt.test] != attempt.context_id {
2780                return Err(RustCompilerTestError::Context(format!(
2781                    "Cargo runner context changed for {}",
2782                    attempt.test
2783                )));
2784            }
2785            let test_id = match unit.runner {
2786                RustCargoRunnerKind::CargoCustomHarness => {
2787                    custom_harness_id(&unit.target, artifact)
2788                }
2789                RustCargoRunnerKind::CargoTest | RustCargoRunnerKind::Nextest => {
2790                    libtest_id(&unit.target, artifact, &attempt.test)
2791                }
2792            };
2793            if unit.runner == RustCargoRunnerKind::Nextest {
2794                if unit.runner_version.as_deref() != nextest_version {
2795                    return Err(RustCompilerTestError::Context(format!(
2796                        "nextest target-runner version disagrees with the authenticated outer handshake for {test_id}"
2797                    )));
2798                }
2799                let expected_binary =
2800                    nextest_binary_by_artifact
2801                        .get(&unit.artifact)
2802                        .ok_or_else(|| {
2803                            RustCompilerTestError::Context(format!(
2804                                "nextest executed an uncatalogued artifact: {}",
2805                                unit.artifact.display()
2806                            ))
2807                        })?;
2808                if unit.runner_binary_id.as_deref() != Some(expected_binary.as_str()) {
2809                    return Err(RustCompilerTestError::Context(format!(
2810                        "nextest runner binary identity disagrees with the selected-test catalog for {}",
2811                        unit.artifact.display()
2812                    )));
2813                }
2814                if !nextest_selected.contains_key(&(unit.artifact.clone(), attempt.test.clone())) {
2815                    return Err(RustCompilerTestError::Context(format!(
2816                        "nextest executed a test excluded by its machine-readable catalog: {test_id}"
2817                    )));
2818                }
2819                nextest_attempted.insert(test_id.clone());
2820            }
2821            if !identities.insert((test_id.clone(), attempt.retry)) {
2822                return Err(RustCompilerTestError::DuplicateTest(format!(
2823                    "{test_id} retry {}",
2824                    attempt.retry
2825                )));
2826            }
2827            if !attempt_ids.insert(attempt.runner_attempt_id.clone()) {
2828                return Err(RustCompilerTestError::Context(format!(
2829                    "Cargo runner attempt ID is duplicated: {}",
2830                    attempt.runner_attempt_id
2831                )));
2832            }
2833            let task = ProcessTask {
2834                ordinal: outcomes.len(),
2835                artifact_index: *artifact_index,
2836                artifact: artifact.clone(),
2837                test: attempt.test.clone(),
2838                test_id,
2839                context_id: attempt.context_id,
2840                retry: attempt.retry,
2841                total_attempts: attempt.total_attempts,
2842                runner_attempt_id: attempt.runner_attempt_id.clone(),
2843                runner: unit.runner,
2844                transport: build.compiler_output_directory.join(format!(
2845                    "cargo-runner/libtest-{:04}-{test_index:08}.json",
2846                    unit.invocation_ordinal
2847                )),
2848                libtest_events: build.compiler_output_directory.join(format!(
2849                    "cargo-runner/libtest-{:04}-{test_index:08}.events",
2850                    unit.invocation_ordinal
2851                )),
2852                test_arguments: Vec::new(),
2853                underlying_runner: None,
2854            };
2855            outcomes.push(ProcessOutcome {
2856                task,
2857                output: SupervisedOutput {
2858                    result: unit.invocation.result.clone(),
2859                    stdout: unit.invocation.stdout.clone(),
2860                    stderr: unit.invocation.stderr.clone(),
2861                },
2862                read: attempt.transport.clone(),
2863                attempt_outcome: attempt.outcome.clone(),
2864                started_at_ms: unit.invocation.started_at_ms,
2865                ended_at_ms: unit.invocation.ended_at_ms,
2866            });
2867        }
2868    }
2869    let standard_cargo_units = build
2870        .cargo_runner_units
2871        .iter()
2872        .filter(|unit| {
2873            matches!(
2874                unit.runner,
2875                RustCargoRunnerKind::CargoTest | RustCargoRunnerKind::CargoCustomHarness
2876            )
2877        })
2878        .count();
2879    if build.execution_exit_code == 0
2880        && build.run_libtests
2881        && standard_cargo_units != 0
2882        && standard_cargo_units != artifacts.len()
2883    {
2884        return Err(RustCompilerTestError::Context(format!(
2885            "Cargo completed successfully but published {} runner unit(s) for {} artifact(s)",
2886            standard_cargo_units,
2887            artifacts.len()
2888        )));
2889    }
2890    let mut nextest_attempt_groups = BTreeMap::<String, Vec<&ProcessOutcome>>::new();
2891    for outcome in &outcomes {
2892        if outcome.task.runner == RustCargoRunnerKind::Nextest {
2893            nextest_attempt_groups
2894                .entry(outcome.task.test_id.clone())
2895                .or_default()
2896                .push(outcome);
2897        }
2898    }
2899    for attempts in nextest_attempt_groups.values_mut() {
2900        attempts.sort_by_key(|outcome| outcome.task.retry);
2901    }
2902    let nextest_terminal_failure = nextest_attempt_groups.values().any(|attempts| {
2903        attempts
2904            .last()
2905            .is_some_and(|outcome| outcome.output.result.exit_code() != 0)
2906    });
2907    let nextest_flaky = nextest_attempt_groups.values().any(|attempts| {
2908        attempts
2909            .last()
2910            .is_some_and(|outcome| outcome.output.result.exit_code() == 0)
2911            && attempts
2912                .iter()
2913                .take(attempts.len().saturating_sub(1))
2914                .any(|outcome| outcome.output.result.exit_code() != 0)
2915    });
2916    let nextest_unstarted = nextest_selected
2917        .values()
2918        .filter(|(target, _, artifact, test)| {
2919            !nextest_attempted.contains(&libtest_id(target, artifact, test))
2920        })
2921        .cloned()
2922        .collect::<Vec<_>>();
2923
2924    let mut raw_results = ctfe_raw_results(
2925        &request.run_id,
2926        build.ctfe_units.clone(),
2927        build.build_started_at_ms,
2928        build.build_ended_at_ms,
2929    );
2930    let (doctest_results, mut transport_health) = doctest_raw_results(
2931        &request.run_id,
2932        &build.doctest_outcomes,
2933        build.build_started_at_ms,
2934        build.build_ended_at_ms,
2935        &build.normalized,
2936    )?;
2937    raw_results.extend(doctest_results);
2938    for unit in &build.cargo_runner_units {
2939        let (artifact_index, artifact) = artifact_by_path.get(&unit.artifact).ok_or_else(|| {
2940            RustCompilerTestError::Context(format!(
2941                "Cargo runner published background evidence for an unknown artifact: {}",
2942                unit.artifact.display()
2943            ))
2944        })?;
2945        let (background, health) = cargo_runner_background_result(
2946            &request.run_id,
2947            *artifact_index,
2948            artifact,
2949            unit,
2950            &build.normalized,
2951        )?;
2952        if let Some(background) = background {
2953            raw_results.push(background);
2954        }
2955        transport_health.push(health);
2956    }
2957    let overall_exit = build.execution_exit_code;
2958    if overall_exit != 0 {
2959        diagnostics
2960            .write_all(&build.execution_stdout)
2961            .and_then(|_| diagnostics.write_all(&build.execution_stderr))
2962            .map_err(|error| RustCompilerTestError::Io {
2963                path: build.compiler_output_directory.clone(),
2964                reason: error.to_string(),
2965            })?;
2966    }
2967    match build.command_kind {
2968        crate::rust_test_runner::RustCargoCommandKind::CargoTest => {
2969            let authenticated_failure = doctest_command_failed(&build.doctest_outcomes)
2970                || outcomes
2971                    .iter()
2972                    .any(|outcome| outcome.output.result.exit_code() != 0);
2973            if (overall_exit != 0) != authenticated_failure {
2974                return Err(RustCompilerTestError::Context(
2975                    "Cargo exit status disagrees with authenticated libtest/doctest outcomes"
2976                        .into(),
2977                ));
2978            }
2979        }
2980        crate::rust_test_runner::RustCargoCommandKind::NextestRun => match overall_exit {
2981            NextestExitCode::OK => {
2982                if nextest_terminal_failure || !nextest_unstarted.is_empty() {
2983                    return Err(RustCompilerTestError::Context(
2984                        "nextest exited successfully without terminal successful attempts for every selected test"
2985                            .into(),
2986                    ));
2987                }
2988            }
2989            NextestExitCode::TEST_RUN_FAILED => {
2990                if !nextest_terminal_failure && !nextest_flaky {
2991                    return Err(RustCompilerTestError::Context(
2992                        "nextest reported test failure without an authenticated terminal failure or flaky attempt sequence"
2993                            .into(),
2994                    ));
2995                }
2996            }
2997            NextestExitCode::NO_TESTS_RUN if nextest_selected.is_empty() => {}
2998            code => {
2999                return Err(RustCompilerTestError::UnverifiedExecution {
3000                    code,
3001                    reason: "nextest returned an infrastructure/status code that is not authenticated by test attempts"
3002                        .into(),
3003                });
3004            }
3005        },
3006    }
3007    for outcome in outcomes {
3008        let (test_status, exit) = status(&outcome.attempt_outcome, &outcome.output);
3009        if outcome.read.dropped != 0 {
3010            return Err(RustCompilerTestError::DroppedEvidence {
3011                test: outcome.task.test_id,
3012                dropped: outcome.read.dropped,
3013            });
3014        }
3015        let attempt_id = outcome.task.runner_attempt_id.clone();
3016        let base_phase = CoveragePhase {
3017            id: phase_id(&request.run_id, &attempt_id),
3018            kind: "test".into(),
3019            operation: format!(
3020                "{} {}",
3021                match outcome.task.runner {
3022                    RustCargoRunnerKind::CargoTest => "Rust libtest",
3023                    RustCargoRunnerKind::CargoCustomHarness => "Rust custom harness",
3024                    RustCargoRunnerKind::Nextest => "Rust nextest test",
3025                },
3026                if outcome.task.runner == RustCargoRunnerKind::CargoCustomHarness {
3027                    &outcome.task.artifact.target_key
3028                } else {
3029                    &outcome.task.test
3030                }
3031            ),
3032            source: Some(outcome.task.artifact.source.clone()),
3033            caused_by_phase_id: None,
3034            started_at_ms: outcome.started_at_ms,
3035            ended_at_ms: Some(outcome.ended_at_ms),
3036            status: Some(test_status.into()),
3037            error: (exit != 0).then(|| {
3038                String::from_utf8_lossy(&outcome.output.stderr)
3039                    .trim()
3040                    .to_owned()
3041            }),
3042        };
3043        let projection = project_rust_compiler_evidence(
3044            outcome.task.context_id,
3045            &base_phase,
3046            &outcome.read,
3047            &build.normalized,
3048        )
3049        .map_err(|error| RustCompilerTestError::Projection {
3050            test: outcome.task.test_id.clone(),
3051            reason: error.to_string(),
3052        })?;
3053        if snapshot_has_evidence(&projection.background) {
3054            return Err(RustCompilerTestError::Projection {
3055                test: outcome.task.test_id,
3056                reason: "a persisted test partition retained context-zero evidence".into(),
3057            });
3058        }
3059        let (result, health) = raw_result(
3060            &request.run_id,
3061            &outcome.task,
3062            test_status,
3063            base_phase,
3064            projection,
3065        );
3066        raw_results.push(result);
3067        transport_health.push(health);
3068    }
3069    for (target, _artifact_index, artifact, test) in nextest_unstarted {
3070        let test_id = libtest_id(&target, &artifact, &test);
3071        raw_results.push(RawTestResult {
3072            test_id: Some(test_id.clone()),
3073            scope: None,
3074            test: test_id,
3075            test_file: Some(artifact.source.clone()),
3076            title: Some(test),
3077            retry: None,
3078            status: Some("unstarted".into()),
3079            expected_status: Some("passed".into()),
3080            flaky: false,
3081            provenance: TestProvenance {
3082                runner: "rust-nextest".into(),
3083                kind: artifact.kind,
3084                project: Some(artifact.package),
3085                source: "nextest-selected-but-not-started".into(),
3086            },
3087            role: "test".into(),
3088            phases: Vec::new(),
3089            runtime: Vec::new(),
3090            browser: Vec::new(),
3091            server: Vec::new(),
3092        });
3093    }
3094
3095    let mut structural_limitations = build
3096        .normalized
3097        .manifest
3098        .limitations
3099        .iter()
3100        .filter_map(|limitation| {
3101            limitation
3102                .get("id")
3103                .and_then(|id| id.as_str())
3104                .map(str::to_owned)
3105        })
3106        .collect::<Vec<_>>();
3107    if !build.doctest_outcomes.is_fully_catalogued() {
3108        structural_limitations.push("rust-doctest-outcome-catalog-incomplete".into());
3109    }
3110    if build.doctest_outcomes.has_ambiguous_outcomes() {
3111        structural_limitations.push("rust-doctest-filter-fail-fast-identity-ambiguous".into());
3112    }
3113    structural_limitations.sort();
3114    structural_limitations.dedup();
3115    let observed_runners = raw_results
3116        .iter()
3117        .map(|result| result.provenance.runner.as_str())
3118        .collect::<BTreeSet<_>>();
3119    if observed_runners.iter().any(|runner| {
3120        !matches!(
3121            *runner,
3122            "rustc" | "rust-libtest" | "rust-custom-harness" | "rust-nextest" | "rustdoc"
3123        )
3124    }) {
3125        return Err(RustCompilerTestError::Context(
3126            "Rust compiler run produced an unknown runner identity".into(),
3127        ));
3128    }
3129    // The frontend contract declares capabilities actually present in this
3130    // run. In particular, `cargo test --doc` must not advertise an unobserved
3131    // libtest runner, and an explicit non-doc target must not advertise
3132    // rustdoc. The shared analyzer deliberately rejects such declarations.
3133    let runners = [
3134        ("rustc", compiler_runner_declaration()),
3135        ("rust-libtest", runner_declaration()),
3136        ("rust-custom-harness", custom_harness_runner_declaration()),
3137        ("rust-nextest", nextest_runner_declaration()),
3138        ("rustdoc", rustdoc_runner_declaration()),
3139    ]
3140    .into_iter()
3141    .filter_map(|(name, declaration)| observed_runners.contains(name).then_some(declaration))
3142    .collect();
3143    let declaration = FrontendRunDeclaration {
3144        protocol_version: LANGUAGE_FRONTEND_PROTOCOL_VERSION,
3145        frontend_id: "rust".into(),
3146        frontend_version: "rust-compiler-v1".into(),
3147        language: "rust".into(),
3148        structural_source: StructuralSource::OwnedProbes,
3149        runners,
3150        structural_limitations,
3151    };
3152    Ok(RustCompilerFrontendRun {
3153        selection: build.selection,
3154        declaration,
3155        request: CoverageReportRequest {
3156            run_id: request.run_id.clone(),
3157            manifest: build.normalized.manifest,
3158            raw_results,
3159            generated_at: request.generated_at.clone(),
3160            coverage_model: Some(rust_compiler_coverage_model()),
3161            integrity: None,
3162            test_exit_code: ExitCodeInput::Present(Some(overall_exit)),
3163        },
3164        exit_code: overall_exit,
3165        artifacts: artifacts.len(),
3166        artifact_files: artifacts
3167            .into_iter()
3168            .map(|artifact| artifact.executable)
3169            .collect(),
3170        transport_health,
3171        build_ms: build.build_ms,
3172        execution_ms: build.execution_ms,
3173    })
3174}
3175
3176#[cfg(test)]
3177mod tests {
3178    use super::*;
3179    use crate::{
3180        coverage_analysis::PointKind,
3181        coverage_report::{CoverageManifest, PointMeta},
3182        rust_doctest::{
3183            RustdocDoctestAttributes, RustdocDoctestCode, RustdocDoctestIgnore,
3184            RustdocDoctestWrapper, RustdocExtractedDoctest, RustdocJoinedOutcome,
3185            RustdocMergedEntry, RustdocOutcomeGroupJoin, RustdocTestOutcome,
3186        },
3187        rust_probe_transport::RustOrdinalHit,
3188    };
3189
3190    fn test_transport() -> RustTransportRead {
3191        RustTransportRead::empty()
3192    }
3193
3194    fn test_invocation(status: i32) -> RustCargoRunnerInvocation {
3195        RustCargoRunnerInvocation {
3196            result: SupervisedResult {
3197                status: Some(status),
3198                signal: None,
3199                timed_out: false,
3200                interrupted_signal: None,
3201            },
3202            started_at_ms: 0,
3203            ended_at_ms: 1,
3204            stdout: Vec::new(),
3205            stderr: Vec::new(),
3206            background_transport: test_transport(),
3207        }
3208    }
3209
3210    fn test_runner_unit() -> RustCargoRunnerUnit {
3211        RustCargoRunnerUnit {
3212            version: RUST_CARGO_RUNNER_VERSION,
3213            run_id: "run_0123456789abcdef".into(),
3214            invocation_ordinal: 0,
3215            runner: RustCargoRunnerKind::CargoTest,
3216            runner_run_id: None,
3217            runner_version: None,
3218            runner_binary_id: None,
3219            target: "aarch64-apple-darwin".into(),
3220            artifact: PathBuf::from("target/test-artifact"),
3221            arguments: Vec::new(),
3222            invocation: test_invocation(0),
3223            attempts: Vec::new(),
3224            thread_scope_limitations: BTreeSet::new(),
3225        }
3226    }
3227
3228    #[test]
3229    fn persisted_runner_transport_rejects_cross_partition_and_count_tampering() {
3230        let mut unit = test_runner_unit();
3231        unit.attempts.push(RustCargoRunnerAttempt {
3232            test: "tests::one".into(),
3233            context_id: 7,
3234            retry: 0,
3235            total_attempts: 1,
3236            runner_attempt_id: "attempt-one".into(),
3237            outcome: RustCargoRunnerAttemptOutcome::Libtest {
3238                result: RustLibtestTerminalResult::Passed,
3239                timed_out: false,
3240            },
3241            transport: test_transport(),
3242        });
3243        validate_persisted_runner_transport(&unit).unwrap();
3244
3245        unit.attempts[0]
3246            .transport
3247            .ordinal_hits
3248            .push(RustOrdinalHit {
3249                process_id: 1,
3250                context_id: 0,
3251                ordinal: 9,
3252            });
3253        unit.attempts[0].transport.committed = 1;
3254        assert!(
3255            validate_persisted_runner_transport(&unit)
3256                .unwrap_err()
3257                .to_string()
3258                .contains("exact transport partition")
3259        );
3260
3261        unit.attempts[0].transport.ordinal_hits[0].context_id = 7;
3262        unit.attempts[0].transport.committed = 2;
3263        assert!(
3264            validate_persisted_runner_transport(&unit)
3265                .unwrap_err()
3266                .to_string()
3267                .contains("persisted attribution")
3268        );
3269    }
3270
3271    #[test]
3272    fn empty_selected_suite_publishes_invocation_background_without_a_test_attempt() {
3273        let point_id = "rs:statement:111111111111111111111111";
3274        let mut unit = test_runner_unit();
3275        unit.invocation.background_transport = RustTransportRead {
3276            ordinal_hits: vec![RustOrdinalHit {
3277                process_id: 1,
3278                context_id: 0,
3279                ordinal: 9,
3280            }],
3281            committed: 1,
3282            attachments: 1,
3283            ..RustTransportRead::empty()
3284        };
3285        validate_persisted_runner_transport(&unit).unwrap();
3286        let normalized = NormalizedRustCompilerManifest {
3287            manifest: CoverageManifest {
3288                unmeasured: Vec::new(),
3289                decisions: Vec::new(),
3290                points: vec![PointMeta {
3291                    id: point_id.into(),
3292                    kind: PointKind::Statement,
3293                    file: "src/lib.rs".into(),
3294                    line: 1,
3295                    column: 1,
3296                    source: "setup();".into(),
3297                    label: None,
3298                }],
3299                branches: Vec::new(),
3300                limitations: Vec::new(),
3301                scope: None,
3302            },
3303            hit_obligations_by_ordinal: BTreeMap::from([(9, vec![point_id.into()])]),
3304            internal_ordinals: BTreeSet::new(),
3305            decision_outcome_obligations: BTreeMap::new(),
3306            decision_loop_obligations: BTreeMap::new(),
3307            decision_logical_selection_obligations: BTreeMap::new(),
3308        };
3309        let artifact = TestArtifact {
3310            executable: unit.artifact.clone(),
3311            runner_argument: None,
3312            package: "package:.".into(),
3313            target_key: "lib:fixture".into(),
3314            kind: "unit".into(),
3315            source: "src/lib.rs".into(),
3316            test_harness: true,
3317        };
3318        let (result, health) =
3319            cargo_runner_background_result(&unit.run_id, 0, &artifact, &unit, &normalized).unwrap();
3320        let result = result.expect("context-zero evidence must remain queryable");
3321        assert_eq!(result.role, "background");
3322        assert!(result.runtime[0].hits.iter().any(|hit| hit == point_id));
3323        assert_eq!(health.scope_kind, "runner-invocation");
3324        assert_eq!(health.transport.committed, 1);
3325    }
3326
3327    #[test]
3328    fn tokens_and_phase_ids_are_fixed_width_and_domain_separated() {
3329        assert_eq!(token_hex(&[0xab; TOKEN_BYTES]), "ab".repeat(TOKEN_BYTES));
3330        let first = phase_id("run-a", "attempt");
3331        assert_eq!(first.len(), "rust-test:".len() + 40);
3332        assert_ne!(first, phase_id("run-b", "attempt"));
3333        assert_ne!(first, phase_id("run-a", "attempt-b"));
3334    }
3335
3336    #[test]
3337    fn test_identities_include_runner_package_target_and_workspace_source() {
3338        let artifact = |package: &str, target_key: &str| TestArtifact {
3339            executable: PathBuf::from("test-artifact"),
3340            runner_argument: None,
3341            package: package.into(),
3342            target_key: target_key.into(),
3343            kind: "unit".into(),
3344            source: "shared/src/lib.rs".into(),
3345            test_harness: true,
3346        };
3347        let root = artifact("package:.", "lib:same");
3348        let sibling = artifact("package:crates/sibling", "lib:same");
3349        let integration = artifact("package:.", "test:same");
3350        assert_eq!(
3351            libtest_id("aarch64-apple-darwin", &root, "tests::same_name"),
3352            "rust:libtest:aarch64-apple-darwin:package:.:lib:same:shared/src/lib.rs::tests::same_name"
3353        );
3354        assert_ne!(
3355            libtest_id("aarch64-apple-darwin", &root, "tests::same_name"),
3356            libtest_id("aarch64-apple-darwin", &sibling, "tests::same_name")
3357        );
3358        assert_ne!(
3359            libtest_id("aarch64-apple-darwin", &root, "tests::same_name"),
3360            libtest_id("aarch64-apple-darwin", &integration, "tests::same_name")
3361        );
3362        assert_ne!(
3363            libtest_id("aarch64-apple-darwin", &root, "tests::same_name"),
3364            libtest_id("x86_64-apple-darwin", &root, "tests::same_name")
3365        );
3366        assert_eq!(
3367            custom_harness_id("aarch64-apple-darwin", &integration),
3368            "rust:custom-harness:aarch64-apple-darwin:package:.:test:same:shared/src/lib.rs"
3369        );
3370        assert_ne!(
3371            custom_harness_id("aarch64-apple-darwin", &integration),
3372            libtest_id("aarch64-apple-darwin", &integration, "custom-harness")
3373        );
3374    }
3375
3376    #[test]
3377    fn cargo_runner_failure_units_are_atomic_and_diagnostic() {
3378        let root = std::env::temp_dir().join(format!(
3379            "supercov-cargo-runner-failure-{}-{}",
3380            std::process::id(),
3381            SystemTime::now()
3382                .duration_since(UNIX_EPOCH)
3383                .unwrap()
3384                .as_nanos()
3385        ));
3386        fs::create_dir(&root).unwrap();
3387        let ordinal = reserve_cargo_runner_ordinal(&root).unwrap();
3388        assert_eq!(ordinal, 0);
3389        let failure = RustCargoRunnerFailure {
3390            version: RUST_CARGO_RUNNER_VERSION,
3391            run_id: "run_0123456789abcdef".into(),
3392            invocation_ordinal: ordinal,
3393            target: Some("aarch64-apple-darwin".into()),
3394            artifact: Some(PathBuf::from("target/test-artifact")),
3395            error: "deliberate runner failure".into(),
3396        };
3397        let published = write_cargo_runner_failure(&root, &failure).unwrap();
3398        assert_eq!(
3399            published.file_name().unwrap(),
3400            "failure-0000000000000000.json"
3401        );
3402        assert!(fs::read_dir(&root).unwrap().all(|entry| {
3403            !entry
3404                .unwrap()
3405                .file_name()
3406                .to_string_lossy()
3407                .ends_with(".partial")
3408        }));
3409        let error = read_cargo_runner_units(
3410            &root,
3411            "run_0123456789abcdef",
3412            &["aarch64-apple-darwin".into()],
3413        )
3414        .unwrap_err();
3415        let error = error.to_string();
3416        assert!(error.contains("invocation 0 failed"), "{error}");
3417        assert!(error.contains("deliberate runner failure"), "{error}");
3418        fs::remove_dir_all(root).unwrap();
3419    }
3420
3421    #[test]
3422    fn cargo_runner_process_death_is_distinct_from_an_internal_failure() {
3423        let root = std::env::temp_dir().join(format!(
3424            "supercov-cargo-runner-death-{}-{}",
3425            std::process::id(),
3426            SystemTime::now()
3427                .duration_since(UNIX_EPOCH)
3428                .unwrap()
3429                .as_nanos()
3430        ));
3431        fs::create_dir(&root).unwrap();
3432        reserve_cargo_runner_ordinal(&root).unwrap();
3433        let error = read_cargo_runner_units(
3434            &root,
3435            "run_0123456789abcdef",
3436            &["aarch64-apple-darwin".into()],
3437        )
3438        .unwrap_err()
3439        .to_string();
3440        assert!(error.contains("without publishing its unit"), "{error}");
3441        fs::remove_dir_all(root).unwrap();
3442    }
3443
3444    #[test]
3445    fn cargo_runner_units_are_bound_to_the_selected_target_set() {
3446        let root = std::env::temp_dir().join(format!(
3447            "supercov-cargo-runner-target-{}-{}",
3448            std::process::id(),
3449            SystemTime::now()
3450                .duration_since(UNIX_EPOCH)
3451                .unwrap()
3452                .as_nanos()
3453        ));
3454        fs::create_dir(&root).unwrap();
3455        let ordinal = reserve_cargo_runner_ordinal(&root).unwrap();
3456        write_cargo_runner_unit(
3457            &root,
3458            &RustCargoRunnerUnit {
3459                version: RUST_CARGO_RUNNER_VERSION,
3460                run_id: "run_0123456789abcdef".into(),
3461                invocation_ordinal: ordinal,
3462                runner: RustCargoRunnerKind::CargoTest,
3463                runner_run_id: None,
3464                runner_version: None,
3465                runner_binary_id: None,
3466                target: "aarch64-apple-darwin".into(),
3467                artifact: PathBuf::from("target/test-artifact"),
3468                arguments: Vec::new(),
3469                invocation: test_invocation(0),
3470                attempts: Vec::new(),
3471                thread_scope_limitations: BTreeSet::new(),
3472            },
3473        )
3474        .unwrap();
3475        let second_ordinal = reserve_cargo_runner_ordinal(&root).unwrap();
3476        write_cargo_runner_unit(
3477            &root,
3478            &RustCargoRunnerUnit {
3479                version: RUST_CARGO_RUNNER_VERSION,
3480                run_id: "run_0123456789abcdef".into(),
3481                invocation_ordinal: second_ordinal,
3482                runner: RustCargoRunnerKind::CargoTest,
3483                runner_run_id: None,
3484                runner_version: None,
3485                runner_binary_id: None,
3486                target: "x86_64-unknown-linux-gnu".into(),
3487                artifact: PathBuf::from("target/test-artifact"),
3488                arguments: Vec::new(),
3489                invocation: test_invocation(0),
3490                attempts: Vec::new(),
3491                thread_scope_limitations: BTreeSet::new(),
3492            },
3493        )
3494        .unwrap();
3495        assert_eq!(
3496            read_cargo_runner_units(
3497                &root,
3498                "run_0123456789abcdef",
3499                &[
3500                    "aarch64-apple-darwin".into(),
3501                    "x86_64-unknown-linux-gnu".into(),
3502                ],
3503            )
3504            .unwrap()
3505            .len(),
3506            2
3507        );
3508        let error = read_cargo_runner_units(
3509            &root,
3510            "run_0123456789abcdef",
3511            &["x86_64-unknown-linux-gnu".into()],
3512        )
3513        .unwrap_err()
3514        .to_string();
3515        assert!(error.contains("unselected target identity"), "{error}");
3516        fs::remove_dir_all(root).unwrap();
3517    }
3518
3519    #[test]
3520    fn nextest_retry_units_preserve_each_exact_attempt_and_reject_gaps() {
3521        let root = std::env::temp_dir().join(format!(
3522            "supercov-nextest-retries-{}-{}",
3523            std::process::id(),
3524            SystemTime::now()
3525                .duration_since(UNIX_EPOCH)
3526                .unwrap()
3527                .as_nanos()
3528        ));
3529        fs::create_dir(&root).unwrap();
3530        let unit = |ordinal: u64, retry: usize, status: i32| RustCargoRunnerUnit {
3531            version: RUST_CARGO_RUNNER_VERSION,
3532            run_id: "run_0123456789abcdef".into(),
3533            invocation_ordinal: ordinal,
3534            runner: RustCargoRunnerKind::Nextest,
3535            runner_run_id: Some("2ae19189-240a-433a-a31d-acc411fe8e1f".into()),
3536            runner_version: Some("0.9.140".into()),
3537            runner_binary_id: Some("fixture".into()),
3538            target: "aarch64-apple-darwin".into(),
3539            artifact: PathBuf::from("target/test-artifact"),
3540            arguments: vec!["--exact".into(), "tests::flaky".into()],
3541            invocation: RustCargoRunnerInvocation {
3542                started_at_ms: retry as i64,
3543                ended_at_ms: retry as i64 + 1,
3544                ..test_invocation(status)
3545            },
3546            attempts: vec![RustCargoRunnerAttempt {
3547                test: "tests::flaky".into(),
3548                context_id: 7,
3549                retry,
3550                total_attempts: 2,
3551                runner_attempt_id: format!(
3552                    "2ae19189-240a-433a-a31d-acc411fe8e1f:fixture$tests::flaky{}",
3553                    if retry == 0 {
3554                        String::new()
3555                    } else {
3556                        format!("#{}", retry + 1)
3557                    }
3558                ),
3559                outcome: RustCargoRunnerAttemptOutcome::Libtest {
3560                    result: if status == 0 {
3561                        RustLibtestTerminalResult::Passed
3562                    } else {
3563                        RustLibtestTerminalResult::Failed
3564                    },
3565                    timed_out: false,
3566                },
3567                transport: test_transport(),
3568            }],
3569            thread_scope_limitations: BTreeSet::new(),
3570        };
3571
3572        let first = reserve_cargo_runner_ordinal(&root).unwrap();
3573        write_cargo_runner_unit(&root, &unit(first, 0, 101)).unwrap();
3574        let second = reserve_cargo_runner_ordinal(&root).unwrap();
3575        write_cargo_runner_unit(&root, &unit(second, 1, 0)).unwrap();
3576        let read = read_cargo_runner_units(
3577            &root,
3578            "run_0123456789abcdef",
3579            &["aarch64-apple-darwin".into()],
3580        )
3581        .unwrap();
3582        assert_eq!(read.len(), 2);
3583        assert_eq!(read[0].attempts[0].retry, 0);
3584        assert_eq!(read[1].attempts[0].retry, 1);
3585        fs::remove_dir_all(&root).unwrap();
3586
3587        let gap_root = root.with_extension("gap");
3588        fs::create_dir(&gap_root).unwrap();
3589        let first = reserve_cargo_runner_ordinal(&gap_root).unwrap();
3590        write_cargo_runner_unit(&gap_root, &unit(first, 1, 0)).unwrap();
3591        let error = read_cargo_runner_units(
3592            &gap_root,
3593            "run_0123456789abcdef",
3594            &["aarch64-apple-darwin".into()],
3595        )
3596        .unwrap_err()
3597        .to_string();
3598        assert!(error.contains("retry sequence"), "{error}");
3599        fs::remove_dir_all(gap_root).unwrap();
3600    }
3601
3602    #[cfg(unix)]
3603    #[test]
3604    fn nextest_list_passthrough_preserves_output_and_publishes_no_unit() {
3605        let artifact = TestArtifact {
3606            executable: PathBuf::from("/bin/echo"),
3607            runner_argument: None,
3608            package: "fixture".into(),
3609            target_key: "lib:fixture".into(),
3610            kind: "unit".into(),
3611            source: "src/lib.rs".into(),
3612            test_harness: true,
3613        };
3614        let mut stdout = Vec::new();
3615        let mut stderr = Vec::new();
3616        let execution = run_nextest_list_passthrough(
3617            Path::new("/tmp"),
3618            &artifact,
3619            None,
3620            vec![OsString::from("--list"), OsString::from("--format=terse")],
3621            None,
3622            &mut stdout,
3623            &mut stderr,
3624        )
3625        .unwrap();
3626        assert_eq!(execution.exit_code, 0);
3627        assert!(execution.unit_path.is_none());
3628        assert_eq!(stdout, b"--list --format=terse\n");
3629        assert!(stderr.is_empty());
3630    }
3631
3632    #[test]
3633    fn compiler_runner_declares_exact_assertion_but_not_action_causality() {
3634        let runner = runner_declaration();
3635        assert_eq!(runner.attribution.assertion, AttributionPrecision::Exact);
3636        assert_eq!(runner.attribution.action, AttributionPrecision::Unavailable);
3637        assert_eq!(runner.limitations.len(), 1);
3638        let compiler = compiler_runner_declaration();
3639        assert_eq!(compiler.attribution.phase, AttributionPrecision::Exact);
3640        assert_eq!(
3641            compiler.attribution.assertion,
3642            AttributionPrecision::Unavailable
3643        );
3644        let rustdoc = rustdoc_runner_declaration();
3645        assert_eq!(
3646            rustdoc.execution_model,
3647            ExecutionModel::ParallelContextPropagated
3648        );
3649        assert_eq!(rustdoc.attribution.test, AttributionPrecision::Exact);
3650        assert_eq!(rustdoc.attribution.assertion, AttributionPrecision::Exact);
3651    }
3652
3653    #[test]
3654    fn doctest_outcomes_project_exact_status_identity_and_fail_fast_state() {
3655        let entry = |module: &str, line: u64| RustdocMergedEntry {
3656            module: module.into(),
3657            display_name: format!("src/lib.rs - (line {line})"),
3658            path: "src/lib.rs".into(),
3659            line,
3660            ignored: false,
3661            no_run: false,
3662            should_panic: false,
3663        };
3664        let catalog = |line: u64| RustdocExtractedDoctest {
3665            file: "src/lib.rs".into(),
3666            line,
3667            doctest_attributes: RustdocDoctestAttributes {
3668                original: String::new(),
3669                should_panic: false,
3670                no_run: false,
3671                ignore: RustdocDoctestIgnore::None,
3672                rust: true,
3673                test_harness: false,
3674                compile_fail: false,
3675                standalone_crate: false,
3676                error_codes: Vec::new(),
3677                edition: None,
3678                added_css_classes: Vec::new(),
3679                unknown: Vec::new(),
3680            },
3681            original_code: "assert!(true);".into(),
3682            doctest_code: Some(RustdocDoctestCode {
3683                crate_level: String::new(),
3684                code: "assert!(true);".into(),
3685                wrapper: Some(RustdocDoctestWrapper {
3686                    before: "fn main() {".into(),
3687                    after: "}".into(),
3688                    returns_result: false,
3689                }),
3690            }),
3691            name: format!("src/lib.rs - (line {line})"),
3692        };
3693        let completed = |catalog_index, entry: RustdocMergedEntry, status| RustdocJoinedOutcome {
3694            catalog_index,
3695            catalog: catalog(entry.line),
3696            merged_entry: Some(entry.clone()),
3697            state: RustdocJoinedOutcomeState::Completed {
3698                outcome: RustdocTestOutcome {
3699                    display_name: entry.display_name,
3700                    status,
3701                    execution_seconds: Some(0.1),
3702                    stdout: None,
3703                    message: None,
3704                    reason: None,
3705                    timeout_warning: false,
3706                },
3707            },
3708        };
3709        let resolution = RustdocOutcomeResolution {
3710            groups: vec![RustdocOutcomeGroupJoin {
3711                invocation_id: "1".repeat(64),
3712                group: "fixture".into(),
3713                companion_build_id: "2".repeat(64),
3714                raw_catalog_sha256: "4".repeat(64),
3715                raw_events_sha256: "3".repeat(64),
3716                transport_sha256: "5".repeat(64),
3717                join: None,
3718                transport: RustTransportRead::empty(),
3719                entries: vec![
3720                    completed(0, entry("__doctest_0", 3), RustdocOutcomeStatus::Passed),
3721                    RustdocJoinedOutcome {
3722                        catalog_index: 1,
3723                        catalog: catalog(10),
3724                        merged_entry: Some(entry("__doctest_1", 10)),
3725                        state: RustdocJoinedOutcomeState::Unstarted,
3726                    },
3727                ],
3728                ambiguous_filtered_out: 0,
3729                ambiguous_unstarted_tests: 0,
3730            }],
3731            unmatched_maps: Vec::new(),
3732        };
3733        let normalized = NormalizedRustCompilerManifest {
3734            manifest: crate::coverage_report::CoverageManifest {
3735                unmeasured: Vec::new(),
3736                decisions: Vec::new(),
3737                points: Vec::new(),
3738                branches: Vec::new(),
3739                limitations: Vec::new(),
3740                scope: None,
3741            },
3742            hit_obligations_by_ordinal: std::collections::BTreeMap::new(),
3743            internal_ordinals: BTreeSet::new(),
3744            decision_outcome_obligations: std::collections::BTreeMap::new(),
3745            decision_loop_obligations: std::collections::BTreeMap::new(),
3746            decision_logical_selection_obligations: std::collections::BTreeMap::new(),
3747        };
3748        let (results, health) =
3749            doctest_raw_results("run", &resolution, 10, 20, &normalized).unwrap();
3750        assert_eq!(results.len(), 2);
3751        assert_eq!(health.len(), 1);
3752        assert_eq!(results[0].status.as_deref(), Some("passed"));
3753        assert_eq!(results[1].status.as_deref(), Some("unknown"));
3754        assert_eq!(results[0].retry, Some(0));
3755        assert_eq!(results[0].test_file.as_deref(), Some("src/lib.rs"));
3756        assert_eq!(
3757            results[0].test_id.as_deref(),
3758            Some("rust:doctest:fixture:src/lib.rs:3")
3759        );
3760        assert_eq!(results[0].scope.as_ref().unwrap().test_id, results[0].test);
3761        assert!(doctest_command_failed(&resolution));
3762        assert!(results[1].phases.is_empty());
3763        assert!(resolution.is_fully_catalogued());
3764    }
3765}