Skip to main content

supercov_engine/
rust_compiler_test_runner.rs

1//! Private execution and attribution for compiler-instrumented Rust artifacts.
2//!
3//! The compiler frontend freezes the complete denominator before this module
4//! launches anything. Ordinary Cargo artifacts run once under the selected
5//! toolchain's exact libtest companion and share one authenticated mmap whose
6//! dynamic contexts are partitioned by test. Nextest attempts and opaque
7//! custom harnesses retain their intrinsic process boundary. Context-zero
8//! records are always published as invocation background, never as test work.
9
10use std::{
11    collections::{BTreeMap, BTreeSet},
12    ffi::OsString,
13    fs::{self, OpenOptions},
14    io::{self, Write},
15    path::{Component, Path, PathBuf},
16    sync::{
17        Mutex,
18        atomic::{AtomicUsize, Ordering},
19    },
20    time::{SystemTime, UNIX_EPOCH},
21};
22
23use nextest_metadata::{BuildPlatform, FilterMatch, NextestExitCode, RustTestSuiteStatusSummary};
24use serde::{Deserialize, Serialize};
25use sha2::{Digest, Sha256};
26use supercov_contracts::{
27    AttributionPrecision, ExecutionModel, FrontendAttribution, FrontendLimitation,
28    FrontendLimitationScope, FrontendRunDeclaration, FrontendRunnerDeclaration,
29    LANGUAGE_FRONTEND_PROTOCOL_VERSION, StructuralSource,
30};
31
32use crate::workspace::canonicalize_simplified;
33use crate::{
34    coverage_report::{
35        CoverageModelDeclaration, CoveragePhase, CoverageReportRequest, ExecutionScope,
36        ExitCodeInput, PersistedCoverageModel, RawTestResult, RuntimeSnapshot, TestProvenance,
37    },
38    evidence_archive::EvidenceArchiveEntry,
39    process_supervision::{
40        CommandSpec, ProcessSupervisor, SupervisedOutput, SupervisedResult, SupervisionOptions,
41    },
42    rust_cargo_configuration::{
43        RustCargoResolvedRunner, RustCargoResolvedTargetRunner, RustCargoRunnerPlan,
44    },
45    rust_compiler_ctfe::RustCompilerCtfeUnit,
46    rust_compiler_evidence::{
47        RustCompilerEvidenceProjection, RustCompilerTransportHealth, project_rust_compiler_evidence,
48    },
49    rust_compiler_manifest::NormalizedRustCompilerManifest,
50    rust_compiler_orchestration::{
51        RustCompilerBuild, RustCompilerBuildRequest, RustCompilerTestArtifact,
52    },
53    rust_doctest::{RustdocJoinedOutcomeState, RustdocOutcomeResolution, RustdocOutcomeStatus},
54    rust_libtest_events::{
55        RUST_LIBTEST_EVENTS_ENV, RUST_LIBTEST_TOKEN_ENV, RustLibtestEvent, RustLibtestRunEvents,
56        RustLibtestTerminalResult, create_rust_libtest_event_file, read_rust_libtest_events,
57        validate_rust_libtest_run_events,
58    },
59    rust_probe_transport::{
60        DEFAULT_DESCRIPTOR_CAPACITY, DEFAULT_PAYLOAD_CAPACITY, RUST_CONTEXT_ENV,
61        RUST_TRANSPORT_ENV, RUST_TRANSPORT_TOKEN_ENV, RustTransportPartition, RustTransportRead,
62        create_rust_transport, partition_rust_transport_by_test_contexts, read_rust_transport,
63    },
64    rust_runner_attempt::{
65        NextestAttemptIdentity, RustRunnerInvocationIdentity, classify_rust_runner_environment,
66    },
67    rust_test_context::preflight_rust_test_contexts,
68    rust_test_runner::rust_libtest_selection,
69};
70
71const TOKEN_BYTES: usize = supercov_contracts::RUST_PROBE_TRANSPORT_TOKEN_SIZE;
72pub const RUST_CARGO_RUNNER_CONFIG_ENV: &str = "SUPERCOV_RUST_CARGO_RUNNER_CONFIG";
73pub const RUST_CARGO_RUNNER_VERSION: u32 = 6;
74
75#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
76#[serde(rename_all = "kebab-case")]
77pub enum RustCargoRunnerKind {
78    CargoTest,
79    CargoCustomHarness,
80    Nextest,
81}
82
83#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
84#[serde(rename_all = "camelCase", deny_unknown_fields)]
85pub struct RustCargoRunnerArtifact {
86    pub executable: PathBuf,
87    pub test_harness: bool,
88}
89
90#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
91#[serde(rename_all = "camelCase", deny_unknown_fields)]
92pub struct RustCargoRunnerConfig {
93    pub version: u32,
94    pub run_id: String,
95    pub target_directory: PathBuf,
96    pub output_directory: PathBuf,
97    pub target_runners: Vec<RustCargoResolvedTargetRunner>,
98    pub artifacts: Vec<RustCargoRunnerArtifact>,
99}
100
101#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
102#[serde(rename_all = "camelCase", deny_unknown_fields)]
103pub struct RustCargoRunnerAttempt {
104    pub test: String,
105    pub context_id: u64,
106    pub retry: usize,
107    pub total_attempts: usize,
108    pub runner_attempt_id: String,
109    pub outcome: RustCargoRunnerAttemptOutcome,
110    pub transport: RustTransportRead,
111}
112
113#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
114#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)]
115pub enum RustCargoRunnerAttemptOutcome {
116    Libtest {
117        result: RustLibtestTerminalResult,
118        timed_out: bool,
119    },
120    Unstarted,
121    OpaqueProcess,
122}
123
124fn attempt_outcome_succeeded(outcome: &RustCargoRunnerAttemptOutcome) -> bool {
125    matches!(
126        outcome,
127        RustCargoRunnerAttemptOutcome::Libtest {
128            result: RustLibtestTerminalResult::Passed
129                | RustLibtestTerminalResult::Ignored
130                | RustLibtestTerminalResult::Benchmarked,
131            ..
132        }
133    )
134}
135
136#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
137#[serde(rename_all = "camelCase", deny_unknown_fields)]
138pub struct RustCargoRunnerInvocation {
139    pub result: SupervisedResult,
140    pub started_at_ms: i64,
141    pub ended_at_ms: i64,
142    pub stdout: Vec<u8>,
143    pub stderr: Vec<u8>,
144    pub background_transport: RustTransportRead,
145}
146
147#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
148#[serde(rename_all = "camelCase", deny_unknown_fields)]
149pub struct RustCargoRunnerUnit {
150    pub version: u32,
151    pub run_id: String,
152    pub invocation_ordinal: u64,
153    pub runner: RustCargoRunnerKind,
154    #[serde(skip_serializing_if = "Option::is_none")]
155    pub runner_run_id: Option<String>,
156    #[serde(skip_serializing_if = "Option::is_none")]
157    pub runner_version: Option<String>,
158    #[serde(skip_serializing_if = "Option::is_none")]
159    pub runner_binary_id: Option<String>,
160    pub target: String,
161    pub artifact: PathBuf,
162    pub arguments: Vec<String>,
163    pub invocation: RustCargoRunnerInvocation,
164    pub attempts: Vec<RustCargoRunnerAttempt>,
165    /// Deterministic join-bounded quarantine notes: one per thread phase whose
166    /// lifetime escaped its creating test in this invocation's transport.
167    pub thread_scope_limitations: BTreeSet<String>,
168}
169
170fn validate_persisted_runner_transport(
171    unit: &RustCargoRunnerUnit,
172) -> Result<(), RustCompilerTestError> {
173    let mut roots = BTreeSet::new();
174    let mut combined = unit.invocation.background_transport.clone();
175    for attempt in &unit.attempts {
176        if matches!(attempt.context_id, 0 | u64::MAX) || !roots.insert(attempt.context_id) {
177            return Err(RustCompilerTestError::Context(format!(
178                "Cargo runner unit {} has a reserved or duplicate test context",
179                unit.invocation_ordinal
180            )));
181        }
182        combined
183            .observations
184            .extend(attempt.transport.observations.iter().cloned());
185        combined
186            .ordinal_hits
187            .extend(attempt.transport.ordinal_hits.iter().copied());
188        combined
189            .phases
190            .extend(attempt.transport.phases.iter().cloned());
191        combined
192            .thread_phases
193            .extend(attempt.transport.thread_phases.iter().copied());
194        combined
195            .thread_ends
196            .extend(attempt.transport.thread_ends.iter().copied());
197        combined
198            .test_boundaries
199            .extend(attempt.transport.test_boundaries.iter().copied());
200        combined.committed = combined
201            .committed
202            .checked_add(attempt.transport.committed)
203            .ok_or_else(|| {
204                RustCompilerTestError::Context(
205                    "Cargo runner committed transport count overflowed u64".into(),
206                )
207            })?;
208        combined.incomplete = combined
209            .incomplete
210            .checked_add(attempt.transport.incomplete)
211            .ok_or_else(|| {
212                RustCompilerTestError::Context(
213                    "Cargo runner incomplete transport count overflowed u64".into(),
214                )
215            })?;
216        combined.dropped = combined
217            .dropped
218            .checked_add(attempt.transport.dropped)
219            .ok_or_else(|| {
220                RustCompilerTestError::Context(
221                    "Cargo runner dropped transport count overflowed u64".into(),
222                )
223            })?;
224        combined.attachments = combined
225            .attachments
226            .checked_add(attempt.transport.attachments)
227            .ok_or_else(|| {
228                RustCompilerTestError::Context(
229                    "Cargo runner transport attachment count overflowed u64".into(),
230                )
231            })?;
232    }
233    let repartitioned =
234        partition_rust_transport_by_test_contexts(&combined, &roots).map_err(|error| {
235            RustCompilerTestError::Context(format!(
236                "Cargo runner unit {} has invalid persisted attribution: {error}",
237                unit.invocation_ordinal
238            ))
239        })?;
240    if repartitioned.background != unit.invocation.background_transport
241        || repartitioned.thread_scope_limitations != unit.thread_scope_limitations
242        || unit.attempts.iter().any(|attempt| {
243            repartitioned.attributed.get(&attempt.context_id) != Some(&attempt.transport)
244        })
245    {
246        return Err(RustCompilerTestError::Context(format!(
247            "Cargo runner unit {} does not preserve its exact transport partition",
248            unit.invocation_ordinal
249        )));
250    }
251    Ok(())
252}
253
254#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
255#[serde(rename_all = "camelCase", deny_unknown_fields)]
256struct RustCargoRunnerFailure {
257    version: u32,
258    run_id: String,
259    invocation_ordinal: u64,
260    target: Option<String>,
261    artifact: Option<PathBuf>,
262    error: String,
263}
264
265#[derive(Debug, Clone, PartialEq, Eq)]
266pub struct RustCargoRunnerExecution {
267    pub exit_code: i32,
268    pub unit_path: Option<PathBuf>,
269}
270
271#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
272#[serde(rename_all = "camelCase", deny_unknown_fields)]
273pub struct RustCompilerRunRequest {
274    pub project_root: PathBuf,
275    pub command: Vec<String>,
276    pub run_id: String,
277    pub generated_at: String,
278    pub wrapper_path: PathBuf,
279    pub companion_candidates: Vec<PathBuf>,
280    pub require_public_capabilities: bool,
281    pub cargo_runner_plan: RustCargoRunnerPlan,
282    pub watchdog_program: Option<PathBuf>,
283}
284
285impl RustCompilerRunRequest {
286    fn build_request(&self) -> RustCompilerBuildRequest {
287        RustCompilerBuildRequest {
288            project_root: self.project_root.clone(),
289            command: self.command.clone(),
290            run_id: self.run_id.clone(),
291            wrapper_path: self.wrapper_path.clone(),
292            companion_candidates: self.companion_candidates.clone(),
293            require_public_capabilities: self.require_public_capabilities,
294            cargo_runner_plan: self.cargo_runner_plan.clone(),
295        }
296    }
297}
298
299#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
300#[serde(rename_all = "camelCase")]
301pub struct RustCompilerTransportHealthRecord {
302    pub scope_id: String,
303    pub scope_kind: String,
304    pub status: String,
305    pub transport: RustCompilerTransportHealth,
306    /// Join-bounded thread quarantine notes for this scope's transport. Work
307    /// under an escaped thread phase is deterministic background evidence.
308    #[serde(skip_serializing_if = "BTreeSet::is_empty")]
309    pub thread_scope_limitations: BTreeSet<String>,
310}
311
312#[derive(Debug, Clone, PartialEq)]
313pub struct RustCompilerFrontendRun {
314    pub selection: crate::rust_compiler_selection::SelectedRustCompilerCompanion,
315    pub declaration: FrontendRunDeclaration,
316    pub request: CoverageReportRequest,
317    pub exit_code: i32,
318    pub artifacts: usize,
319    pub artifact_files: Vec<PathBuf>,
320    pub transport_health: Vec<RustCompilerTransportHealthRecord>,
321    pub build_ms: f64,
322    pub execution_ms: f64,
323}
324
325impl RustCompilerFrontendRun {
326    pub fn archive_entries(&self) -> Result<Vec<EvidenceArchiveEntry>, serde_json::Error> {
327        let model = PersistedCoverageModel::from_declaration(
328            self.request
329                .coverage_model
330                .as_ref()
331                .expect("Rust compiler frontend always declares a coverage model"),
332        )
333        .expect("Rust compiler coverage model is contract-valid");
334        let mut entries = vec![
335            EvidenceArchiveEntry {
336                path: "coverage-model.json".into(),
337                contents: serde_json::to_vec(&model)?,
338            },
339            EvidenceArchiveEntry {
340                path: "frontend.json".into(),
341                contents: serde_json::to_vec(&self.declaration)?,
342            },
343            EvidenceArchiveEntry {
344                path: "manifest.json".into(),
345                contents: serde_json::to_vec(&self.request.manifest)?,
346            },
347        ];
348        for (index, result) in self.request.raw_results.iter().enumerate() {
349            entries.push(EvidenceArchiveEntry {
350                path: format!("results/{index:08}/mcdc.json"),
351                contents: serde_json::to_vec(result)?,
352            });
353        }
354        entries.push(EvidenceArchiveEntry {
355            path: "rust/transport-health.json".into(),
356            contents: serde_json::to_vec(&self.transport_health)?,
357        });
358        Ok(entries)
359    }
360}
361
362#[derive(Debug)]
363pub enum RustCompilerTestError {
364    Build(String),
365    Io { path: PathBuf, reason: String },
366    UnsafeArtifact(String),
367    List { artifact: PathBuf, reason: String },
368    Context(String),
369    DuplicateTest(String),
370    Random(String),
371    Launch { test: String, reason: String },
372    Transport { test: String, reason: String },
373    DroppedEvidence { test: String, dropped: u64 },
374    Projection { test: String, reason: String },
375    UnsupportedCommand(String),
376    UnverifiedExecution { code: i32, reason: String },
377    Interrupted { code: i32, signal: String },
378}
379
380impl std::fmt::Display for RustCompilerTestError {
381    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
382        match self {
383            Self::Build(reason) => write!(formatter, "Rust compiler build failed: {reason}"),
384            Self::Io { path, reason } => write!(formatter, "{}: {reason}", path.display()),
385            Self::UnsafeArtifact(reason) => {
386                write!(formatter, "unsafe Rust test artifact: {reason}")
387            }
388            Self::List { artifact, reason } => write!(
389                formatter,
390                "could not enumerate tests in {}: {reason}",
391                artifact.display()
392            ),
393            Self::Context(reason) => write!(formatter, "invalid Rust test context: {reason}"),
394            Self::DuplicateTest(test) => write!(formatter, "duplicate Rust test identity: {test}"),
395            Self::Random(reason) => {
396                write!(formatter, "could not authenticate Rust evidence: {reason}")
397            }
398            Self::Launch { test, reason } => {
399                write!(formatter, "could not launch Rust test {test}: {reason}")
400            }
401            Self::Transport { test, reason } => {
402                write!(formatter, "invalid Rust transport for {test}: {reason}")
403            }
404            Self::DroppedEvidence { test, dropped } => write!(
405                formatter,
406                "Rust transport dropped {dropped} record(s) for {test}; refusing partial coverage"
407            ),
408            Self::Projection { test, reason } => {
409                write!(formatter, "invalid Rust evidence for {test}: {reason}")
410            }
411            Self::UnsupportedCommand(reason) => formatter.write_str(reason),
412            Self::UnverifiedExecution { code, reason } => write!(
413                formatter,
414                "Rust test command exited {code}, but Supercov could not authenticate complete coverage evidence: {reason}"
415            ),
416            Self::Interrupted { signal, .. } => {
417                write!(formatter, "Rust test run was interrupted by {signal}")
418            }
419        }
420    }
421}
422
423impl std::error::Error for RustCompilerTestError {}
424
425fn io_error(path: &Path, error: impl std::fmt::Display) -> RustCompilerTestError {
426    RustCompilerTestError::Io {
427        path: path.to_path_buf(),
428        reason: error.to_string(),
429    }
430}
431
432#[derive(Debug, Clone)]
433struct TestArtifact {
434    executable: PathBuf,
435    runner_argument: Option<OsString>,
436    package: String,
437    target_key: String,
438    kind: String,
439    source: String,
440    test_harness: bool,
441}
442
443#[derive(Debug, Clone)]
444struct ProcessTask {
445    ordinal: usize,
446    artifact_index: usize,
447    artifact: TestArtifact,
448    test: String,
449    test_id: String,
450    context_id: u64,
451    retry: usize,
452    total_attempts: usize,
453    runner_attempt_id: String,
454    runner: RustCargoRunnerKind,
455    transport: PathBuf,
456    libtest_events: PathBuf,
457    test_arguments: Vec<OsString>,
458    underlying_runner: Option<RustCargoResolvedRunner>,
459}
460
461#[derive(Debug)]
462struct ProcessOutcome {
463    task: ProcessTask,
464    output: SupervisedOutput,
465    read: RustTransportRead,
466    attempt_outcome: RustCargoRunnerAttemptOutcome,
467    started_at_ms: i64,
468    ended_at_ms: i64,
469}
470
471struct StockLibtestExecution {
472    output: SupervisedOutput,
473    events: RustLibtestRunEvents,
474    partition: RustTransportPartition,
475    started_at_ms: i64,
476    ended_at_ms: i64,
477}
478
479struct RemoveFileOnDrop(Option<PathBuf>);
480
481impl Drop for RemoveFileOnDrop {
482    fn drop(&mut self) {
483        if let Some(path) = self.0.take() {
484            let _ = fs::remove_file(path);
485        }
486    }
487}
488
489fn stock_libtest_transport_reason(
490    reason: impl std::fmt::Display,
491    output: &SupervisedOutput,
492) -> String {
493    const LIMIT: usize = 16 * 1024;
494    fn tail(bytes: &[u8]) -> String {
495        let start = bytes.len().saturating_sub(LIMIT);
496        String::from_utf8_lossy(&bytes[start..]).into_owned()
497    }
498
499    let mut message = format!(
500        "{reason}; stock libtest process exit={} ",
501        output.result.exit_code()
502    );
503    if !output.stdout.is_empty() {
504        message.push_str("\nstdout tail:\n");
505        message.push_str(&tail(&output.stdout));
506    }
507    if !output.stderr.is_empty() {
508        message.push_str("\nstderr tail:\n");
509        message.push_str(&tail(&output.stderr));
510    }
511    message
512}
513
514fn epoch_ms() -> Result<i64, RustCompilerTestError> {
515    let millis = SystemTime::now()
516        .duration_since(UNIX_EPOCH)
517        .map_err(|error| RustCompilerTestError::Random(error.to_string()))?
518        .as_millis();
519    i64::try_from(millis).map_err(|error| RustCompilerTestError::Random(error.to_string()))
520}
521
522fn relative_source(root: &Path, source: &Path) -> Result<String, RustCompilerTestError> {
523    let source = canonicalize_simplified(source).map_err(|error| io_error(source, error))?;
524    let relative = source
525        .strip_prefix(root)
526        .map_err(|_| RustCompilerTestError::UnsafeArtifact(source.display().to_string()))?;
527    if relative.as_os_str().is_empty()
528        || relative
529            .components()
530            .any(|component| !matches!(component, Component::Normal(_)))
531    {
532        return Err(RustCompilerTestError::UnsafeArtifact(
533            source.display().to_string(),
534        ));
535    }
536    Ok(relative.to_string_lossy().replace('\\', "/"))
537}
538
539fn normalize_artifacts(
540    project_root: &Path,
541    target_directory: &Path,
542    artifacts: &[RustCompilerTestArtifact],
543) -> Result<Vec<TestArtifact>, RustCompilerTestError> {
544    let target_directory = canonicalize_simplified(target_directory)
545        .map_err(|error| io_error(target_directory, error))?;
546    artifacts
547        .iter()
548        .map(|artifact| {
549            let executable = canonicalize_simplified(&artifact.executable)
550                .map_err(|error| io_error(&artifact.executable, error))?;
551            if !executable.starts_with(&target_directory) {
552                return Err(RustCompilerTestError::UnsafeArtifact(
553                    executable.display().to_string(),
554                ));
555            }
556            let mut target_kinds = artifact.target_kinds.clone();
557            target_kinds.sort();
558            target_kinds.dedup();
559            Ok(TestArtifact {
560                executable,
561                runner_argument: None,
562                package: artifact.package.clone(),
563                target_key: format!("{}:{}", target_kinds.join("+"), artifact.target_name),
564                kind: if artifact.target_kinds.iter().any(|kind| kind == "test") {
565                    "integration".into()
566                } else {
567                    "unit".into()
568                },
569                source: relative_source(project_root, &artifact.source_path)?,
570                test_harness: artifact.test_harness,
571            })
572        })
573        .collect()
574}
575
576fn libtest_id(compilation_target: &str, artifact: &TestArtifact, test: &str) -> String {
577    format!(
578        "rust:libtest:{compilation_target}:{}:{}:{}::{test}",
579        artifact.package, artifact.target_key, artifact.source,
580    )
581}
582
583fn custom_harness_id(compilation_target: &str, artifact: &TestArtifact) -> String {
584    format!(
585        "rust:custom-harness:{compilation_target}:{}:{}:{}",
586        artifact.package, artifact.target_key, artifact.source,
587    )
588}
589
590fn list_tests(
591    project_root: &Path,
592    artifact: &TestArtifact,
593    selection_arguments: &[String],
594    underlying_runner: Option<&RustCargoResolvedRunner>,
595    supervisor: &ProcessSupervisor,
596    options: SupervisionOptions,
597    event_path: &Path,
598) -> Result<Vec<String>, RustCompilerTestError> {
599    let mut test_arguments = selection_arguments
600        .iter()
601        .map(OsString::from)
602        .collect::<Vec<_>>();
603    test_arguments.extend(["--list".into(), "--format".into(), "terse".into()]);
604    let (program, arguments) = artifact_command(artifact, underlying_runner, test_arguments);
605    let mut event_token = [0_u8; supercov_contracts::RUST_LIBTEST_EVENT_TOKEN_SIZE];
606    getrandom::fill(&mut event_token).map_err(|error| {
607        RustCompilerTestError::Random(format!("libtest listing token: {error}"))
608    })?;
609    create_rust_libtest_event_file(event_path, event_token).map_err(|error| {
610        RustCompilerTestError::List {
611            artifact: artifact.executable.clone(),
612            reason: error.to_string(),
613        }
614    })?;
615    let mut event_cleanup = RemoveFileOnDrop(Some(event_path.to_owned()));
616    let output = supervisor
617        .supervise_captured(
618            &CommandSpec {
619                program,
620                arguments,
621                cwd: project_root.to_owned(),
622                environment: Some(inherited_environment([
623                    (
624                        OsString::from(RUST_LIBTEST_EVENTS_ENV),
625                        event_path.as_os_str().to_owned(),
626                    ),
627                    (
628                        OsString::from(RUST_LIBTEST_TOKEN_ENV),
629                        OsString::from(token_hex(&event_token)),
630                    ),
631                ])),
632                captured_output: None,
633            },
634            options,
635            &mut io::sink(),
636        )
637        .map_err(|error| RustCompilerTestError::List {
638            artifact: artifact.executable.clone(),
639            reason: error.to_string(),
640        })?;
641    if output.result.exit_code() != 0 {
642        return Err(RustCompilerTestError::List {
643            artifact: artifact.executable.clone(),
644            reason: format!(
645                "{}{}",
646                String::from_utf8_lossy(&output.stderr),
647                String::from_utf8_lossy(&output.stdout)
648            )
649            .trim()
650            .to_owned(),
651        });
652    }
653    let mut tests = String::from_utf8_lossy(&output.stdout)
654        .lines()
655        .filter_map(|line| {
656            line.strip_suffix(": test")
657                .or_else(|| line.strip_suffix(": benchmark"))
658        })
659        .map(str::to_owned)
660        .collect::<Vec<_>>();
661    tests.sort();
662    tests.dedup();
663    let events = read_rust_libtest_events(event_path, &event_token).map_err(|error| {
664        RustCompilerTestError::List {
665            artifact: artifact.executable.clone(),
666            reason: error.to_string(),
667        }
668    })?;
669    if !matches!(
670        events.as_slice(),
671        [RustLibtestEvent::FilteredOut { .. }, RustLibtestEvent::Filtered { count, .. }]
672            if *count == tests.len() as u64
673    ) {
674        return Err(RustCompilerTestError::List {
675            artifact: artifact.executable.clone(),
676            reason: "authenticated libtest listing events disagree with terse output".into(),
677        });
678    }
679    fs::remove_file(event_path).map_err(|error| io_error(event_path, error))?;
680    event_cleanup.0 = None;
681    Ok(tests)
682}
683
684fn artifact_command(
685    artifact: &TestArtifact,
686    underlying_runner: Option<&RustCargoResolvedRunner>,
687    test_arguments: Vec<OsString>,
688) -> (OsString, Vec<OsString>) {
689    match underlying_runner {
690        Some(runner) => {
691            let mut arguments = runner
692                .arguments
693                .iter()
694                .map(OsString::from)
695                .collect::<Vec<_>>();
696            arguments.push(
697                artifact
698                    .runner_argument
699                    .clone()
700                    .unwrap_or_else(|| artifact.executable.clone().into_os_string()),
701            );
702            arguments.extend(test_arguments);
703            (runner.program.clone().into_os_string(), arguments)
704        }
705        None => (artifact.executable.clone().into_os_string(), test_arguments),
706    }
707}
708
709fn token_hex<const N: usize>(token: &[u8; N]) -> String {
710    token.iter().map(|byte| format!("{byte:02x}")).collect()
711}
712
713fn phase_id(run_id: &str, attempt_id: &str) -> String {
714    let mut digest = Sha256::new();
715    digest.update((run_id.len() as u64).to_be_bytes());
716    digest.update(run_id.as_bytes());
717    digest.update((attempt_id.len() as u64).to_be_bytes());
718    digest.update(attempt_id.as_bytes());
719    let hex = format!("{:x}", digest.finalize());
720    format!("rust-test:{}", &hex[..40])
721}
722
723fn snapshot_has_evidence(snapshot: &RuntimeSnapshot) -> bool {
724    !snapshot.hits.is_empty() || !snapshot.decisions.is_empty() || !snapshot.events.is_empty()
725}
726
727fn inherited_environment(
728    overrides: impl IntoIterator<Item = (OsString, OsString)>,
729) -> Vec<(OsString, OsString)> {
730    let mut environment = std::env::vars_os().collect::<BTreeMap<_, _>>();
731    environment.extend(overrides);
732    environment.into_iter().collect()
733}
734
735fn run_process(
736    project_root: &Path,
737    task: &ProcessTask,
738    supervisor: &ProcessSupervisor,
739    options: SupervisionOptions,
740) -> Result<ProcessOutcome, String> {
741    let mut token = [0_u8; TOKEN_BYTES];
742    getrandom::fill(&mut token).map_err(|error| error.to_string())?;
743    create_rust_transport(
744        &task.transport,
745        token,
746        DEFAULT_DESCRIPTOR_CAPACITY,
747        DEFAULT_PAYLOAD_CAPACITY,
748    )
749    .map_err(|error| error.to_string())?;
750    let mut transport_cleanup = RemoveFileOnDrop(Some(task.transport.clone()));
751    let event_token = if task.runner == RustCargoRunnerKind::CargoCustomHarness {
752        None
753    } else {
754        let mut token = [0_u8; supercov_contracts::RUST_LIBTEST_EVENT_TOKEN_SIZE];
755        getrandom::fill(&mut token).map_err(|error| error.to_string())?;
756        create_rust_libtest_event_file(&task.libtest_events, token)
757            .map_err(|error| error.to_string())?;
758        Some(token)
759    };
760    let mut event_cleanup =
761        RemoveFileOnDrop(event_token.is_some().then(|| task.libtest_events.clone()));
762    let started_at_ms = epoch_ms().map_err(|error| error.to_string())?;
763    let (program, arguments) = artifact_command(
764        &task.artifact,
765        task.underlying_runner.as_ref(),
766        task.test_arguments.clone(),
767    );
768    let mut environment = vec![
769        (
770            OsString::from(RUST_TRANSPORT_ENV),
771            task.transport.clone().into_os_string(),
772        ),
773        (
774            OsString::from(RUST_TRANSPORT_TOKEN_ENV),
775            OsString::from(token_hex(&token)),
776        ),
777        (
778            OsString::from(RUST_CONTEXT_ENV),
779            OsString::from(format!("{:016x}", task.context_id)),
780        ),
781    ];
782    if let Some(event_token) = &event_token {
783        environment.extend([
784            (
785                OsString::from(RUST_LIBTEST_EVENTS_ENV),
786                task.libtest_events.clone().into_os_string(),
787            ),
788            (
789                OsString::from(RUST_LIBTEST_TOKEN_ENV),
790                OsString::from(token_hex(event_token)),
791            ),
792        ]);
793    }
794    let output = supervisor
795        .supervise_captured(
796            &CommandSpec {
797                program,
798                arguments,
799                cwd: project_root.to_owned(),
800                environment: Some(inherited_environment(environment)),
801                captured_output: None,
802            },
803            options,
804            &mut io::sink(),
805        )
806        .map_err(|error| error.to_string())?;
807    let ended_at_ms = epoch_ms().map_err(|error| error.to_string())?;
808    let read = read_rust_transport(&task.transport, &token).map_err(|error| error.to_string())?;
809    let attempt_outcome = if let Some(event_token) = &event_token {
810        let events = read_rust_libtest_events(&task.libtest_events, event_token)
811            .map_err(|error| error.to_string())?;
812        let joined = validate_rust_libtest_run_events(&events, [task.test.clone()])
813            .map_err(|error| error.to_string())?;
814        let [attempt] = joined.attempts.as_slice() else {
815            return Err("exact libtest attempt did not produce one terminal event".into());
816        };
817        if !joined.unstarted.is_empty()
818            || (matches!(
819                attempt.result,
820                RustLibtestTerminalResult::Passed
821                    | RustLibtestTerminalResult::Ignored
822                    | RustLibtestTerminalResult::Benchmarked
823            ) != (output.result.exit_code() == 0))
824        {
825            return Err("exact libtest terminal event disagrees with process status".into());
826        }
827        RustCargoRunnerAttemptOutcome::Libtest {
828            result: attempt.result,
829            timed_out: attempt.timed_out,
830        }
831    } else {
832        RustCargoRunnerAttemptOutcome::OpaqueProcess
833    };
834    fs::remove_file(&task.transport).map_err(|error| error.to_string())?;
835    transport_cleanup.0 = None;
836    if event_token.is_some() {
837        fs::remove_file(&task.libtest_events).map_err(|error| error.to_string())?;
838        event_cleanup.0 = None;
839    }
840    Ok(ProcessOutcome {
841        task: task.clone(),
842        output,
843        read,
844        attempt_outcome,
845        started_at_ms,
846        ended_at_ms,
847    })
848}
849
850#[allow(clippy::too_many_arguments)]
851fn run_stock_libtest_artifact(
852    project_root: &Path,
853    artifact: &TestArtifact,
854    underlying_runner: Option<&RustCargoResolvedRunner>,
855    arguments: Vec<OsString>,
856    selected_tests: &[String],
857    contexts: &BTreeMap<String, u64>,
858    transport_path: &Path,
859    event_path: &Path,
860    supervisor: &ProcessSupervisor,
861    options: SupervisionOptions,
862) -> Result<StockLibtestExecution, RustCompilerTestError> {
863    let mut token = [0_u8; TOKEN_BYTES];
864    getrandom::fill(&mut token)
865        .map_err(|error| RustCompilerTestError::Random(error.to_string()))?;
866    create_rust_transport(
867        transport_path,
868        token,
869        DEFAULT_DESCRIPTOR_CAPACITY,
870        DEFAULT_PAYLOAD_CAPACITY,
871    )
872    .map_err(|error| RustCompilerTestError::Transport {
873        test: artifact.target_key.clone(),
874        reason: error.to_string(),
875    })?;
876    let mut transport_cleanup = RemoveFileOnDrop(Some(transport_path.to_owned()));
877    let mut event_token = [0_u8; supercov_contracts::RUST_LIBTEST_EVENT_TOKEN_SIZE];
878    getrandom::fill(&mut event_token)
879        .map_err(|error| RustCompilerTestError::Random(error.to_string()))?;
880    create_rust_libtest_event_file(event_path, event_token).map_err(|error| {
881        RustCompilerTestError::Transport {
882            test: artifact.target_key.clone(),
883            reason: error.to_string(),
884        }
885    })?;
886    let mut event_cleanup = RemoveFileOnDrop(Some(event_path.to_owned()));
887    let (program, arguments) = artifact_command(artifact, underlying_runner, arguments);
888    let started_at_ms = epoch_ms()?;
889    let output = supervisor
890        .supervise_captured(
891            &CommandSpec {
892                program,
893                arguments,
894                cwd: project_root.to_owned(),
895                environment: Some(inherited_environment([
896                    (
897                        OsString::from(RUST_TRANSPORT_ENV),
898                        transport_path.as_os_str().to_owned(),
899                    ),
900                    (
901                        OsString::from(RUST_TRANSPORT_TOKEN_ENV),
902                        OsString::from(token_hex(&token)),
903                    ),
904                    (
905                        OsString::from(RUST_CONTEXT_ENV),
906                        OsString::from("0000000000000000"),
907                    ),
908                    (
909                        OsString::from(RUST_LIBTEST_EVENTS_ENV),
910                        event_path.as_os_str().to_owned(),
911                    ),
912                    (
913                        OsString::from(RUST_LIBTEST_TOKEN_ENV),
914                        OsString::from(token_hex(&event_token)),
915                    ),
916                ])),
917                captured_output: None,
918            },
919            options,
920            &mut io::sink(),
921        )
922        .map_err(|error| RustCompilerTestError::Launch {
923            test: artifact.target_key.clone(),
924            reason: error.to_string(),
925        })?;
926    let ended_at_ms = epoch_ms()?;
927    let read = read_rust_transport(transport_path, &token).map_err(|error| {
928        RustCompilerTestError::Transport {
929            test: artifact.target_key.clone(),
930            reason: error.to_string(),
931        }
932    })?;
933    let events = read_rust_libtest_events(event_path, &event_token).map_err(|error| {
934        RustCompilerTestError::Transport {
935            test: artifact.target_key.clone(),
936            reason: stock_libtest_transport_reason(error, &output),
937        }
938    })?;
939    let events = validate_rust_libtest_run_events(&events, selected_tests.iter().cloned())
940        .map_err(|error| RustCompilerTestError::Transport {
941            test: artifact.target_key.clone(),
942            reason: stock_libtest_transport_reason(error, &output),
943        })?;
944    let terminal_failure = events
945        .attempts
946        .iter()
947        .any(|attempt| attempt.result == RustLibtestTerminalResult::Failed);
948    let expected_success = !terminal_failure && events.unstarted.is_empty();
949    if expected_success != (output.result.exit_code() == 0) {
950        return Err(RustCompilerTestError::UnverifiedExecution {
951            code: output.result.exit_code(),
952            reason: "stock libtest process status disagrees with authenticated terminal and fail-fast events"
953                .into(),
954        });
955    }
956    let roots = contexts.values().copied().collect::<BTreeSet<_>>();
957    let partition = partition_rust_transport_by_test_contexts(&read, &roots).map_err(|error| {
958        RustCompilerTestError::Transport {
959            test: artifact.target_key.clone(),
960            reason: error.to_string(),
961        }
962    })?;
963    fs::remove_file(transport_path).map_err(|error| io_error(transport_path, error))?;
964    transport_cleanup.0 = None;
965    fs::remove_file(event_path).map_err(|error| io_error(event_path, error))?;
966    event_cleanup.0 = None;
967    Ok(StockLibtestExecution {
968        output,
969        events,
970        partition,
971        started_at_ms,
972        ended_at_ms,
973    })
974}
975
976fn execute_process_tasks(
977    project_root: &Path,
978    tasks: &[ProcessTask],
979    requested_workers: usize,
980    supervisor: &ProcessSupervisor,
981    options: SupervisionOptions,
982) -> Result<Vec<ProcessOutcome>, RustCompilerTestError> {
983    let workers = requested_workers.min(tasks.len());
984    let next = AtomicUsize::new(0);
985    let outcomes = Mutex::new(Vec::<Result<ProcessOutcome, String>>::with_capacity(
986        tasks.len(),
987    ));
988    std::thread::scope(|scope| {
989        for _ in 0..workers {
990            scope.spawn(|| {
991                loop {
992                    let index = next.fetch_add(1, Ordering::Relaxed);
993                    let Some(task) = tasks.get(index) else { break };
994                    outcomes
995                        .lock()
996                        .expect("Rust compiler result lock poisoned")
997                        .push(run_process(project_root, task, supervisor, options));
998                }
999            });
1000        }
1001    });
1002    let mut outcomes = outcomes
1003        .into_inner()
1004        .map_err(|_| RustCompilerTestError::Context("Rust compiler result lock poisoned".into()))?
1005        .into_iter()
1006        .collect::<Result<Vec<_>, _>>()
1007        .map_err(|reason| RustCompilerTestError::Launch {
1008            test: "unknown attempt".into(),
1009            reason,
1010        })?;
1011    outcomes.sort_by_key(|outcome| outcome.task.ordinal);
1012    Ok(outcomes)
1013}
1014
1015fn regular_directory(path: &Path) -> Result<PathBuf, RustCompilerTestError> {
1016    let metadata = fs::symlink_metadata(path).map_err(|error| io_error(path, error))?;
1017    if !metadata.file_type().is_dir() {
1018        return Err(RustCompilerTestError::UnsafeArtifact(
1019            path.display().to_string(),
1020        ));
1021    }
1022    canonicalize_simplified(path).map_err(|error| io_error(path, error))
1023}
1024
1025#[cfg(unix)]
1026fn sync_directory(path: &Path) -> Result<(), RustCompilerTestError> {
1027    fs::File::open(path)
1028        .and_then(|directory| directory.sync_all())
1029        .map_err(|error| io_error(path, error))
1030}
1031
1032#[cfg(not(unix))]
1033fn sync_directory(_path: &Path) -> Result<(), RustCompilerTestError> {
1034    Ok(())
1035}
1036
1037fn write_cargo_runner_unit(
1038    output_directory: &Path,
1039    unit: &RustCargoRunnerUnit,
1040) -> Result<PathBuf, RustCompilerTestError> {
1041    let artifact = unit.artifact.to_str().ok_or_else(|| {
1042        RustCompilerTestError::Context("Cargo test artifact path is not UTF-8".into())
1043    })?;
1044    let mut identity = Sha256::new();
1045    identity.update((unit.target.len() as u64).to_be_bytes());
1046    identity.update(unit.target.as_bytes());
1047    identity.update((artifact.len() as u64).to_be_bytes());
1048    identity.update(artifact.as_bytes());
1049    let digest = format!("{:x}", identity.finalize());
1050    let destination = output_directory.join(format!(
1051        "libtest-{:016}-{}.json",
1052        unit.invocation_ordinal,
1053        &digest[..24]
1054    ));
1055    let partial = output_directory.join(format!(
1056        ".libtest-{:016}-{}-{}.partial",
1057        unit.invocation_ordinal,
1058        &digest[..24],
1059        std::process::id()
1060    ));
1061    let bytes = serde_json::to_vec(unit)
1062        .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1063    let mut options = OpenOptions::new();
1064    options.write(true).create_new(true);
1065    #[cfg(unix)]
1066    {
1067        use std::os::unix::fs::OpenOptionsExt as _;
1068        options.mode(0o600);
1069    }
1070    let mut file = options
1071        .open(&partial)
1072        .map_err(|error| io_error(&partial, error))?;
1073    let write_result = (|| {
1074        file.write_all(&bytes)
1075            .map_err(|error| io_error(&partial, error))?;
1076        file.sync_all().map_err(|error| io_error(&partial, error))?;
1077        drop(file);
1078        fs::rename(&partial, &destination).map_err(|error| io_error(&destination, error))?;
1079        sync_directory(output_directory)
1080    })();
1081    if write_result.is_err() {
1082        let _ = fs::remove_file(&partial);
1083    }
1084    write_result.map(|()| destination)
1085}
1086
1087fn write_cargo_runner_failure(
1088    output_directory: &Path,
1089    failure: &RustCargoRunnerFailure,
1090) -> Result<PathBuf, RustCompilerTestError> {
1091    let destination =
1092        output_directory.join(format!("failure-{:016}.json", failure.invocation_ordinal));
1093    let partial = output_directory.join(format!(
1094        ".failure-{:016}-{}.partial",
1095        failure.invocation_ordinal,
1096        std::process::id()
1097    ));
1098    let bytes = serde_json::to_vec(failure)
1099        .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1100    let mut options = OpenOptions::new();
1101    options.write(true).create_new(true);
1102    #[cfg(unix)]
1103    {
1104        use std::os::unix::fs::OpenOptionsExt as _;
1105        options.mode(0o600);
1106    }
1107    let mut file = options
1108        .open(&partial)
1109        .map_err(|error| io_error(&partial, error))?;
1110    let write_result = (|| {
1111        file.write_all(&bytes)
1112            .map_err(|error| io_error(&partial, error))?;
1113        file.sync_all().map_err(|error| io_error(&partial, error))?;
1114        drop(file);
1115        fs::rename(&partial, &destination).map_err(|error| io_error(&destination, error))?;
1116        sync_directory(output_directory)
1117    })();
1118    if write_result.is_err() {
1119        let _ = fs::remove_file(&partial);
1120    }
1121    write_result.map(|()| destination)
1122}
1123
1124fn reserve_cargo_runner_ordinal(output_directory: &Path) -> Result<u64, RustCompilerTestError> {
1125    for ordinal in 0..1_000_000_u64 {
1126        let reservation = output_directory.join(format!(".sequence-{ordinal:016}.reserved"));
1127        let mut options = OpenOptions::new();
1128        options.write(true).create_new(true);
1129        #[cfg(unix)]
1130        {
1131            use std::os::unix::fs::OpenOptionsExt as _;
1132            options.mode(0o600);
1133        }
1134        match options.open(&reservation) {
1135            Ok(file) => {
1136                file.sync_all()
1137                    .map_err(|error| io_error(&reservation, error))?;
1138                sync_directory(output_directory)?;
1139                return Ok(ordinal);
1140            }
1141            Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}
1142            Err(error) => return Err(io_error(&reservation, error)),
1143        }
1144    }
1145    Err(RustCompilerTestError::Context(
1146        "Cargo runner invocation ordinal space is exhausted".into(),
1147    ))
1148}
1149
1150fn run_nextest_list_passthrough(
1151    current_directory: &Path,
1152    artifact: &TestArtifact,
1153    underlying_runner: Option<&RustCargoResolvedRunner>,
1154    arguments: Vec<OsString>,
1155    watchdog_program: Option<&Path>,
1156    stdout: &mut dyn Write,
1157    stderr: &mut dyn Write,
1158) -> Result<RustCargoRunnerExecution, RustCompilerTestError> {
1159    let supervisor = watchdog_program
1160        .map_or_else(ProcessSupervisor::new, ProcessSupervisor::new_crash_safe)
1161        .map_err(|error| RustCompilerTestError::Launch {
1162            test: "nextest list".into(),
1163            reason: error.to_string(),
1164        })?;
1165    let options =
1166        SupervisionOptions::from_environment().map_err(|error| RustCompilerTestError::Launch {
1167            test: "nextest list".into(),
1168            reason: error.to_string(),
1169        })?;
1170    let (program, arguments) = artifact_command(artifact, underlying_runner, arguments);
1171    let output = supervisor
1172        .supervise_captured(
1173            &CommandSpec {
1174                program,
1175                arguments,
1176                cwd: current_directory.to_owned(),
1177                environment: Some(inherited_environment([])),
1178                captured_output: None,
1179            },
1180            options,
1181            &mut io::sink(),
1182        )
1183        .map_err(|error| RustCompilerTestError::Launch {
1184            test: "nextest list".into(),
1185            reason: error.to_string(),
1186        })?;
1187    stdout
1188        .write_all(&output.stdout)
1189        .map_err(|error| io_error(current_directory, error))?;
1190    stderr
1191        .write_all(&output.stderr)
1192        .map_err(|error| io_error(current_directory, error))?;
1193    Ok(RustCargoRunnerExecution {
1194        exit_code: output.result.exit_code(),
1195        unit_path: None,
1196    })
1197}
1198
1199pub fn run_cargo_libtest_runner(
1200    config_path: &Path,
1201    arguments: Vec<OsString>,
1202    watchdog_program: Option<PathBuf>,
1203    stdout: &mut dyn Write,
1204    stderr: &mut dyn Write,
1205) -> Result<RustCargoRunnerExecution, RustCompilerTestError> {
1206    let config_metadata =
1207        fs::symlink_metadata(config_path).map_err(|error| io_error(config_path, error))?;
1208    if !config_metadata.file_type().is_file() {
1209        return Err(RustCompilerTestError::UnsafeArtifact(
1210            config_path.display().to_string(),
1211        ));
1212    }
1213    let config: RustCargoRunnerConfig = serde_json::from_slice(
1214        &fs::read(config_path).map_err(|error| io_error(config_path, error))?,
1215    )
1216    .map_err(|error| {
1217        RustCompilerTestError::Context(format!("invalid Cargo runner config: {error}"))
1218    })?;
1219    if config.version != RUST_CARGO_RUNNER_VERSION
1220        || !config.run_id.starts_with("run_")
1221        || config.run_id.len() != 20
1222        || !config.run_id[4..]
1223            .bytes()
1224            .all(|byte| byte.is_ascii_hexdigit())
1225    {
1226        return Err(RustCompilerTestError::Context(
1227            "Cargo runner config has an unsupported version or invalid run ID".into(),
1228        ));
1229    }
1230    let target_directory = regular_directory(&config.target_directory)?;
1231    let output_directory = regular_directory(&config.output_directory)?;
1232    let run_root = target_directory
1233        .parent()
1234        .ok_or_else(|| RustCompilerTestError::Context("Cargo target has no run root".into()))?;
1235    if !output_directory.starts_with(run_root) || output_directory == target_directory {
1236        return Err(RustCompilerTestError::UnsafeArtifact(
1237            output_directory.display().to_string(),
1238        ));
1239    }
1240    let mut configured_targets = BTreeSet::new();
1241    if config.target_runners.is_empty()
1242        || config
1243            .target_runners
1244            .iter()
1245            .any(|target| target.target.is_empty() || !configured_targets.insert(&target.target))
1246    {
1247        return Err(RustCompilerTestError::Context(
1248            "Cargo runner config has empty or duplicate target identities".into(),
1249        ));
1250    }
1251    let mut configured_artifacts = BTreeSet::new();
1252    if config.artifacts.iter().any(|artifact| {
1253        !configured_artifacts.insert(&artifact.executable)
1254            || !artifact.executable.starts_with(&target_directory)
1255    }) {
1256        return Err(RustCompilerTestError::Context(
1257            "Cargo runner config has duplicate or out-of-target artifacts".into(),
1258        ));
1259    }
1260    let runner_identity = classify_rust_runner_environment()
1261        .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1262    let run_id = config.run_id.clone();
1263    let failure_target = arguments
1264        .first()
1265        .and_then(|target| target.clone().into_string().ok());
1266    let failure_artifact = arguments.get(1).map(PathBuf::from);
1267    let mut runner_arguments = arguments.into_iter();
1268    let target = runner_arguments
1269        .next()
1270        .ok_or_else(|| {
1271            RustCompilerTestError::Context("Cargo runner received no target identity".into())
1272        })?
1273        .into_string()
1274        .map_err(|_| {
1275            RustCompilerTestError::Context(
1276                "Cargo runner received a non-UTF-8 target identity".into(),
1277            )
1278        })?;
1279    let target_runner = config
1280        .target_runners
1281        .iter()
1282        .find(|candidate| candidate.target == target)
1283        .ok_or_else(|| {
1284            RustCompilerTestError::Context(format!(
1285                "Cargo runner received an unconfigured target identity: {target}"
1286            ))
1287        })?;
1288    let artifact_argument = runner_arguments.next().ok_or_else(|| {
1289        RustCompilerTestError::Context("Cargo runner received no artifact".into())
1290    })?;
1291    let artifact_path = PathBuf::from(&artifact_argument);
1292    let artifact =
1293        canonicalize_simplified(&artifact_path).map_err(|error| io_error(&artifact_path, error))?;
1294    if !artifact.starts_with(&target_directory)
1295        || !fs::symlink_metadata(&artifact).is_ok_and(|metadata| metadata.file_type().is_file())
1296    {
1297        return Err(RustCompilerTestError::UnsafeArtifact(
1298            artifact.display().to_string(),
1299        ));
1300    }
1301    let test_harness = config
1302        .artifacts
1303        .iter()
1304        .find(|candidate| candidate.executable == artifact)
1305        .map(|candidate| candidate.test_harness);
1306    if test_harness.is_none()
1307        && !matches!(
1308            runner_identity,
1309            RustRunnerInvocationIdentity::NextestList(_)
1310        )
1311    {
1312        return Err(RustCompilerTestError::Context(format!(
1313            "Cargo runner executed an unclassified artifact: {}",
1314            artifact.display()
1315        )));
1316    }
1317    let arguments = runner_arguments.collect::<Vec<_>>();
1318    let current_directory = std::env::current_dir()
1319        .and_then(fs::canonicalize)
1320        .map_err(|error| io_error(Path::new("."), error))?;
1321    let test_artifact = TestArtifact {
1322        executable: artifact.clone(),
1323        runner_argument: Some(artifact_argument),
1324        package: "cargo-pending".into(),
1325        target_key: "cargo-pending".into(),
1326        kind: "cargo-pending".into(),
1327        source: "cargo-pending".into(),
1328        test_harness: test_harness.unwrap_or(true),
1329    };
1330    let underlying_runner = target_runner.underlying_runner.clone();
1331    if matches!(
1332        runner_identity,
1333        RustRunnerInvocationIdentity::NextestList(_)
1334    ) {
1335        return run_nextest_list_passthrough(
1336            &current_directory,
1337            &test_artifact,
1338            underlying_runner.as_ref(),
1339            arguments,
1340            watchdog_program.as_deref(),
1341            stdout,
1342            stderr,
1343        );
1344    }
1345
1346    let invocation_ordinal = reserve_cargo_runner_ordinal(&output_directory)?;
1347    let result = (|| {
1348        let utf8_arguments = arguments
1349            .iter()
1350            .cloned()
1351            .map(|argument| {
1352                argument.into_string().map_err(|_| {
1353                    RustCompilerTestError::Context(
1354                        "Cargo runner received a non-UTF-8 libtest argument".into(),
1355                    )
1356                })
1357            })
1358            .collect::<Result<Vec<_>, _>>()?;
1359        let invocation = crate::rust_test_runner::CargoTestInvocation {
1360            program: "cargo".into(),
1361            kind: crate::rust_test_runner::RustCargoCommandKind::CargoTest,
1362            arguments: vec!["test".into()],
1363            runner_arguments: utf8_arguments.clone(),
1364        };
1365        let artifact_digest = format!(
1366            "{:x}",
1367            Sha256::digest(artifact.as_os_str().as_encoded_bytes())
1368        );
1369        let transport_directory = output_directory.join("attempts").join(format!(
1370            "{invocation_ordinal:016}-{}",
1371            &artifact_digest[..24]
1372        ));
1373        fs::create_dir_all(&transport_directory)
1374            .map_err(|error| io_error(&transport_directory, error))?;
1375        let transport_directory = regular_directory(&transport_directory)?;
1376        let supervisor = watchdog_program
1377            .as_deref()
1378            .map_or_else(ProcessSupervisor::new, ProcessSupervisor::new_crash_safe)
1379            .map_err(|error| RustCompilerTestError::Launch {
1380                test: "Cargo runner".into(),
1381                reason: error.to_string(),
1382            })?;
1383        let options = SupervisionOptions::from_environment().map_err(|error| {
1384            RustCompilerTestError::Launch {
1385                test: "Cargo runner".into(),
1386                reason: error.to_string(),
1387            }
1388        })?;
1389        let (runner, runner_run_id, runner_version, runner_binary_id, requested_workers, attempts) =
1390            match &runner_identity {
1391                RustRunnerInvocationIdentity::CargoSingleAttempt if test_artifact.test_harness => {
1392                    let selection = rust_libtest_selection(&invocation).map_err(|error| {
1393                        RustCompilerTestError::UnsupportedCommand(error.to_string())
1394                    })?;
1395                    let tests = list_tests(
1396                        &current_directory,
1397                        &test_artifact,
1398                        &selection.list_arguments,
1399                        underlying_runner.as_ref(),
1400                        &supervisor,
1401                        options,
1402                        &transport_directory.join("list.events"),
1403                    )?;
1404                    let contexts = preflight_rust_test_contexts(tests.clone())
1405                        .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1406                    let stock = run_stock_libtest_artifact(
1407                        &current_directory,
1408                        &test_artifact,
1409                        underlying_runner.as_ref(),
1410                        arguments.clone(),
1411                        &tests,
1412                        &contexts,
1413                        &transport_directory.join("artifact.mmap"),
1414                        &transport_directory.join("artifact.libtest-events"),
1415                        &supervisor,
1416                        options,
1417                    )?;
1418                    let StockLibtestExecution {
1419                        output,
1420                        events,
1421                        mut partition,
1422                        started_at_ms,
1423                        ended_at_ms,
1424                    } = stock;
1425                    let exit_code = output.result.exit_code();
1426                    stdout
1427                        .write_all(&output.stdout)
1428                        .and_then(|()| stderr.write_all(&output.stderr))
1429                        .map_err(|error| io_error(&output_directory, error))?;
1430                    let mut attempts = Vec::with_capacity(tests.len());
1431                    for attempt in events.attempts {
1432                        let context_id = contexts[&attempt.name];
1433                        let index = attempts.len();
1434                        attempts.push(RustCargoRunnerAttempt {
1435                            test: attempt.name,
1436                            context_id,
1437                            retry: 0,
1438                            total_attempts: 1,
1439                            runner_attempt_id: format!(
1440                                "{}:cargo:{invocation_ordinal:016}:{index:08}",
1441                                config.run_id
1442                            ),
1443                            outcome: RustCargoRunnerAttemptOutcome::Libtest {
1444                                result: attempt.result,
1445                                timed_out: attempt.timed_out,
1446                            },
1447                            transport: partition
1448                                .attributed
1449                                .remove(&context_id)
1450                                .expect("every selected test context was partitioned"),
1451                        });
1452                    }
1453                    for test in events.unstarted {
1454                        let context_id = contexts[&test];
1455                        let index = attempts.len();
1456                        attempts.push(RustCargoRunnerAttempt {
1457                            test,
1458                            context_id,
1459                            retry: 0,
1460                            total_attempts: 1,
1461                            runner_attempt_id: format!(
1462                                "{}:cargo:{invocation_ordinal:016}:{index:08}",
1463                                config.run_id
1464                            ),
1465                            outcome: RustCargoRunnerAttemptOutcome::Unstarted,
1466                            transport: partition
1467                                .attributed
1468                                .remove(&context_id)
1469                                .expect("every unstarted test context was partitioned"),
1470                        });
1471                    }
1472                    if !partition.attributed.is_empty() {
1473                        return Err(RustCompilerTestError::Context(
1474                            "stock libtest event join left selected test contexts unclaimed".into(),
1475                        ));
1476                    }
1477                    let unit = RustCargoRunnerUnit {
1478                        version: RUST_CARGO_RUNNER_VERSION,
1479                        run_id: run_id.clone(),
1480                        invocation_ordinal,
1481                        runner: RustCargoRunnerKind::CargoTest,
1482                        runner_run_id: None,
1483                        runner_version: None,
1484                        runner_binary_id: None,
1485                        target,
1486                        artifact,
1487                        arguments: utf8_arguments,
1488                        invocation: RustCargoRunnerInvocation {
1489                            result: output.result,
1490                            started_at_ms,
1491                            ended_at_ms,
1492                            stdout: output.stdout,
1493                            stderr: output.stderr,
1494                            background_transport: partition.background,
1495                        },
1496                        attempts,
1497                        thread_scope_limitations: partition.thread_scope_limitations,
1498                    };
1499                    let unit_path = write_cargo_runner_unit(&output_directory, &unit)?;
1500                    fs::remove_dir(&transport_directory)
1501                        .map_err(|error| io_error(&transport_directory, error))?;
1502                    return Ok(RustCargoRunnerExecution {
1503                        exit_code,
1504                        unit_path: Some(unit_path),
1505                    });
1506                }
1507                RustRunnerInvocationIdentity::CargoSingleAttempt => (
1508                    RustCargoRunnerKind::CargoCustomHarness,
1509                    None,
1510                    None,
1511                    None,
1512                    1,
1513                    vec![(
1514                        "custom-harness".into(),
1515                        0,
1516                        1,
1517                        format!("{}:cargo:{invocation_ordinal:016}:00000000", config.run_id),
1518                        arguments.clone(),
1519                    )],
1520                ),
1521                RustRunnerInvocationIdentity::NextestAttempt(NextestAttemptIdentity {
1522                    invocation,
1523                    test_name,
1524                    retry,
1525                    total_attempts,
1526                    runner_attempt_id,
1527                }) => {
1528                    if !test_artifact.test_harness {
1529                        return Err(RustCompilerTestError::Context(
1530                            "nextest attempted to execute a custom Cargo harness".into(),
1531                        ));
1532                    }
1533                    if !utf8_arguments
1534                        .windows(2)
1535                        .any(|pair| pair == ["--exact", test_name])
1536                    {
1537                        return Err(RustCompilerTestError::Context(
1538                            "nextest target-runner arguments do not select NEXTEST_TEST_NAME exactly"
1539                                .into(),
1540                        ));
1541                    }
1542                    (
1543                        RustCargoRunnerKind::Nextest,
1544                        Some(invocation.run_id.clone()),
1545                        Some(invocation.version.clone()),
1546                        Some(invocation.binary_id.clone()),
1547                        1,
1548                        vec![(
1549                            test_name.clone(),
1550                            *retry,
1551                            *total_attempts,
1552                            runner_attempt_id.clone(),
1553                            arguments.clone(),
1554                        )],
1555                    )
1556                }
1557                RustRunnerInvocationIdentity::NextestList(_) => unreachable!("handled above"),
1558            };
1559        let tests = attempts
1560            .iter()
1561            .map(|(test, _, _, _, _)| test.clone())
1562            .collect::<Vec<_>>();
1563        let contexts = preflight_rust_test_contexts(tests.clone())
1564            .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1565        let tasks = attempts
1566            .into_iter()
1567            .enumerate()
1568            .map(
1569                |(index, (test, retry, total_attempts, runner_attempt_id, test_arguments))| {
1570                    ProcessTask {
1571                        ordinal: index,
1572                        artifact_index: 0,
1573                        artifact: test_artifact.clone(),
1574                        test: test.clone(),
1575                        test_id: format!("rust:cargo-runner:{}::{test}", &artifact_digest[..24]),
1576                        context_id: contexts[&test],
1577                        retry,
1578                        total_attempts,
1579                        runner_attempt_id,
1580                        runner,
1581                        transport: transport_directory.join(format!("{index:08}.mmap")),
1582                        libtest_events: transport_directory
1583                            .join(format!("{index:08}.libtest-events")),
1584                        test_arguments,
1585                        underlying_runner: underlying_runner.clone(),
1586                    }
1587                },
1588            )
1589            .collect::<Vec<_>>();
1590        let mut outcomes = execute_process_tasks(
1591            &current_directory,
1592            &tasks,
1593            requested_workers,
1594            &supervisor,
1595            options,
1596        )?;
1597        let [outcome] = outcomes.as_mut_slice() else {
1598            return Err(RustCompilerTestError::Context(
1599                "custom-harness and nextest runner invocations must own exactly one process".into(),
1600            ));
1601        };
1602        stdout
1603            .write_all(&outcome.output.stdout)
1604            .and_then(|()| stderr.write_all(&outcome.output.stderr))
1605            .map_err(|error| io_error(&output_directory, error))?;
1606        let exit_code = outcome.output.result.exit_code();
1607        let roots = BTreeSet::from([outcome.task.context_id]);
1608        let mut partition = partition_rust_transport_by_test_contexts(&outcome.read, &roots)
1609            .map_err(|error| RustCompilerTestError::Transport {
1610                test: outcome.task.test.clone(),
1611                reason: error.to_string(),
1612            })?;
1613        let attempt_outcome = match runner {
1614            RustCargoRunnerKind::CargoCustomHarness => RustCargoRunnerAttemptOutcome::OpaqueProcess,
1615            RustCargoRunnerKind::Nextest => match outcome.attempt_outcome {
1616                RustCargoRunnerAttemptOutcome::Libtest { result, timed_out } => {
1617                    RustCargoRunnerAttemptOutcome::Libtest { result, timed_out }
1618                }
1619                _ => {
1620                    return Err(RustCompilerTestError::Context(
1621                        "nextest process has no authenticated libtest terminal event".into(),
1622                    ));
1623                }
1624            },
1625            RustCargoRunnerKind::CargoTest => unreachable!("stock libtest returned above"),
1626        };
1627        let attempts = vec![RustCargoRunnerAttempt {
1628            test: outcome.task.test.clone(),
1629            context_id: outcome.task.context_id,
1630            retry: outcome.task.retry,
1631            total_attempts: outcome.task.total_attempts,
1632            runner_attempt_id: outcome.task.runner_attempt_id.clone(),
1633            outcome: attempt_outcome,
1634            transport: partition
1635                .attributed
1636                .remove(&outcome.task.context_id)
1637                .expect("the one attempt context was partitioned"),
1638        }];
1639        let unit = RustCargoRunnerUnit {
1640            version: RUST_CARGO_RUNNER_VERSION,
1641            run_id: run_id.clone(),
1642            invocation_ordinal,
1643            runner,
1644            runner_run_id,
1645            runner_version,
1646            runner_binary_id,
1647            target,
1648            artifact,
1649            arguments: utf8_arguments,
1650            invocation: RustCargoRunnerInvocation {
1651                result: outcome.output.result.clone(),
1652                started_at_ms: outcome.started_at_ms,
1653                ended_at_ms: outcome.ended_at_ms,
1654                stdout: outcome.output.stdout.clone(),
1655                stderr: outcome.output.stderr.clone(),
1656                background_transport: partition.background,
1657            },
1658            attempts,
1659            thread_scope_limitations: partition.thread_scope_limitations,
1660        };
1661        let unit_path = write_cargo_runner_unit(&output_directory, &unit)?;
1662        fs::remove_dir(&transport_directory)
1663            .map_err(|error| io_error(&transport_directory, error))?;
1664        Ok(RustCargoRunnerExecution {
1665            exit_code,
1666            unit_path: Some(unit_path),
1667        })
1668    })();
1669    if let Err(error) = &result {
1670        let failure = RustCargoRunnerFailure {
1671            version: RUST_CARGO_RUNNER_VERSION,
1672            run_id,
1673            invocation_ordinal,
1674            target: failure_target,
1675            artifact: failure_artifact,
1676            error: error.to_string(),
1677        };
1678        if let Err(publication_error) = write_cargo_runner_failure(&output_directory, &failure) {
1679            return Err(RustCompilerTestError::Context(format!(
1680                "{error}; Cargo runner also could not publish its failure: {publication_error}"
1681            )));
1682        }
1683    }
1684    result
1685}
1686
1687pub fn read_cargo_runner_units(
1688    output_directory: &Path,
1689    run_id: &str,
1690    expected_targets: &[String],
1691) -> Result<Vec<RustCargoRunnerUnit>, RustCompilerTestError> {
1692    let output_directory = regular_directory(output_directory)?;
1693    let expected_target_count = expected_targets.len();
1694    let expected_targets = expected_targets.iter().collect::<BTreeSet<_>>();
1695    if expected_targets.is_empty() || expected_targets.len() != expected_target_count {
1696        return Err(RustCompilerTestError::Context(
1697            "Cargo runner expected-target set is empty or duplicated".into(),
1698        ));
1699    }
1700    let mut reservations = BTreeSet::new();
1701    let mut units = Vec::new();
1702    let mut failures = Vec::new();
1703    let mut retained_attempt_state = false;
1704    for entry in
1705        fs::read_dir(&output_directory).map_err(|error| io_error(&output_directory, error))?
1706    {
1707        let entry = entry.map_err(|error| io_error(&output_directory, error))?;
1708        let path = entry.path();
1709        let metadata = fs::symlink_metadata(&path).map_err(|error| io_error(&path, error))?;
1710        let name = entry.file_name();
1711        let name = name.to_str().ok_or_else(|| {
1712            RustCompilerTestError::Context("Cargo runner output name is not UTF-8".into())
1713        })?;
1714        if name == "attempts" && metadata.file_type().is_dir() {
1715            retained_attempt_state = fs::read_dir(&path)
1716                .map_err(|error| io_error(&path, error))?
1717                .next()
1718                .transpose()
1719                .map_err(|error| io_error(&path, error))?
1720                .is_some();
1721            continue;
1722        }
1723        if name.starts_with(".sequence-") && name.ends_with(".reserved") {
1724            if !metadata.file_type().is_file() || metadata.len() != 0 {
1725                return Err(RustCompilerTestError::UnsafeArtifact(
1726                    path.display().to_string(),
1727                ));
1728            }
1729            let ordinal = name
1730                .strip_prefix(".sequence-")
1731                .and_then(|name| name.strip_suffix(".reserved"))
1732                .filter(|value| {
1733                    value.len() == 16 && value.bytes().all(|byte| byte.is_ascii_digit())
1734                })
1735                .and_then(|value| value.parse::<u64>().ok())
1736                .ok_or_else(|| {
1737                    RustCompilerTestError::Context("malformed Cargo runner reservation".into())
1738                })?;
1739            if !reservations.insert(ordinal) {
1740                return Err(RustCompilerTestError::Context(
1741                    "duplicate Cargo runner reservation".into(),
1742                ));
1743            }
1744            continue;
1745        }
1746        if name.starts_with("failure-") && name.ends_with(".json") {
1747            if !metadata.file_type().is_file() {
1748                return Err(RustCompilerTestError::UnsafeArtifact(
1749                    path.display().to_string(),
1750                ));
1751            }
1752            let failure: RustCargoRunnerFailure =
1753                serde_json::from_slice(&fs::read(&path).map_err(|error| io_error(&path, error))?)
1754                    .map_err(|error| {
1755                    RustCompilerTestError::Context(format!(
1756                        "invalid Cargo runner failure unit: {error}"
1757                    ))
1758                })?;
1759            if failure.version != RUST_CARGO_RUNNER_VERSION || failure.run_id != run_id {
1760                return Err(RustCompilerTestError::Context(
1761                    "Cargo runner failure unit has incompatible identity".into(),
1762                ));
1763            }
1764            failures.push(failure);
1765            continue;
1766        }
1767        if !name.starts_with("libtest-")
1768            || !name.ends_with(".json")
1769            || !metadata.file_type().is_file()
1770        {
1771            return Err(RustCompilerTestError::UnsafeArtifact(
1772                path.display().to_string(),
1773            ));
1774        }
1775        let unit: RustCargoRunnerUnit =
1776            serde_json::from_slice(&fs::read(&path).map_err(|error| io_error(&path, error))?)
1777                .map_err(|error| {
1778                    RustCompilerTestError::Context(format!("invalid Cargo runner unit: {error}"))
1779                })?;
1780        if unit.version != RUST_CARGO_RUNNER_VERSION || unit.run_id != run_id {
1781            return Err(RustCompilerTestError::Context(
1782                "Cargo runner unit has incompatible identity".into(),
1783            ));
1784        }
1785        if !expected_targets.contains(&unit.target) {
1786            return Err(RustCompilerTestError::Context(format!(
1787                "Cargo runner unit has an unselected target identity: {}",
1788                unit.target
1789            )));
1790        }
1791        validate_persisted_runner_transport(&unit)?;
1792        units.push(unit);
1793    }
1794    units.sort_by_key(|unit| unit.invocation_ordinal);
1795    failures.sort_by_key(|failure| failure.invocation_ordinal);
1796    let mut publications = BTreeSet::new();
1797    for ordinal in units
1798        .iter()
1799        .map(|unit| unit.invocation_ordinal)
1800        .chain(failures.iter().map(|failure| failure.invocation_ordinal))
1801    {
1802        if !reservations.contains(&ordinal) || !publications.insert(ordinal) {
1803            return Err(RustCompilerTestError::Context(
1804                "Cargo runner invocation publications are malformed or duplicated".into(),
1805            ));
1806        }
1807    }
1808    if reservations.len() != publications.len()
1809        || reservations
1810            .iter()
1811            .enumerate()
1812            .any(|(expected, ordinal)| *ordinal != expected as u64)
1813    {
1814        return Err(RustCompilerTestError::Context(
1815            "Cargo runner reserved an invocation without publishing its unit".into(),
1816        ));
1817    }
1818    if let Some(failure) = failures.first() {
1819        return Err(RustCompilerTestError::Context(format!(
1820            "Cargo runner invocation {} failed for {}: {}",
1821            failure.invocation_ordinal,
1822            failure.artifact.as_ref().map_or_else(
1823                || "an unknown artifact".into(),
1824                |path| path.display().to_string()
1825            ),
1826            failure.error
1827        )));
1828    }
1829    if retained_attempt_state {
1830        return Err(RustCompilerTestError::Context(
1831            "Cargo runner retained attempt transport state".into(),
1832        ));
1833    }
1834    let runner_kinds = units
1835        .iter()
1836        .map(|unit| unit.runner)
1837        .collect::<BTreeSet<_>>();
1838    if runner_kinds.contains(&RustCargoRunnerKind::Nextest) && runner_kinds.len() > 1 {
1839        return Err(RustCompilerTestError::Context(
1840            "Cargo runner mixed standard Cargo and nextest units".into(),
1841        ));
1842    }
1843    let mut attempt_ids = BTreeSet::new();
1844    if units.iter().flat_map(|unit| &unit.attempts).any(|attempt| {
1845        attempt.runner_attempt_id.is_empty()
1846            || !attempt_ids.insert(attempt.runner_attempt_id.clone())
1847    }) {
1848        return Err(RustCompilerTestError::Context(
1849            "Cargo runner attempt identity is empty or duplicated".into(),
1850        ));
1851    }
1852    if runner_kinds.contains(&RustCargoRunnerKind::Nextest) {
1853        let mut nextest_identity = None;
1854        let mut logical_attempts =
1855            BTreeMap::<(String, PathBuf, String), Vec<&RustCargoRunnerAttempt>>::new();
1856        for unit in &units {
1857            let identity = (unit.runner_run_id.as_ref(), unit.runner_version.as_ref());
1858            if identity.0.is_none()
1859                || identity.1.is_none()
1860                || unit.runner_binary_id.is_none()
1861                || unit.attempts.len() != 1
1862            {
1863                return Err(RustCompilerTestError::Context(
1864                    "nextest runner unit lacks exact invocation or attempt identity".into(),
1865                ));
1866            }
1867            match &nextest_identity {
1868                Some(expected) if *expected != identity => {
1869                    return Err(RustCompilerTestError::Context(
1870                        "nextest runner units belong to different runs or versions".into(),
1871                    ));
1872                }
1873                None => nextest_identity = Some(identity),
1874                _ => {}
1875            }
1876            let attempt = &unit.attempts[0];
1877            logical_attempts
1878                .entry((
1879                    unit.target.clone(),
1880                    unit.artifact.clone(),
1881                    attempt.test.clone(),
1882                ))
1883                .or_default()
1884                .push(attempt);
1885        }
1886        for attempts in logical_attempts.values_mut() {
1887            attempts.sort_by_key(|attempt| attempt.retry);
1888            let total_attempts = attempts[0].total_attempts;
1889            for (expected_retry, attempt) in attempts.iter().enumerate() {
1890                if attempt.retry != expected_retry
1891                    || attempt.total_attempts != total_attempts
1892                    || attempt.retry >= total_attempts
1893                    || (expected_retry + 1 < attempts.len()
1894                        && attempt_outcome_succeeded(&attempt.outcome))
1895                {
1896                    return Err(RustCompilerTestError::Context(
1897                        "nextest retry sequence is noncontiguous, inconsistent, or continues after success"
1898                            .into(),
1899                    ));
1900                }
1901            }
1902        }
1903    } else {
1904        let mut artifacts = BTreeSet::new();
1905        if units.iter().any(|unit| {
1906            unit.runner_run_id.is_some()
1907                || unit.runner_version.is_some()
1908                || unit.runner_binary_id.is_some()
1909                || !artifacts.insert((unit.target.clone(), unit.artifact.clone()))
1910                || unit
1911                    .attempts
1912                    .iter()
1913                    .any(|attempt| attempt.retry != 0 || attempt.total_attempts != 1)
1914                || (unit.runner == RustCargoRunnerKind::CargoCustomHarness
1915                    && (unit.attempts.len() != 1 || unit.attempts[0].test != "custom-harness"))
1916        }) {
1917            return Err(RustCompilerTestError::Context(
1918                "Cargo runner units violate the single-attempt artifact identity contract".into(),
1919            ));
1920        }
1921    }
1922    Ok(units)
1923}
1924
1925fn rust_compiler_coverage_model() -> CoverageModelDeclaration {
1926    CoverageModelDeclaration {
1927        language: "rust".into(),
1928        variant: "rustc-mir-owned-v1".into(),
1929        name: "supercov-rust-compiler-v1".into(),
1930        completeness_meaning: "Every compiler-derived obligation in the frozen owned-source denominator was observed; explicit compiler limitations identify Rust surfaces not yet measured.".into(),
1931        measured: vec![
1932            "compiler-derived owned Rust statement and function-entry obligations".into(),
1933            "compiler-derived control-flow alternatives and atomic decision vectors".into(),
1934            "macro-expanded and generated owned code with exact compiler provenance".into(),
1935            "exact stock-libtest in-process context attribution".into(),
1936            "exact process-per-custom-harness-invocation attribution".into(),
1937            "exact assertion-phase attribution for supported assertion macros".into(),
1938        ],
1939        not_measured: vec![
1940            "capabilities explicitly listed in the compiler manifest limitations".into(),
1941            "all input values, semantic partitions, paths, or concurrency interleavings".into(),
1942            "mutation score or assertion fault-detection strength".into(),
1943        ],
1944    }
1945}
1946
1947fn runner_declaration() -> FrontendRunnerDeclaration {
1948    FrontendRunnerDeclaration {
1949        runner: "rust-libtest".into(),
1950        execution_model: ExecutionModel::ParallelContextPropagated,
1951        attribution: FrontendAttribution {
1952            run: AttributionPrecision::Exact,
1953            worker: AttributionPrecision::Exact,
1954            test: AttributionPrecision::Exact,
1955            retry: AttributionPrecision::Exact,
1956            phase: AttributionPrecision::Exact,
1957            action: AttributionPrecision::Unavailable,
1958            assertion: AttributionPrecision::Exact,
1959        },
1960        limitations: vec![FrontendLimitation {
1961            id: "rust-action-linkage-unavailable".into(),
1962            scopes: vec![FrontendLimitationScope::Action],
1963            reason: "Rust libtest exposes no general application-action lifecycle".into(),
1964        }],
1965    }
1966}
1967
1968fn nextest_runner_declaration() -> FrontendRunnerDeclaration {
1969    let mut declaration = runner_declaration();
1970    declaration.runner = "rust-nextest".into();
1971    declaration.execution_model = ExecutionModel::ProcessPerTest;
1972    declaration.limitations[0].reason =
1973        "nextest exposes no general application-action lifecycle".into();
1974    declaration
1975}
1976
1977fn custom_harness_runner_declaration() -> FrontendRunnerDeclaration {
1978    let mut declaration = runner_declaration();
1979    declaration.runner = "rust-custom-harness".into();
1980    declaration.execution_model = ExecutionModel::ProcessPerTest;
1981    declaration.limitations[0].id = "rust-custom-harness-action-linkage-unavailable".into();
1982    declaration.limitations[0].reason =
1983        "A custom Cargo harness exposes no general application-action lifecycle".into();
1984    declaration.limitations.push(FrontendLimitation {
1985        id: "rust-custom-harness-internal-tests-opaque".into(),
1986        scopes: vec![FrontendLimitationScope::Test],
1987        reason: "Cargo exposes a custom harness as one target invocation; Supercov attributes that invocation exactly without inventing internal test-case identities".into(),
1988    });
1989    declaration
1990}
1991
1992fn compiler_runner_declaration() -> FrontendRunnerDeclaration {
1993    FrontendRunnerDeclaration {
1994        runner: "rustc".into(),
1995        execution_model: ExecutionModel::ProcessPerTest,
1996        attribution: FrontendAttribution {
1997            run: AttributionPrecision::Exact,
1998            worker: AttributionPrecision::Exact,
1999            test: AttributionPrecision::Exact,
2000            retry: AttributionPrecision::Exact,
2001            phase: AttributionPrecision::Exact,
2002            action: AttributionPrecision::Unavailable,
2003            assertion: AttributionPrecision::Unavailable,
2004        },
2005        limitations: vec![
2006            FrontendLimitation {
2007                id: "rust-ctfe-action-linkage-unavailable".into(),
2008                scopes: vec![FrontendLimitationScope::Action],
2009                reason: "Compile-time evaluation is build execution, not an application action"
2010                    .into(),
2011            },
2012            FrontendLimitation {
2013                id: "rust-ctfe-assertion-linkage-unavailable".into(),
2014                scopes: vec![FrontendLimitationScope::Assertion],
2015                reason: "Compile-time execution has no user-test assertion lifecycle".into(),
2016            },
2017        ],
2018    }
2019}
2020
2021fn rustdoc_runner_declaration() -> FrontendRunnerDeclaration {
2022    FrontendRunnerDeclaration {
2023        runner: "rustdoc".into(),
2024        execution_model: ExecutionModel::ParallelContextPropagated,
2025        attribution: FrontendAttribution {
2026            run: AttributionPrecision::Exact,
2027            worker: AttributionPrecision::Exact,
2028            test: AttributionPrecision::Exact,
2029            retry: AttributionPrecision::Exact,
2030            phase: AttributionPrecision::Exact,
2031            action: AttributionPrecision::Unavailable,
2032            assertion: AttributionPrecision::Exact,
2033        },
2034        limitations: vec![FrontendLimitation {
2035            id: "rustdoc-action-linkage-unavailable".into(),
2036            scopes: vec![FrontendLimitationScope::Action],
2037            reason: "Rust doctests expose assertions but no general application-action lifecycle"
2038                .into(),
2039        }],
2040    }
2041}
2042
2043fn doctest_raw_results(
2044    run_id: &str,
2045    resolution: &RustdocOutcomeResolution,
2046    started_at_ms: i64,
2047    ended_at_ms: i64,
2048    normalized: &NormalizedRustCompilerManifest,
2049) -> Result<(Vec<RawTestResult>, Vec<RustCompilerTransportHealthRecord>), RustCompilerTestError> {
2050    let mut results = Vec::new();
2051    let mut health = Vec::new();
2052    for group in &resolution.groups {
2053        if group.transport.dropped != 0 {
2054            return Err(RustCompilerTestError::DroppedEvidence {
2055                test: format!("rustdoc:{}", group.group),
2056                dropped: group.transport.dropped,
2057            });
2058        }
2059        let worker_id = format!("rustdoc-{}", &group.invocation_id[..16]);
2060        let mut accounted_committed = 0_u64;
2061        for joined in &group.entries {
2062            let entry = &joined.catalog;
2063            let test_id = format!("rust:doctest:{}:{}:{}", group.group, entry.file, entry.line);
2064            let attempt_id = format!(
2065                "{run_id}:doctest:{}:{}",
2066                group.invocation_id, joined.catalog_index
2067            );
2068            let (status, error, started, completed) = match &joined.state {
2069                RustdocJoinedOutcomeState::Completed { outcome } => (
2070                    match outcome.status {
2071                        RustdocOutcomeStatus::Passed => "passed",
2072                        RustdocOutcomeStatus::Failed => "failed",
2073                        RustdocOutcomeStatus::Ignored => "skipped",
2074                    },
2075                    (outcome.status == RustdocOutcomeStatus::Failed).then(|| {
2076                        outcome
2077                            .message
2078                            .as_deref()
2079                            .or(outcome.reason.as_deref())
2080                            .unwrap_or("rustdoc reported a failed doctest")
2081                            .to_owned()
2082                    }),
2083                    true,
2084                    true,
2085                ),
2086                RustdocJoinedOutcomeState::UnfinishedStarted => (
2087                    "unknown",
2088                    Some("rustdoc fail-fast ended after this doctest started".into()),
2089                    true,
2090                    false,
2091                ),
2092                RustdocJoinedOutcomeState::Unstarted => (
2093                    "unknown",
2094                    Some("rustdoc fail-fast ended before this doctest started".into()),
2095                    false,
2096                    false,
2097                ),
2098                RustdocJoinedOutcomeState::FilteredOut => ("skipped", None, false, false),
2099                RustdocJoinedOutcomeState::NotRunAmbiguous => (
2100                    "unknown",
2101                    Some(
2102                        "rustdoc did not identify whether this doctest was filtered or left unstarted by fail-fast"
2103                            .into(),
2104                    ),
2105                    false,
2106                    false,
2107                ),
2108            };
2109            let mut phases = started
2110                .then(|| CoveragePhase {
2111                    id: phase_id(run_id, &attempt_id),
2112                    kind: "test".into(),
2113                    operation: format!("Rust doctest {}", entry.name),
2114                    source: Some(entry.file.clone()),
2115                    caused_by_phase_id: None,
2116                    // Pinned libtest reports duration but no wall-clock
2117                    // boundaries. The authenticated rustdoc invocation is the
2118                    // narrowest non-invented interval available; an unfinished
2119                    // test deliberately has no terminal timestamp.
2120                    started_at_ms,
2121                    ended_at_ms: completed.then_some(ended_at_ms),
2122                    status: Some(status.into()),
2123                    error: error.clone(),
2124                })
2125                .into_iter()
2126                .collect::<Vec<_>>();
2127            let (base_context, transport) =
2128                group.attributed_transport(joined).map_err(|error| {
2129                    RustCompilerTestError::Projection {
2130                        test: test_id.clone(),
2131                        reason: error.to_string(),
2132                    }
2133                })?;
2134            accounted_committed = accounted_committed
2135                .checked_add(transport.committed)
2136                .ok_or_else(|| RustCompilerTestError::Projection {
2137                    test: test_id.clone(),
2138                    reason: "rustdoc committed evidence count overflow".into(),
2139                })?;
2140            if !started && transport.committed != 0 {
2141                return Err(RustCompilerTestError::Projection {
2142                    test: test_id,
2143                    reason: "a filtered or unstarted doctest emitted runtime evidence".into(),
2144                });
2145            }
2146            let runtime = if let Some(base_phase) = phases.first() {
2147                let projection = project_rust_compiler_evidence(
2148                    base_context,
2149                    base_phase,
2150                    &transport,
2151                    normalized,
2152                )
2153                .map_err(|error| RustCompilerTestError::Projection {
2154                    test: test_id.clone(),
2155                    reason: error.to_string(),
2156                })?;
2157                if snapshot_has_evidence(&projection.background) {
2158                    return Err(RustCompilerTestError::Projection {
2159                        test: test_id.clone(),
2160                        reason: "doctest context partition retained background evidence".into(),
2161                    });
2162                }
2163                phases.extend(projection.assertion_phases);
2164                vec![projection.attributed]
2165            } else {
2166                Vec::new()
2167            };
2168            results.push(RawTestResult {
2169                test_id: Some(test_id.clone()),
2170                scope: Some(ExecutionScope {
2171                    version: 1,
2172                    run_id: run_id.into(),
2173                    worker_id: worker_id.clone(),
2174                    test_id: test_id.clone(),
2175                    test_key: test_id.clone(),
2176                    retry: 0,
2177                    attempt_id,
2178                }),
2179                test: test_id,
2180                test_file: Some(entry.file.clone()),
2181                title: Some(entry.name.clone()),
2182                retry: Some(0),
2183                status: Some(status.into()),
2184                expected_status: Some("passed".into()),
2185                flaky: false,
2186                provenance: TestProvenance {
2187                    runner: "rustdoc".into(),
2188                    kind: "doctest".into(),
2189                    project: Some(group.group.clone()),
2190                    source: "supercov-rustdoc-outcome".into(),
2191                },
2192                role: "test".into(),
2193                phases,
2194                runtime,
2195                browser: Vec::new(),
2196                server: Vec::new(),
2197            });
2198        }
2199
2200        let background =
2201            group
2202                .background_transport()
2203                .map_err(|error| RustCompilerTestError::Projection {
2204                    test: format!("rustdoc:{}", group.group),
2205                    reason: error.to_string(),
2206                })?;
2207        accounted_committed = accounted_committed
2208            .checked_add(background.committed)
2209            .ok_or_else(|| RustCompilerTestError::Projection {
2210                test: format!("rustdoc:{}", group.group),
2211                reason: "rustdoc background evidence count overflow".into(),
2212            })?;
2213        if accounted_committed != group.transport.committed {
2214            return Err(RustCompilerTestError::Projection {
2215                test: format!("rustdoc:{}", group.group),
2216                reason: format!(
2217                    "rustdoc context partition accounted for {accounted_committed} of {} committed records",
2218                    group.transport.committed
2219                ),
2220            });
2221        }
2222        if background.committed != 0 {
2223            let background_id = format!("background:rustdoc:{}", group.invocation_id);
2224            let background_phase = CoveragePhase {
2225                id: phase_id(run_id, &background_id),
2226                kind: "setup".into(),
2227                operation: format!("Background while running Rust doctests for {}", group.group),
2228                source: None,
2229                caused_by_phase_id: None,
2230                started_at_ms,
2231                ended_at_ms: Some(ended_at_ms),
2232                status: Some("passed".into()),
2233                error: None,
2234            };
2235            // The rustdoc background partition may hold join-bounded
2236            // quarantined thread evidence with real contexts; flatten every
2237            // record to context zero so escaped-thread work can never become
2238            // test or phase attributed downstream.
2239            let mut flattened = background.clone();
2240            for observation in &mut flattened.observations {
2241                observation.context_id = 0;
2242            }
2243            for hit in &mut flattened.ordinal_hits {
2244                hit.context_id = 0;
2245            }
2246            flattened.phases.clear();
2247            flattened.thread_phases.clear();
2248            flattened.thread_ends.clear();
2249            flattened.test_boundaries.clear();
2250            let projection =
2251                project_rust_compiler_evidence(1, &background_phase, &flattened, normalized)
2252                    .map_err(|error| RustCompilerTestError::Projection {
2253                        test: background_id.clone(),
2254                        reason: error.to_string(),
2255                    })?;
2256            if snapshot_has_evidence(&projection.attributed)
2257                || !projection.assertion_phases.is_empty()
2258            {
2259                return Err(RustCompilerTestError::Projection {
2260                    test: background_id,
2261                    reason: "context-zero doctest evidence became test-attributed".into(),
2262                });
2263            }
2264            results.push(RawTestResult {
2265                test_id: Some(background_id.clone()),
2266                scope: Some(ExecutionScope {
2267                    version: 1,
2268                    run_id: run_id.into(),
2269                    worker_id: worker_id.clone(),
2270                    test_id: background_id.clone(),
2271                    test_key: background_id.clone(),
2272                    retry: 0,
2273                    attempt_id: format!("{run_id}:doctest:{}:background", group.invocation_id),
2274                }),
2275                test: background_id,
2276                test_file: None,
2277                title: Some(format!("Background Rust doctest work for {}", group.group)),
2278                retry: Some(0),
2279                status: Some("passed".into()),
2280                expected_status: Some("passed".into()),
2281                flaky: false,
2282                provenance: TestProvenance {
2283                    runner: "rustdoc".into(),
2284                    kind: "doctest".into(),
2285                    project: Some(group.group.clone()),
2286                    source: "supercov-rustdoc-context-zero".into(),
2287                },
2288                role: "background".into(),
2289                phases: Vec::new(),
2290                runtime: vec![projection.background],
2291                browser: Vec::new(),
2292                server: Vec::new(),
2293            });
2294        }
2295        health.push(RustCompilerTransportHealthRecord {
2296            scope_id: format!("rustdoc:{}", group.invocation_id),
2297            scope_kind: "runner-invocation".into(),
2298            status: if group.transport.dropped == 0 && group.transport.incomplete == 0 {
2299                "passed".into()
2300            } else {
2301                "unknown".into()
2302            },
2303            transport: RustCompilerTransportHealth {
2304                committed: group.transport.committed,
2305                incomplete: group.transport.incomplete,
2306                dropped: group.transport.dropped,
2307                attachments: group.transport.attachments,
2308            },
2309            thread_scope_limitations: group.thread_scope_limitations().map_err(|error| {
2310                RustCompilerTestError::Projection {
2311                    test: format!("rustdoc:{}", group.group),
2312                    reason: error.to_string(),
2313                }
2314            })?,
2315        });
2316    }
2317    Ok((results, health))
2318}
2319
2320fn doctest_command_failed(resolution: &RustdocOutcomeResolution) -> bool {
2321    resolution.groups.iter().any(|group| {
2322        group.entries.iter().any(|joined| match &joined.state {
2323            RustdocJoinedOutcomeState::Completed { outcome } => {
2324                outcome.status == RustdocOutcomeStatus::Failed
2325            }
2326            RustdocJoinedOutcomeState::UnfinishedStarted | RustdocJoinedOutcomeState::Unstarted => {
2327                true
2328            }
2329            RustdocJoinedOutcomeState::FilteredOut => false,
2330            RustdocJoinedOutcomeState::NotRunAmbiguous => true,
2331        }) || group.ambiguous_unstarted_tests != 0
2332    })
2333}
2334
2335fn ctfe_raw_results(
2336    run_id: &str,
2337    units: Vec<RustCompilerCtfeUnit>,
2338    started_at_ms: i64,
2339    ended_at_ms: i64,
2340) -> Vec<RawTestResult> {
2341    units
2342        .into_iter()
2343        .enumerate()
2344        .map(|(index, unit)| {
2345            let worker_id = format!("rustc-{index:04}");
2346            let test_id = format!("rust:build:ctfe:{}:{index:04}", unit.crate_name);
2347            let attempt_id = format!("{run_id}:ctfe:{index:04}");
2348            let phase_id = phase_id(run_id, &attempt_id);
2349            let mut snapshot = unit.snapshot;
2350            for event in &mut snapshot.events {
2351                event.phase_id = Some(phase_id.clone());
2352            }
2353            RawTestResult {
2354                test_id: Some(test_id.clone()),
2355                scope: Some(ExecutionScope {
2356                    version: 1,
2357                    run_id: run_id.into(),
2358                    worker_id,
2359                    test_id: test_id.clone(),
2360                    test_key: test_id.clone(),
2361                    retry: 0,
2362                    attempt_id,
2363                }),
2364                test: test_id.clone(),
2365                test_file: None,
2366                title: Some(format!(
2367                    "Compile-time evaluation for {} ({})",
2368                    unit.crate_name, unit.identity
2369                )),
2370                retry: Some(0),
2371                status: Some("passed".into()),
2372                expected_status: Some("passed".into()),
2373                flaky: false,
2374                provenance: TestProvenance {
2375                    runner: "rustc".into(),
2376                    kind: "build".into(),
2377                    project: Some(unit.crate_name),
2378                    source: "supercov-rustc-ctfe".into(),
2379                },
2380                role: "setup".into(),
2381                phases: vec![CoveragePhase {
2382                    id: phase_id,
2383                    kind: "setup".into(),
2384                    operation: "Rust constant evaluation".into(),
2385                    source: None,
2386                    caused_by_phase_id: None,
2387                    started_at_ms,
2388                    ended_at_ms: Some(ended_at_ms),
2389                    status: Some("passed".into()),
2390                    error: None,
2391                }],
2392                runtime: vec![snapshot],
2393                browser: Vec::new(),
2394                server: Vec::new(),
2395            }
2396        })
2397        .collect()
2398}
2399
2400fn status(
2401    outcome: &RustCargoRunnerAttemptOutcome,
2402    output: &SupervisedOutput,
2403) -> (&'static str, i32) {
2404    match outcome {
2405        RustCargoRunnerAttemptOutcome::Libtest {
2406            result: RustLibtestTerminalResult::Passed | RustLibtestTerminalResult::Benchmarked,
2407            ..
2408        } => ("passed", 0),
2409        RustCargoRunnerAttemptOutcome::Libtest {
2410            result: RustLibtestTerminalResult::Ignored,
2411            ..
2412        } => ("skipped", 0),
2413        RustCargoRunnerAttemptOutcome::Libtest {
2414            result: RustLibtestTerminalResult::Failed,
2415            ..
2416        } => ("failed", 101),
2417        RustCargoRunnerAttemptOutcome::Unstarted => ("unstarted", 0),
2418        RustCargoRunnerAttemptOutcome::OpaqueProcess => {
2419            let exit = output.result.exit_code();
2420            (if exit == 0 { "passed" } else { "failed" }, exit)
2421        }
2422    }
2423}
2424
2425fn raw_result(
2426    run_id: &str,
2427    task: &ProcessTask,
2428    status: &str,
2429    base_phase: CoveragePhase,
2430    projection: RustCompilerEvidenceProjection,
2431) -> (RawTestResult, RustCompilerTransportHealthRecord) {
2432    let worker_id = format!("artifact-{:04}", task.artifact_index);
2433    let attempt_id = task.runner_attempt_id.clone();
2434    let runner = match task.runner {
2435        RustCargoRunnerKind::CargoTest => "rust-libtest",
2436        RustCargoRunnerKind::CargoCustomHarness => "rust-custom-harness",
2437        RustCargoRunnerKind::Nextest => "rust-nextest",
2438    };
2439    let scope = ExecutionScope {
2440        version: 1,
2441        run_id: run_id.into(),
2442        worker_id: worker_id.clone(),
2443        test_id: task.test_id.clone(),
2444        test_key: task.test_id.clone(),
2445        retry: task.retry,
2446        attempt_id: attempt_id.clone(),
2447    };
2448    let mut phases = vec![base_phase];
2449    phases.extend(projection.assertion_phases);
2450    let result = RawTestResult {
2451        test_id: Some(task.test_id.clone()),
2452        scope: Some(scope),
2453        test: task.test_id.clone(),
2454        test_file: Some(task.artifact.source.clone()),
2455        title: Some(if task.runner == RustCargoRunnerKind::CargoCustomHarness {
2456            task.artifact.target_key.clone()
2457        } else {
2458            task.test.clone()
2459        }),
2460        retry: Some(task.retry),
2461        status: Some(status.into()),
2462        expected_status: Some("passed".into()),
2463        flaky: false,
2464        provenance: TestProvenance {
2465            runner: runner.into(),
2466            kind: task.artifact.kind.clone(),
2467            project: Some(task.artifact.package.clone()),
2468            source: match task.runner {
2469                RustCargoRunnerKind::CargoTest => "supercov-rustc-stock-libtest-context",
2470                RustCargoRunnerKind::CargoCustomHarness => "supercov-rustc-custom-harness-process",
2471                RustCargoRunnerKind::Nextest => "supercov-rustc-nextest-process",
2472            }
2473            .into(),
2474        },
2475        role: "test".into(),
2476        phases,
2477        runtime: vec![projection.attributed],
2478        browser: Vec::new(),
2479        server: Vec::new(),
2480    };
2481    let health = RustCompilerTransportHealthRecord {
2482        scope_id: task.test_id.clone(),
2483        scope_kind: "test-attempt".into(),
2484        status: status.into(),
2485        transport: projection.health,
2486        thread_scope_limitations: BTreeSet::new(),
2487    };
2488    (result, health)
2489}
2490
2491fn cargo_runner_background_result(
2492    run_id: &str,
2493    artifact_index: usize,
2494    artifact: &TestArtifact,
2495    unit: &RustCargoRunnerUnit,
2496    normalized: &NormalizedRustCompilerManifest,
2497) -> Result<(Option<RawTestResult>, RustCompilerTransportHealthRecord), RustCompilerTestError> {
2498    let transport = &unit.invocation.background_transport;
2499    let background_id = format!("background:rust-runner:{:016}", unit.invocation_ordinal);
2500    let invocation_status = if unit.invocation.result.exit_code() == 0 {
2501        "passed"
2502    } else {
2503        "failed"
2504    };
2505    let base_phase = CoveragePhase {
2506        id: phase_id(run_id, &background_id),
2507        kind: "setup".into(),
2508        operation: format!("Background while running {}", artifact.target_key),
2509        source: Some(artifact.source.clone()),
2510        caused_by_phase_id: None,
2511        started_at_ms: unit.invocation.started_at_ms,
2512        ended_at_ms: Some(unit.invocation.ended_at_ms),
2513        status: Some(invocation_status.into()),
2514        error: None,
2515    };
2516    // The persisted background partition holds context-zero records plus
2517    // join-bounded quarantined thread evidence that keeps its real contexts.
2518    // Projection deliberately flattens every record to context zero under a
2519    // non-reserved synthetic base: the shared projector still validates probe
2520    // identities while escaped-thread work can never become test or phase
2521    // attributed downstream.
2522    let mut flattened = transport.clone();
2523    for observation in &mut flattened.observations {
2524        observation.context_id = 0;
2525    }
2526    for hit in &mut flattened.ordinal_hits {
2527        hit.context_id = 0;
2528    }
2529    flattened.phases.clear();
2530    flattened.thread_phases.clear();
2531    flattened.thread_ends.clear();
2532    flattened.test_boundaries.clear();
2533    let projection = project_rust_compiler_evidence(1, &base_phase, &flattened, normalized)
2534        .map_err(|error| RustCompilerTestError::Projection {
2535            test: background_id.clone(),
2536            reason: error.to_string(),
2537        })?;
2538    if snapshot_has_evidence(&projection.attributed) || !projection.assertion_phases.is_empty() {
2539        return Err(RustCompilerTestError::Projection {
2540            test: background_id,
2541            reason: "context-zero Cargo evidence became test-attributed".into(),
2542        });
2543    }
2544    let runner = match unit.runner {
2545        RustCargoRunnerKind::CargoTest => "rust-libtest",
2546        RustCargoRunnerKind::CargoCustomHarness => "rust-custom-harness",
2547        RustCargoRunnerKind::Nextest => "rust-nextest",
2548    };
2549    let result = snapshot_has_evidence(&projection.background).then(|| RawTestResult {
2550        test_id: Some(background_id.clone()),
2551        scope: Some(ExecutionScope {
2552            version: 1,
2553            run_id: run_id.into(),
2554            worker_id: format!("artifact-{artifact_index:04}"),
2555            test_id: background_id.clone(),
2556            test_key: background_id.clone(),
2557            retry: 0,
2558            attempt_id: format!("{run_id}:cargo:{:016}:background", unit.invocation_ordinal),
2559        }),
2560        test: background_id.clone(),
2561        test_file: Some(artifact.source.clone()),
2562        title: Some(format!("Background Rust work for {}", artifact.target_key)),
2563        retry: Some(0),
2564        status: Some(invocation_status.into()),
2565        expected_status: Some("passed".into()),
2566        flaky: false,
2567        provenance: TestProvenance {
2568            runner: runner.into(),
2569            kind: artifact.kind.clone(),
2570            project: Some(artifact.package.clone()),
2571            source: "supercov-rustc-context-zero".into(),
2572        },
2573        role: "background".into(),
2574        phases: Vec::new(),
2575        runtime: vec![projection.background],
2576        browser: Vec::new(),
2577        server: Vec::new(),
2578    });
2579    Ok((
2580        result,
2581        RustCompilerTransportHealthRecord {
2582            scope_id: background_id,
2583            scope_kind: "runner-invocation".into(),
2584            status: invocation_status.into(),
2585            transport: projection.health,
2586            thread_scope_limitations: unit.thread_scope_limitations.clone(),
2587        },
2588    ))
2589}
2590
2591pub fn run_rust_compiler_frontend(
2592    request: &RustCompilerRunRequest,
2593    diagnostics: &mut dyn Write,
2594) -> Result<RustCompilerFrontendRun, RustCompilerTestError> {
2595    let supervisor = request
2596        .watchdog_program
2597        .as_deref()
2598        .map_or_else(ProcessSupervisor::new, ProcessSupervisor::new_crash_safe)
2599        .map_err(|error| RustCompilerTestError::Build(error.to_string()))?;
2600    let options = SupervisionOptions::from_environment()
2601        .map_err(|error| RustCompilerTestError::Build(error.to_string()))?;
2602    let build = crate::rust_compiler_orchestration::build_with_rust_compiler_companion_supervised(
2603        &request.build_request(),
2604        &supervisor,
2605        options,
2606        diagnostics,
2607    )
2608    .map_err(|error| {
2609        match error {
2610        crate::rust_compiler_orchestration::RustCompilerOrchestrationError::Interrupted {
2611            code,
2612            signal,
2613        } => RustCompilerTestError::Interrupted { code, signal },
2614        crate::rust_compiler_orchestration::RustCompilerOrchestrationError::UnverifiedExecution {
2615            code,
2616            reason,
2617        } => RustCompilerTestError::UnverifiedExecution { code, reason },
2618        error => RustCompilerTestError::Build(error.to_string()),
2619    }
2620    })?;
2621    execute_compiler_build(request, build, diagnostics)
2622}
2623
2624fn execute_compiler_build(
2625    request: &RustCompilerRunRequest,
2626    build: RustCompilerBuild,
2627    diagnostics: &mut dyn Write,
2628) -> Result<RustCompilerFrontendRun, RustCompilerTestError> {
2629    let project_root = canonicalize_simplified(&request.project_root)
2630        .map_err(|error| io_error(&request.project_root, error))?;
2631    let artifacts = normalize_artifacts(&project_root, &build.target_directory, &build.artifacts)?;
2632    let artifact_by_path = artifacts
2633        .iter()
2634        .enumerate()
2635        .map(|(index, artifact)| (artifact.executable.clone(), (index, artifact.clone())))
2636        .collect::<BTreeMap<_, _>>();
2637    let nextest_version = match build.command_kind {
2638        crate::rust_test_runner::RustCargoCommandKind::NextestRun => {
2639            Some(build.nextest_version.as_deref().ok_or_else(|| {
2640                RustCompilerTestError::Context(
2641                    "nextest execution lacks its authenticated version handshake".into(),
2642                )
2643            })?)
2644        }
2645        crate::rust_test_runner::RustCargoCommandKind::CargoTest => {
2646            if build.nextest_version.is_some() || build.nextest_catalog.is_some() {
2647                return Err(RustCompilerTestError::Context(
2648                    "standard Cargo execution carries foreign nextest preflight state".into(),
2649                ));
2650            }
2651            None
2652        }
2653    };
2654    let mut nextest_selected =
2655        BTreeMap::<(PathBuf, String), (String, usize, TestArtifact, String)>::new();
2656    let mut nextest_selected_ids = BTreeSet::new();
2657    let mut nextest_binary_by_artifact = BTreeMap::<PathBuf, String>::new();
2658    if let Some(catalog) = &build.nextest_catalog {
2659        if build.command_kind != crate::rust_test_runner::RustCargoCommandKind::NextestRun {
2660            return Err(RustCompilerTestError::Context(
2661                "a nextest catalog was attached to a non-nextest build".into(),
2662            ));
2663        }
2664        for (binary_id, suite) in &catalog.rust_suites {
2665            if suite.status != RustTestSuiteStatusSummary::LISTED && !suite.test_cases.is_empty() {
2666                return Err(RustCompilerTestError::Context(format!(
2667                    "nextest skipped suite {binary_id} contains test cases"
2668                )));
2669            }
2670            let executable = canonicalize_simplified(suite.binary.binary_path.as_std_path())
2671                .map_err(|error| io_error(suite.binary.binary_path.as_std_path(), error))?;
2672            let (artifact_index, artifact) =
2673                artifact_by_path.get(&executable).ok_or_else(|| {
2674                    RustCompilerTestError::Context(format!(
2675                        "nextest catalog contains an unknown artifact: {}",
2676                        executable.display()
2677                    ))
2678                })?;
2679            if nextest_binary_by_artifact
2680                .insert(executable.clone(), binary_id.to_string())
2681                .is_some()
2682            {
2683                return Err(RustCompilerTestError::Context(
2684                    "nextest catalog aliases two binary identities to one artifact".into(),
2685                ));
2686            }
2687            let compilation_target = match suite.binary.build_platform {
2688                BuildPlatform::Host => catalog
2689                    .rust_build_meta
2690                    .platforms
2691                    .as_ref()
2692                    .map(|platforms| platforms.host.platform.triple.as_str()),
2693                BuildPlatform::Target => {
2694                    catalog
2695                        .rust_build_meta
2696                        .platforms
2697                        .as_ref()
2698                        .and_then(|platforms| {
2699                            if platforms.targets.len() > 1 {
2700                                None
2701                            } else {
2702                                platforms
2703                                    .targets
2704                                    .first()
2705                                    .map(|target| target.platform.triple.as_str())
2706                                    .or(Some(platforms.host.platform.triple.as_str()))
2707                            }
2708                        })
2709                }
2710            }
2711            .ok_or_else(|| {
2712                RustCompilerTestError::Context(format!(
2713                    "nextest binary {binary_id} lacks one exact compilation target"
2714                ))
2715            })?
2716            .to_owned();
2717            if !request
2718                .cargo_runner_plan
2719                .targets
2720                .iter()
2721                .any(|target| target.target == compilation_target)
2722            {
2723                return Err(RustCompilerTestError::Context(format!(
2724                    "nextest binary {binary_id} uses unselected target {compilation_target}"
2725                )));
2726            }
2727            for (test, summary) in &suite.test_cases {
2728                if summary.kind.is_none() {
2729                    return Err(RustCompilerTestError::Context(format!(
2730                        "nextest catalog test {binary_id}::{test} lacks a test kind"
2731                    )));
2732                }
2733                if summary.filter_match == FilterMatch::Matches {
2734                    let test = test.to_string();
2735                    let test_id = libtest_id(&compilation_target, artifact, &test);
2736                    if !nextest_selected_ids.insert(test_id.clone()) {
2737                        return Err(RustCompilerTestError::DuplicateTest(test_id));
2738                    }
2739                    if nextest_selected
2740                        .insert(
2741                            (executable.clone(), test.clone()),
2742                            (
2743                                compilation_target.clone(),
2744                                *artifact_index,
2745                                artifact.clone(),
2746                                test,
2747                            ),
2748                        )
2749                        .is_some()
2750                    {
2751                        return Err(RustCompilerTestError::DuplicateTest(test_id));
2752                    }
2753                }
2754            }
2755        }
2756    } else if build.command_kind == crate::rust_test_runner::RustCargoCommandKind::NextestRun {
2757        return Err(RustCompilerTestError::Context(
2758            "nextest execution lacks its exact selected-test catalog".into(),
2759        ));
2760    }
2761    let mut outcomes = Vec::new();
2762    let mut identities = BTreeSet::new();
2763    let mut attempt_ids = BTreeSet::new();
2764    let mut nextest_attempted = BTreeSet::new();
2765    for unit in &build.cargo_runner_units {
2766        let (artifact_index, artifact) = artifact_by_path.get(&unit.artifact).ok_or_else(|| {
2767            RustCompilerTestError::Context(format!(
2768                "Cargo runner executed an unknown artifact: {}",
2769                unit.artifact.display()
2770            ))
2771        })?;
2772        let tests = unit
2773            .attempts
2774            .iter()
2775            .map(|attempt| attempt.test.clone())
2776            .collect::<Vec<_>>();
2777        let contexts = preflight_rust_test_contexts(tests)
2778            .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
2779        for (test_index, attempt) in unit.attempts.iter().enumerate() {
2780            if contexts[&attempt.test] != attempt.context_id {
2781                return Err(RustCompilerTestError::Context(format!(
2782                    "Cargo runner context changed for {}",
2783                    attempt.test
2784                )));
2785            }
2786            let test_id = match unit.runner {
2787                RustCargoRunnerKind::CargoCustomHarness => {
2788                    custom_harness_id(&unit.target, artifact)
2789                }
2790                RustCargoRunnerKind::CargoTest | RustCargoRunnerKind::Nextest => {
2791                    libtest_id(&unit.target, artifact, &attempt.test)
2792                }
2793            };
2794            if unit.runner == RustCargoRunnerKind::Nextest {
2795                if unit.runner_version.as_deref() != nextest_version {
2796                    return Err(RustCompilerTestError::Context(format!(
2797                        "nextest target-runner version disagrees with the authenticated outer handshake for {test_id}"
2798                    )));
2799                }
2800                let expected_binary =
2801                    nextest_binary_by_artifact
2802                        .get(&unit.artifact)
2803                        .ok_or_else(|| {
2804                            RustCompilerTestError::Context(format!(
2805                                "nextest executed an uncatalogued artifact: {}",
2806                                unit.artifact.display()
2807                            ))
2808                        })?;
2809                if unit.runner_binary_id.as_deref() != Some(expected_binary.as_str()) {
2810                    return Err(RustCompilerTestError::Context(format!(
2811                        "nextest runner binary identity disagrees with the selected-test catalog for {}",
2812                        unit.artifact.display()
2813                    )));
2814                }
2815                if !nextest_selected.contains_key(&(unit.artifact.clone(), attempt.test.clone())) {
2816                    return Err(RustCompilerTestError::Context(format!(
2817                        "nextest executed a test excluded by its machine-readable catalog: {test_id}"
2818                    )));
2819                }
2820                nextest_attempted.insert(test_id.clone());
2821            }
2822            if !identities.insert((test_id.clone(), attempt.retry)) {
2823                return Err(RustCompilerTestError::DuplicateTest(format!(
2824                    "{test_id} retry {}",
2825                    attempt.retry
2826                )));
2827            }
2828            if !attempt_ids.insert(attempt.runner_attempt_id.clone()) {
2829                return Err(RustCompilerTestError::Context(format!(
2830                    "Cargo runner attempt ID is duplicated: {}",
2831                    attempt.runner_attempt_id
2832                )));
2833            }
2834            let task = ProcessTask {
2835                ordinal: outcomes.len(),
2836                artifact_index: *artifact_index,
2837                artifact: artifact.clone(),
2838                test: attempt.test.clone(),
2839                test_id,
2840                context_id: attempt.context_id,
2841                retry: attempt.retry,
2842                total_attempts: attempt.total_attempts,
2843                runner_attempt_id: attempt.runner_attempt_id.clone(),
2844                runner: unit.runner,
2845                transport: build.compiler_output_directory.join(format!(
2846                    "cargo-runner/libtest-{:04}-{test_index:08}.json",
2847                    unit.invocation_ordinal
2848                )),
2849                libtest_events: build.compiler_output_directory.join(format!(
2850                    "cargo-runner/libtest-{:04}-{test_index:08}.events",
2851                    unit.invocation_ordinal
2852                )),
2853                test_arguments: Vec::new(),
2854                underlying_runner: None,
2855            };
2856            outcomes.push(ProcessOutcome {
2857                task,
2858                output: SupervisedOutput {
2859                    result: unit.invocation.result.clone(),
2860                    stdout: unit.invocation.stdout.clone(),
2861                    stderr: unit.invocation.stderr.clone(),
2862                },
2863                read: attempt.transport.clone(),
2864                attempt_outcome: attempt.outcome.clone(),
2865                started_at_ms: unit.invocation.started_at_ms,
2866                ended_at_ms: unit.invocation.ended_at_ms,
2867            });
2868        }
2869    }
2870    let standard_cargo_units = build
2871        .cargo_runner_units
2872        .iter()
2873        .filter(|unit| {
2874            matches!(
2875                unit.runner,
2876                RustCargoRunnerKind::CargoTest | RustCargoRunnerKind::CargoCustomHarness
2877            )
2878        })
2879        .count();
2880    if build.execution_exit_code == 0
2881        && build.run_libtests
2882        && standard_cargo_units != 0
2883        && standard_cargo_units != artifacts.len()
2884    {
2885        return Err(RustCompilerTestError::Context(format!(
2886            "Cargo completed successfully but published {} runner unit(s) for {} artifact(s)",
2887            standard_cargo_units,
2888            artifacts.len()
2889        )));
2890    }
2891    let mut nextest_attempt_groups = BTreeMap::<String, Vec<&ProcessOutcome>>::new();
2892    for outcome in &outcomes {
2893        if outcome.task.runner == RustCargoRunnerKind::Nextest {
2894            nextest_attempt_groups
2895                .entry(outcome.task.test_id.clone())
2896                .or_default()
2897                .push(outcome);
2898        }
2899    }
2900    for attempts in nextest_attempt_groups.values_mut() {
2901        attempts.sort_by_key(|outcome| outcome.task.retry);
2902    }
2903    let nextest_terminal_failure = nextest_attempt_groups.values().any(|attempts| {
2904        attempts
2905            .last()
2906            .is_some_and(|outcome| outcome.output.result.exit_code() != 0)
2907    });
2908    let nextest_flaky = nextest_attempt_groups.values().any(|attempts| {
2909        attempts
2910            .last()
2911            .is_some_and(|outcome| outcome.output.result.exit_code() == 0)
2912            && attempts
2913                .iter()
2914                .take(attempts.len().saturating_sub(1))
2915                .any(|outcome| outcome.output.result.exit_code() != 0)
2916    });
2917    let nextest_unstarted = nextest_selected
2918        .values()
2919        .filter(|(target, _, artifact, test)| {
2920            !nextest_attempted.contains(&libtest_id(target, artifact, test))
2921        })
2922        .cloned()
2923        .collect::<Vec<_>>();
2924
2925    let mut raw_results = ctfe_raw_results(
2926        &request.run_id,
2927        build.ctfe_units.clone(),
2928        build.build_started_at_ms,
2929        build.build_ended_at_ms,
2930    );
2931    let (doctest_results, mut transport_health) = doctest_raw_results(
2932        &request.run_id,
2933        &build.doctest_outcomes,
2934        build.build_started_at_ms,
2935        build.build_ended_at_ms,
2936        &build.normalized,
2937    )?;
2938    raw_results.extend(doctest_results);
2939    for unit in &build.cargo_runner_units {
2940        let (artifact_index, artifact) = artifact_by_path.get(&unit.artifact).ok_or_else(|| {
2941            RustCompilerTestError::Context(format!(
2942                "Cargo runner published background evidence for an unknown artifact: {}",
2943                unit.artifact.display()
2944            ))
2945        })?;
2946        let (background, health) = cargo_runner_background_result(
2947            &request.run_id,
2948            *artifact_index,
2949            artifact,
2950            unit,
2951            &build.normalized,
2952        )?;
2953        if let Some(background) = background {
2954            raw_results.push(background);
2955        }
2956        transport_health.push(health);
2957    }
2958    let overall_exit = build.execution_exit_code;
2959    if overall_exit != 0 {
2960        diagnostics
2961            .write_all(&build.execution_stdout)
2962            .and_then(|_| diagnostics.write_all(&build.execution_stderr))
2963            .map_err(|error| RustCompilerTestError::Io {
2964                path: build.compiler_output_directory.clone(),
2965                reason: error.to_string(),
2966            })?;
2967    }
2968    match build.command_kind {
2969        crate::rust_test_runner::RustCargoCommandKind::CargoTest => {
2970            let authenticated_failure = doctest_command_failed(&build.doctest_outcomes)
2971                || outcomes
2972                    .iter()
2973                    .any(|outcome| outcome.output.result.exit_code() != 0);
2974            if (overall_exit != 0) != authenticated_failure {
2975                return Err(RustCompilerTestError::Context(
2976                    "Cargo exit status disagrees with authenticated libtest/doctest outcomes"
2977                        .into(),
2978                ));
2979            }
2980        }
2981        crate::rust_test_runner::RustCargoCommandKind::NextestRun => match overall_exit {
2982            NextestExitCode::OK => {
2983                if nextest_terminal_failure || !nextest_unstarted.is_empty() {
2984                    return Err(RustCompilerTestError::Context(
2985                        "nextest exited successfully without terminal successful attempts for every selected test"
2986                            .into(),
2987                    ));
2988                }
2989            }
2990            NextestExitCode::TEST_RUN_FAILED => {
2991                if !nextest_terminal_failure && !nextest_flaky {
2992                    return Err(RustCompilerTestError::Context(
2993                        "nextest reported test failure without an authenticated terminal failure or flaky attempt sequence"
2994                            .into(),
2995                    ));
2996                }
2997            }
2998            NextestExitCode::NO_TESTS_RUN if nextest_selected.is_empty() => {}
2999            code => {
3000                return Err(RustCompilerTestError::UnverifiedExecution {
3001                    code,
3002                    reason: "nextest returned an infrastructure/status code that is not authenticated by test attempts"
3003                        .into(),
3004                });
3005            }
3006        },
3007    }
3008    for outcome in outcomes {
3009        let (test_status, exit) = status(&outcome.attempt_outcome, &outcome.output);
3010        if outcome.read.dropped != 0 {
3011            return Err(RustCompilerTestError::DroppedEvidence {
3012                test: outcome.task.test_id,
3013                dropped: outcome.read.dropped,
3014            });
3015        }
3016        let attempt_id = outcome.task.runner_attempt_id.clone();
3017        let base_phase = CoveragePhase {
3018            id: phase_id(&request.run_id, &attempt_id),
3019            kind: "test".into(),
3020            operation: format!(
3021                "{} {}",
3022                match outcome.task.runner {
3023                    RustCargoRunnerKind::CargoTest => "Rust libtest",
3024                    RustCargoRunnerKind::CargoCustomHarness => "Rust custom harness",
3025                    RustCargoRunnerKind::Nextest => "Rust nextest test",
3026                },
3027                if outcome.task.runner == RustCargoRunnerKind::CargoCustomHarness {
3028                    &outcome.task.artifact.target_key
3029                } else {
3030                    &outcome.task.test
3031                }
3032            ),
3033            source: Some(outcome.task.artifact.source.clone()),
3034            caused_by_phase_id: None,
3035            started_at_ms: outcome.started_at_ms,
3036            ended_at_ms: Some(outcome.ended_at_ms),
3037            status: Some(test_status.into()),
3038            error: (exit != 0).then(|| {
3039                String::from_utf8_lossy(&outcome.output.stderr)
3040                    .trim()
3041                    .to_owned()
3042            }),
3043        };
3044        let projection = project_rust_compiler_evidence(
3045            outcome.task.context_id,
3046            &base_phase,
3047            &outcome.read,
3048            &build.normalized,
3049        )
3050        .map_err(|error| RustCompilerTestError::Projection {
3051            test: outcome.task.test_id.clone(),
3052            reason: error.to_string(),
3053        })?;
3054        if snapshot_has_evidence(&projection.background) {
3055            return Err(RustCompilerTestError::Projection {
3056                test: outcome.task.test_id,
3057                reason: "a persisted test partition retained context-zero evidence".into(),
3058            });
3059        }
3060        let (result, health) = raw_result(
3061            &request.run_id,
3062            &outcome.task,
3063            test_status,
3064            base_phase,
3065            projection,
3066        );
3067        raw_results.push(result);
3068        transport_health.push(health);
3069    }
3070    for (target, _artifact_index, artifact, test) in nextest_unstarted {
3071        let test_id = libtest_id(&target, &artifact, &test);
3072        raw_results.push(RawTestResult {
3073            test_id: Some(test_id.clone()),
3074            scope: None,
3075            test: test_id,
3076            test_file: Some(artifact.source.clone()),
3077            title: Some(test),
3078            retry: None,
3079            status: Some("unstarted".into()),
3080            expected_status: Some("passed".into()),
3081            flaky: false,
3082            provenance: TestProvenance {
3083                runner: "rust-nextest".into(),
3084                kind: artifact.kind,
3085                project: Some(artifact.package),
3086                source: "nextest-selected-but-not-started".into(),
3087            },
3088            role: "test".into(),
3089            phases: Vec::new(),
3090            runtime: Vec::new(),
3091            browser: Vec::new(),
3092            server: Vec::new(),
3093        });
3094    }
3095
3096    let mut structural_limitations = build
3097        .normalized
3098        .manifest
3099        .limitations
3100        .iter()
3101        .filter_map(|limitation| {
3102            limitation
3103                .get("id")
3104                .and_then(|id| id.as_str())
3105                .map(str::to_owned)
3106        })
3107        .collect::<Vec<_>>();
3108    if !build.doctest_outcomes.is_fully_catalogued() {
3109        structural_limitations.push("rust-doctest-outcome-catalog-incomplete".into());
3110    }
3111    if build.doctest_outcomes.has_ambiguous_outcomes() {
3112        structural_limitations.push("rust-doctest-filter-fail-fast-identity-ambiguous".into());
3113    }
3114    structural_limitations.sort();
3115    structural_limitations.dedup();
3116    let observed_runners = raw_results
3117        .iter()
3118        .map(|result| result.provenance.runner.as_str())
3119        .collect::<BTreeSet<_>>();
3120    if observed_runners.iter().any(|runner| {
3121        !matches!(
3122            *runner,
3123            "rustc" | "rust-libtest" | "rust-custom-harness" | "rust-nextest" | "rustdoc"
3124        )
3125    }) {
3126        return Err(RustCompilerTestError::Context(
3127            "Rust compiler run produced an unknown runner identity".into(),
3128        ));
3129    }
3130    // The frontend contract declares capabilities actually present in this
3131    // run. In particular, `cargo test --doc` must not advertise an unobserved
3132    // libtest runner, and an explicit non-doc target must not advertise
3133    // rustdoc. The shared analyzer deliberately rejects such declarations.
3134    let runners = [
3135        ("rustc", compiler_runner_declaration()),
3136        ("rust-libtest", runner_declaration()),
3137        ("rust-custom-harness", custom_harness_runner_declaration()),
3138        ("rust-nextest", nextest_runner_declaration()),
3139        ("rustdoc", rustdoc_runner_declaration()),
3140    ]
3141    .into_iter()
3142    .filter_map(|(name, declaration)| observed_runners.contains(name).then_some(declaration))
3143    .collect();
3144    let declaration = FrontendRunDeclaration {
3145        protocol_version: LANGUAGE_FRONTEND_PROTOCOL_VERSION,
3146        frontend_id: "rust".into(),
3147        frontend_version: "rust-compiler-v1".into(),
3148        language: "rust".into(),
3149        structural_source: StructuralSource::OwnedProbes,
3150        runners,
3151        structural_limitations,
3152    };
3153    Ok(RustCompilerFrontendRun {
3154        selection: build.selection,
3155        declaration,
3156        request: CoverageReportRequest {
3157            run_id: request.run_id.clone(),
3158            manifest: build.normalized.manifest,
3159            raw_results,
3160            generated_at: request.generated_at.clone(),
3161            coverage_model: Some(rust_compiler_coverage_model()),
3162            integrity: None,
3163            test_exit_code: ExitCodeInput::Present(Some(overall_exit)),
3164        },
3165        exit_code: overall_exit,
3166        artifacts: artifacts.len(),
3167        artifact_files: artifacts
3168            .into_iter()
3169            .map(|artifact| artifact.executable)
3170            .collect(),
3171        transport_health,
3172        build_ms: build.build_ms,
3173        execution_ms: build.execution_ms,
3174    })
3175}
3176
3177#[cfg(test)]
3178mod tests {
3179    use super::*;
3180    use crate::{
3181        coverage_analysis::PointKind,
3182        coverage_report::{CoverageManifest, PointMeta},
3183        rust_doctest::{
3184            RustdocDoctestAttributes, RustdocDoctestCode, RustdocDoctestIgnore,
3185            RustdocDoctestWrapper, RustdocExtractedDoctest, RustdocJoinedOutcome,
3186            RustdocMergedEntry, RustdocOutcomeGroupJoin, RustdocTestOutcome,
3187        },
3188        rust_probe_transport::RustOrdinalHit,
3189    };
3190
3191    fn test_transport() -> RustTransportRead {
3192        RustTransportRead::empty()
3193    }
3194
3195    fn test_invocation(status: i32) -> RustCargoRunnerInvocation {
3196        RustCargoRunnerInvocation {
3197            result: SupervisedResult {
3198                status: Some(status),
3199                signal: None,
3200                timed_out: false,
3201                interrupted_signal: None,
3202            },
3203            started_at_ms: 0,
3204            ended_at_ms: 1,
3205            stdout: Vec::new(),
3206            stderr: Vec::new(),
3207            background_transport: test_transport(),
3208        }
3209    }
3210
3211    fn test_runner_unit() -> RustCargoRunnerUnit {
3212        RustCargoRunnerUnit {
3213            version: RUST_CARGO_RUNNER_VERSION,
3214            run_id: "run_0123456789abcdef".into(),
3215            invocation_ordinal: 0,
3216            runner: RustCargoRunnerKind::CargoTest,
3217            runner_run_id: None,
3218            runner_version: None,
3219            runner_binary_id: None,
3220            target: "aarch64-apple-darwin".into(),
3221            artifact: PathBuf::from("target/test-artifact"),
3222            arguments: Vec::new(),
3223            invocation: test_invocation(0),
3224            attempts: Vec::new(),
3225            thread_scope_limitations: BTreeSet::new(),
3226        }
3227    }
3228
3229    #[test]
3230    fn persisted_runner_transport_rejects_cross_partition_and_count_tampering() {
3231        let mut unit = test_runner_unit();
3232        unit.attempts.push(RustCargoRunnerAttempt {
3233            test: "tests::one".into(),
3234            context_id: 7,
3235            retry: 0,
3236            total_attempts: 1,
3237            runner_attempt_id: "attempt-one".into(),
3238            outcome: RustCargoRunnerAttemptOutcome::Libtest {
3239                result: RustLibtestTerminalResult::Passed,
3240                timed_out: false,
3241            },
3242            transport: test_transport(),
3243        });
3244        validate_persisted_runner_transport(&unit).unwrap();
3245
3246        unit.attempts[0]
3247            .transport
3248            .ordinal_hits
3249            .push(RustOrdinalHit {
3250                process_id: 1,
3251                context_id: 0,
3252                ordinal: 9,
3253            });
3254        unit.attempts[0].transport.committed = 1;
3255        assert!(
3256            validate_persisted_runner_transport(&unit)
3257                .unwrap_err()
3258                .to_string()
3259                .contains("exact transport partition")
3260        );
3261
3262        unit.attempts[0].transport.ordinal_hits[0].context_id = 7;
3263        unit.attempts[0].transport.committed = 2;
3264        assert!(
3265            validate_persisted_runner_transport(&unit)
3266                .unwrap_err()
3267                .to_string()
3268                .contains("persisted attribution")
3269        );
3270    }
3271
3272    #[test]
3273    fn empty_selected_suite_publishes_invocation_background_without_a_test_attempt() {
3274        let point_id = "rs:statement:111111111111111111111111";
3275        let mut unit = test_runner_unit();
3276        unit.invocation.background_transport = RustTransportRead {
3277            ordinal_hits: vec![RustOrdinalHit {
3278                process_id: 1,
3279                context_id: 0,
3280                ordinal: 9,
3281            }],
3282            committed: 1,
3283            attachments: 1,
3284            ..RustTransportRead::empty()
3285        };
3286        validate_persisted_runner_transport(&unit).unwrap();
3287        let normalized = NormalizedRustCompilerManifest {
3288            manifest: CoverageManifest {
3289                unmeasured: Vec::new(),
3290                decisions: Vec::new(),
3291                points: vec![PointMeta {
3292                    id: point_id.into(),
3293                    kind: PointKind::Statement,
3294                    file: "src/lib.rs".into(),
3295                    line: 1,
3296                    column: 1,
3297                    source: "setup();".into(),
3298                    label: None,
3299                }],
3300                branches: Vec::new(),
3301                limitations: Vec::new(),
3302                scope: None,
3303            },
3304            hit_obligations_by_ordinal: BTreeMap::from([(9, vec![point_id.into()])]),
3305            internal_ordinals: BTreeSet::new(),
3306            decision_outcome_obligations: BTreeMap::new(),
3307            decision_loop_obligations: BTreeMap::new(),
3308            decision_logical_selection_obligations: BTreeMap::new(),
3309        };
3310        let artifact = TestArtifact {
3311            executable: unit.artifact.clone(),
3312            runner_argument: None,
3313            package: "package:.".into(),
3314            target_key: "lib:fixture".into(),
3315            kind: "unit".into(),
3316            source: "src/lib.rs".into(),
3317            test_harness: true,
3318        };
3319        let (result, health) =
3320            cargo_runner_background_result(&unit.run_id, 0, &artifact, &unit, &normalized).unwrap();
3321        let result = result.expect("context-zero evidence must remain queryable");
3322        assert_eq!(result.role, "background");
3323        assert!(result.runtime[0].hits.iter().any(|hit| hit == point_id));
3324        assert_eq!(health.scope_kind, "runner-invocation");
3325        assert_eq!(health.transport.committed, 1);
3326    }
3327
3328    #[test]
3329    fn tokens_and_phase_ids_are_fixed_width_and_domain_separated() {
3330        assert_eq!(token_hex(&[0xab; TOKEN_BYTES]), "ab".repeat(TOKEN_BYTES));
3331        let first = phase_id("run-a", "attempt");
3332        assert_eq!(first.len(), "rust-test:".len() + 40);
3333        assert_ne!(first, phase_id("run-b", "attempt"));
3334        assert_ne!(first, phase_id("run-a", "attempt-b"));
3335    }
3336
3337    #[test]
3338    fn test_identities_include_runner_package_target_and_workspace_source() {
3339        let artifact = |package: &str, target_key: &str| TestArtifact {
3340            executable: PathBuf::from("test-artifact"),
3341            runner_argument: None,
3342            package: package.into(),
3343            target_key: target_key.into(),
3344            kind: "unit".into(),
3345            source: "shared/src/lib.rs".into(),
3346            test_harness: true,
3347        };
3348        let root = artifact("package:.", "lib:same");
3349        let sibling = artifact("package:crates/sibling", "lib:same");
3350        let integration = artifact("package:.", "test:same");
3351        assert_eq!(
3352            libtest_id("aarch64-apple-darwin", &root, "tests::same_name"),
3353            "rust:libtest:aarch64-apple-darwin:package:.:lib:same:shared/src/lib.rs::tests::same_name"
3354        );
3355        assert_ne!(
3356            libtest_id("aarch64-apple-darwin", &root, "tests::same_name"),
3357            libtest_id("aarch64-apple-darwin", &sibling, "tests::same_name")
3358        );
3359        assert_ne!(
3360            libtest_id("aarch64-apple-darwin", &root, "tests::same_name"),
3361            libtest_id("aarch64-apple-darwin", &integration, "tests::same_name")
3362        );
3363        assert_ne!(
3364            libtest_id("aarch64-apple-darwin", &root, "tests::same_name"),
3365            libtest_id("x86_64-apple-darwin", &root, "tests::same_name")
3366        );
3367        assert_eq!(
3368            custom_harness_id("aarch64-apple-darwin", &integration),
3369            "rust:custom-harness:aarch64-apple-darwin:package:.:test:same:shared/src/lib.rs"
3370        );
3371        assert_ne!(
3372            custom_harness_id("aarch64-apple-darwin", &integration),
3373            libtest_id("aarch64-apple-darwin", &integration, "custom-harness")
3374        );
3375    }
3376
3377    #[test]
3378    fn cargo_runner_failure_units_are_atomic_and_diagnostic() {
3379        let root = std::env::temp_dir().join(format!(
3380            "supercov-cargo-runner-failure-{}-{}",
3381            std::process::id(),
3382            SystemTime::now()
3383                .duration_since(UNIX_EPOCH)
3384                .unwrap()
3385                .as_nanos()
3386        ));
3387        fs::create_dir(&root).unwrap();
3388        let ordinal = reserve_cargo_runner_ordinal(&root).unwrap();
3389        assert_eq!(ordinal, 0);
3390        let failure = RustCargoRunnerFailure {
3391            version: RUST_CARGO_RUNNER_VERSION,
3392            run_id: "run_0123456789abcdef".into(),
3393            invocation_ordinal: ordinal,
3394            target: Some("aarch64-apple-darwin".into()),
3395            artifact: Some(PathBuf::from("target/test-artifact")),
3396            error: "deliberate runner failure".into(),
3397        };
3398        let published = write_cargo_runner_failure(&root, &failure).unwrap();
3399        assert_eq!(
3400            published.file_name().unwrap(),
3401            "failure-0000000000000000.json"
3402        );
3403        assert!(fs::read_dir(&root).unwrap().all(|entry| {
3404            !entry
3405                .unwrap()
3406                .file_name()
3407                .to_string_lossy()
3408                .ends_with(".partial")
3409        }));
3410        let error = read_cargo_runner_units(
3411            &root,
3412            "run_0123456789abcdef",
3413            &["aarch64-apple-darwin".into()],
3414        )
3415        .unwrap_err();
3416        let error = error.to_string();
3417        assert!(error.contains("invocation 0 failed"), "{error}");
3418        assert!(error.contains("deliberate runner failure"), "{error}");
3419        fs::remove_dir_all(root).unwrap();
3420    }
3421
3422    #[test]
3423    fn cargo_runner_process_death_is_distinct_from_an_internal_failure() {
3424        let root = std::env::temp_dir().join(format!(
3425            "supercov-cargo-runner-death-{}-{}",
3426            std::process::id(),
3427            SystemTime::now()
3428                .duration_since(UNIX_EPOCH)
3429                .unwrap()
3430                .as_nanos()
3431        ));
3432        fs::create_dir(&root).unwrap();
3433        reserve_cargo_runner_ordinal(&root).unwrap();
3434        let error = read_cargo_runner_units(
3435            &root,
3436            "run_0123456789abcdef",
3437            &["aarch64-apple-darwin".into()],
3438        )
3439        .unwrap_err()
3440        .to_string();
3441        assert!(error.contains("without publishing its unit"), "{error}");
3442        fs::remove_dir_all(root).unwrap();
3443    }
3444
3445    #[test]
3446    fn cargo_runner_units_are_bound_to_the_selected_target_set() {
3447        let root = std::env::temp_dir().join(format!(
3448            "supercov-cargo-runner-target-{}-{}",
3449            std::process::id(),
3450            SystemTime::now()
3451                .duration_since(UNIX_EPOCH)
3452                .unwrap()
3453                .as_nanos()
3454        ));
3455        fs::create_dir(&root).unwrap();
3456        let ordinal = reserve_cargo_runner_ordinal(&root).unwrap();
3457        write_cargo_runner_unit(
3458            &root,
3459            &RustCargoRunnerUnit {
3460                version: RUST_CARGO_RUNNER_VERSION,
3461                run_id: "run_0123456789abcdef".into(),
3462                invocation_ordinal: ordinal,
3463                runner: RustCargoRunnerKind::CargoTest,
3464                runner_run_id: None,
3465                runner_version: None,
3466                runner_binary_id: None,
3467                target: "aarch64-apple-darwin".into(),
3468                artifact: PathBuf::from("target/test-artifact"),
3469                arguments: Vec::new(),
3470                invocation: test_invocation(0),
3471                attempts: Vec::new(),
3472                thread_scope_limitations: BTreeSet::new(),
3473            },
3474        )
3475        .unwrap();
3476        let second_ordinal = reserve_cargo_runner_ordinal(&root).unwrap();
3477        write_cargo_runner_unit(
3478            &root,
3479            &RustCargoRunnerUnit {
3480                version: RUST_CARGO_RUNNER_VERSION,
3481                run_id: "run_0123456789abcdef".into(),
3482                invocation_ordinal: second_ordinal,
3483                runner: RustCargoRunnerKind::CargoTest,
3484                runner_run_id: None,
3485                runner_version: None,
3486                runner_binary_id: None,
3487                target: "x86_64-unknown-linux-gnu".into(),
3488                artifact: PathBuf::from("target/test-artifact"),
3489                arguments: Vec::new(),
3490                invocation: test_invocation(0),
3491                attempts: Vec::new(),
3492                thread_scope_limitations: BTreeSet::new(),
3493            },
3494        )
3495        .unwrap();
3496        assert_eq!(
3497            read_cargo_runner_units(
3498                &root,
3499                "run_0123456789abcdef",
3500                &[
3501                    "aarch64-apple-darwin".into(),
3502                    "x86_64-unknown-linux-gnu".into(),
3503                ],
3504            )
3505            .unwrap()
3506            .len(),
3507            2
3508        );
3509        let error = read_cargo_runner_units(
3510            &root,
3511            "run_0123456789abcdef",
3512            &["x86_64-unknown-linux-gnu".into()],
3513        )
3514        .unwrap_err()
3515        .to_string();
3516        assert!(error.contains("unselected target identity"), "{error}");
3517        fs::remove_dir_all(root).unwrap();
3518    }
3519
3520    #[test]
3521    fn nextest_retry_units_preserve_each_exact_attempt_and_reject_gaps() {
3522        let root = std::env::temp_dir().join(format!(
3523            "supercov-nextest-retries-{}-{}",
3524            std::process::id(),
3525            SystemTime::now()
3526                .duration_since(UNIX_EPOCH)
3527                .unwrap()
3528                .as_nanos()
3529        ));
3530        fs::create_dir(&root).unwrap();
3531        let unit = |ordinal: u64, retry: usize, status: i32| RustCargoRunnerUnit {
3532            version: RUST_CARGO_RUNNER_VERSION,
3533            run_id: "run_0123456789abcdef".into(),
3534            invocation_ordinal: ordinal,
3535            runner: RustCargoRunnerKind::Nextest,
3536            runner_run_id: Some("2ae19189-240a-433a-a31d-acc411fe8e1f".into()),
3537            runner_version: Some("0.9.140".into()),
3538            runner_binary_id: Some("fixture".into()),
3539            target: "aarch64-apple-darwin".into(),
3540            artifact: PathBuf::from("target/test-artifact"),
3541            arguments: vec!["--exact".into(), "tests::flaky".into()],
3542            invocation: RustCargoRunnerInvocation {
3543                started_at_ms: retry as i64,
3544                ended_at_ms: retry as i64 + 1,
3545                ..test_invocation(status)
3546            },
3547            attempts: vec![RustCargoRunnerAttempt {
3548                test: "tests::flaky".into(),
3549                context_id: 7,
3550                retry,
3551                total_attempts: 2,
3552                runner_attempt_id: format!(
3553                    "2ae19189-240a-433a-a31d-acc411fe8e1f:fixture$tests::flaky{}",
3554                    if retry == 0 {
3555                        String::new()
3556                    } else {
3557                        format!("#{}", retry + 1)
3558                    }
3559                ),
3560                outcome: RustCargoRunnerAttemptOutcome::Libtest {
3561                    result: if status == 0 {
3562                        RustLibtestTerminalResult::Passed
3563                    } else {
3564                        RustLibtestTerminalResult::Failed
3565                    },
3566                    timed_out: false,
3567                },
3568                transport: test_transport(),
3569            }],
3570            thread_scope_limitations: BTreeSet::new(),
3571        };
3572
3573        let first = reserve_cargo_runner_ordinal(&root).unwrap();
3574        write_cargo_runner_unit(&root, &unit(first, 0, 101)).unwrap();
3575        let second = reserve_cargo_runner_ordinal(&root).unwrap();
3576        write_cargo_runner_unit(&root, &unit(second, 1, 0)).unwrap();
3577        let read = read_cargo_runner_units(
3578            &root,
3579            "run_0123456789abcdef",
3580            &["aarch64-apple-darwin".into()],
3581        )
3582        .unwrap();
3583        assert_eq!(read.len(), 2);
3584        assert_eq!(read[0].attempts[0].retry, 0);
3585        assert_eq!(read[1].attempts[0].retry, 1);
3586        fs::remove_dir_all(&root).unwrap();
3587
3588        let gap_root = root.with_extension("gap");
3589        fs::create_dir(&gap_root).unwrap();
3590        let first = reserve_cargo_runner_ordinal(&gap_root).unwrap();
3591        write_cargo_runner_unit(&gap_root, &unit(first, 1, 0)).unwrap();
3592        let error = read_cargo_runner_units(
3593            &gap_root,
3594            "run_0123456789abcdef",
3595            &["aarch64-apple-darwin".into()],
3596        )
3597        .unwrap_err()
3598        .to_string();
3599        assert!(error.contains("retry sequence"), "{error}");
3600        fs::remove_dir_all(gap_root).unwrap();
3601    }
3602
3603    #[cfg(unix)]
3604    #[test]
3605    fn nextest_list_passthrough_preserves_output_and_publishes_no_unit() {
3606        let artifact = TestArtifact {
3607            executable: PathBuf::from("/bin/echo"),
3608            runner_argument: None,
3609            package: "fixture".into(),
3610            target_key: "lib:fixture".into(),
3611            kind: "unit".into(),
3612            source: "src/lib.rs".into(),
3613            test_harness: true,
3614        };
3615        let mut stdout = Vec::new();
3616        let mut stderr = Vec::new();
3617        let execution = run_nextest_list_passthrough(
3618            Path::new("/tmp"),
3619            &artifact,
3620            None,
3621            vec![OsString::from("--list"), OsString::from("--format=terse")],
3622            None,
3623            &mut stdout,
3624            &mut stderr,
3625        )
3626        .unwrap();
3627        assert_eq!(execution.exit_code, 0);
3628        assert!(execution.unit_path.is_none());
3629        assert_eq!(stdout, b"--list --format=terse\n");
3630        assert!(stderr.is_empty());
3631    }
3632
3633    #[test]
3634    fn compiler_runner_declares_exact_assertion_but_not_action_causality() {
3635        let runner = runner_declaration();
3636        assert_eq!(runner.attribution.assertion, AttributionPrecision::Exact);
3637        assert_eq!(runner.attribution.action, AttributionPrecision::Unavailable);
3638        assert_eq!(runner.limitations.len(), 1);
3639        let compiler = compiler_runner_declaration();
3640        assert_eq!(compiler.attribution.phase, AttributionPrecision::Exact);
3641        assert_eq!(
3642            compiler.attribution.assertion,
3643            AttributionPrecision::Unavailable
3644        );
3645        let rustdoc = rustdoc_runner_declaration();
3646        assert_eq!(
3647            rustdoc.execution_model,
3648            ExecutionModel::ParallelContextPropagated
3649        );
3650        assert_eq!(rustdoc.attribution.test, AttributionPrecision::Exact);
3651        assert_eq!(rustdoc.attribution.assertion, AttributionPrecision::Exact);
3652    }
3653
3654    #[test]
3655    fn doctest_outcomes_project_exact_status_identity_and_fail_fast_state() {
3656        let entry = |module: &str, line: u64| RustdocMergedEntry {
3657            module: module.into(),
3658            display_name: format!("src/lib.rs - (line {line})"),
3659            path: "src/lib.rs".into(),
3660            line,
3661            ignored: false,
3662            no_run: false,
3663            should_panic: false,
3664        };
3665        let catalog = |line: u64| RustdocExtractedDoctest {
3666            file: "src/lib.rs".into(),
3667            line,
3668            doctest_attributes: RustdocDoctestAttributes {
3669                original: String::new(),
3670                should_panic: false,
3671                no_run: false,
3672                ignore: RustdocDoctestIgnore::None,
3673                rust: true,
3674                test_harness: false,
3675                compile_fail: false,
3676                standalone_crate: false,
3677                error_codes: Vec::new(),
3678                edition: None,
3679                added_css_classes: Vec::new(),
3680                unknown: Vec::new(),
3681            },
3682            original_code: "assert!(true);".into(),
3683            doctest_code: Some(RustdocDoctestCode {
3684                crate_level: String::new(),
3685                code: "assert!(true);".into(),
3686                wrapper: Some(RustdocDoctestWrapper {
3687                    before: "fn main() {".into(),
3688                    after: "}".into(),
3689                    returns_result: false,
3690                }),
3691            }),
3692            name: format!("src/lib.rs - (line {line})"),
3693        };
3694        let completed = |catalog_index, entry: RustdocMergedEntry, status| RustdocJoinedOutcome {
3695            catalog_index,
3696            catalog: catalog(entry.line),
3697            merged_entry: Some(entry.clone()),
3698            state: RustdocJoinedOutcomeState::Completed {
3699                outcome: RustdocTestOutcome {
3700                    display_name: entry.display_name,
3701                    status,
3702                    execution_seconds: Some(0.1),
3703                    stdout: None,
3704                    message: None,
3705                    reason: None,
3706                    timeout_warning: false,
3707                },
3708            },
3709        };
3710        let resolution = RustdocOutcomeResolution {
3711            groups: vec![RustdocOutcomeGroupJoin {
3712                invocation_id: "1".repeat(64),
3713                group: "fixture".into(),
3714                companion_build_id: "2".repeat(64),
3715                raw_catalog_sha256: "4".repeat(64),
3716                raw_events_sha256: "3".repeat(64),
3717                transport_sha256: "5".repeat(64),
3718                join: None,
3719                transport: RustTransportRead::empty(),
3720                entries: vec![
3721                    completed(0, entry("__doctest_0", 3), RustdocOutcomeStatus::Passed),
3722                    RustdocJoinedOutcome {
3723                        catalog_index: 1,
3724                        catalog: catalog(10),
3725                        merged_entry: Some(entry("__doctest_1", 10)),
3726                        state: RustdocJoinedOutcomeState::Unstarted,
3727                    },
3728                ],
3729                ambiguous_filtered_out: 0,
3730                ambiguous_unstarted_tests: 0,
3731            }],
3732            unmatched_maps: Vec::new(),
3733        };
3734        let normalized = NormalizedRustCompilerManifest {
3735            manifest: crate::coverage_report::CoverageManifest {
3736                unmeasured: Vec::new(),
3737                decisions: Vec::new(),
3738                points: Vec::new(),
3739                branches: Vec::new(),
3740                limitations: Vec::new(),
3741                scope: None,
3742            },
3743            hit_obligations_by_ordinal: std::collections::BTreeMap::new(),
3744            internal_ordinals: BTreeSet::new(),
3745            decision_outcome_obligations: std::collections::BTreeMap::new(),
3746            decision_loop_obligations: std::collections::BTreeMap::new(),
3747            decision_logical_selection_obligations: std::collections::BTreeMap::new(),
3748        };
3749        let (results, health) =
3750            doctest_raw_results("run", &resolution, 10, 20, &normalized).unwrap();
3751        assert_eq!(results.len(), 2);
3752        assert_eq!(health.len(), 1);
3753        assert_eq!(results[0].status.as_deref(), Some("passed"));
3754        assert_eq!(results[1].status.as_deref(), Some("unknown"));
3755        assert_eq!(results[0].retry, Some(0));
3756        assert_eq!(results[0].test_file.as_deref(), Some("src/lib.rs"));
3757        assert_eq!(
3758            results[0].test_id.as_deref(),
3759            Some("rust:doctest:fixture:src/lib.rs:3")
3760        );
3761        assert_eq!(results[0].scope.as_ref().unwrap().test_id, results[0].test);
3762        assert!(doctest_command_failed(&resolution));
3763        assert!(results[1].phases.is_empty());
3764        assert!(resolution.is_fully_catalogued());
3765    }
3766}