Skip to main content

supercov_engine/
rust_compiler_test_runner.rs

1//! Private execution and attribution for compiler-instrumented Rust artifacts.
2//!
3//! The compiler frontend freezes the complete denominator before this module
4//! launches anything. Ordinary Cargo artifacts run once under the selected
5//! toolchain's exact libtest companion and share one authenticated mmap whose
6//! dynamic contexts are partitioned by test. Nextest attempts and opaque
7//! custom harnesses retain their intrinsic process boundary. Context-zero
8//! records are always published as invocation background, never as test work.
9
10use std::{
11    collections::{BTreeMap, BTreeSet},
12    ffi::OsString,
13    fs::{self, OpenOptions},
14    io::{self, Write},
15    path::{Component, Path, PathBuf},
16    sync::{
17        Mutex,
18        atomic::{AtomicUsize, Ordering},
19    },
20    time::{SystemTime, UNIX_EPOCH},
21};
22
23use nextest_metadata::{BuildPlatform, FilterMatch, NextestExitCode, RustTestSuiteStatusSummary};
24use serde::{Deserialize, Serialize};
25use sha2::{Digest, Sha256};
26use supercov_contracts::{
27    AttributionPrecision, ExecutionModel, FrontendAttribution, FrontendLimitation,
28    FrontendLimitationScope, FrontendRunDeclaration, FrontendRunnerDeclaration,
29    LANGUAGE_FRONTEND_PROTOCOL_VERSION, StructuralSource,
30};
31
32use crate::workspace::canonicalize_simplified;
33use crate::{
34    coverage_report::{
35        CoverageModelDeclaration, CoveragePhase, CoverageReportRequest, ExecutionScope,
36        ExitCodeInput, PersistedCoverageModel, RawTestResult, RuntimeSnapshot, TestProvenance,
37    },
38    evidence_archive::EvidenceArchiveEntry,
39    process_supervision::{
40        CommandSpec, ProcessSupervisor, SupervisedOutput, SupervisedResult, SupervisionOptions,
41    },
42    rust_cargo_configuration::{
43        RustCargoResolvedRunner, RustCargoResolvedTargetRunner, RustCargoRunnerPlan,
44    },
45    rust_compiler_ctfe::RustCompilerCtfeUnit,
46    rust_compiler_evidence::{
47        RustCompilerEvidenceProjection, RustCompilerTransportHealth, project_rust_compiler_evidence,
48    },
49    rust_compiler_manifest::NormalizedRustCompilerManifest,
50    rust_compiler_orchestration::{
51        RustCompilerBuild, RustCompilerBuildRequest, RustCompilerTestArtifact,
52    },
53    rust_doctest::{RustdocJoinedOutcomeState, RustdocOutcomeResolution, RustdocOutcomeStatus},
54    rust_libtest_events::{
55        RUST_LIBTEST_EVENTS_ENV, RUST_LIBTEST_TOKEN_ENV, RustLibtestEvent, RustLibtestRunEvents,
56        RustLibtestTerminalResult, create_rust_libtest_event_file, read_rust_libtest_events,
57        validate_rust_libtest_run_events,
58    },
59    rust_probe_transport::{
60        DEFAULT_DESCRIPTOR_CAPACITY, DEFAULT_PAYLOAD_CAPACITY, RUST_CONTEXT_ENV,
61        RUST_TRANSPORT_ENV, RUST_TRANSPORT_TOKEN_ENV, RustTransportPartition, RustTransportRead,
62        create_rust_transport, partition_rust_transport_by_test_contexts, read_rust_transport,
63    },
64    rust_runner_attempt::{
65        NextestAttemptIdentity, RustRunnerInvocationIdentity, classify_rust_runner_environment,
66    },
67    rust_test_context::preflight_rust_test_contexts,
68    rust_test_runner::rust_libtest_selection,
69};
70
71const TOKEN_BYTES: usize = supercov_contracts::RUST_PROBE_TRANSPORT_TOKEN_SIZE;
72pub const RUST_CARGO_RUNNER_CONFIG_ENV: &str = "SUPERCOV_RUST_CARGO_RUNNER_CONFIG";
73pub const RUST_CARGO_RUNNER_VERSION: u32 = 6;
74
75#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
76#[serde(rename_all = "kebab-case")]
77pub enum RustCargoRunnerKind {
78    CargoTest,
79    CargoCustomHarness,
80    Nextest,
81}
82
83#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
84#[serde(rename_all = "camelCase", deny_unknown_fields)]
85pub struct RustCargoRunnerArtifact {
86    pub executable: PathBuf,
87    pub test_harness: bool,
88}
89
90#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
91#[serde(rename_all = "camelCase", deny_unknown_fields)]
92pub struct RustCargoRunnerConfig {
93    pub version: u32,
94    pub run_id: String,
95    pub target_directory: PathBuf,
96    pub output_directory: PathBuf,
97    pub target_runners: Vec<RustCargoResolvedTargetRunner>,
98    pub artifacts: Vec<RustCargoRunnerArtifact>,
99}
100
101#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
102#[serde(rename_all = "camelCase", deny_unknown_fields)]
103pub struct RustCargoRunnerAttempt {
104    pub test: String,
105    pub context_id: u64,
106    pub retry: usize,
107    pub total_attempts: usize,
108    pub runner_attempt_id: String,
109    pub outcome: RustCargoRunnerAttemptOutcome,
110    pub transport: RustTransportRead,
111}
112
113#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
114#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)]
115pub enum RustCargoRunnerAttemptOutcome {
116    Libtest {
117        result: RustLibtestTerminalResult,
118        timed_out: bool,
119    },
120    Unstarted,
121    OpaqueProcess,
122}
123
124fn attempt_outcome_succeeded(outcome: &RustCargoRunnerAttemptOutcome) -> bool {
125    matches!(
126        outcome,
127        RustCargoRunnerAttemptOutcome::Libtest {
128            result: RustLibtestTerminalResult::Passed
129                | RustLibtestTerminalResult::Ignored
130                | RustLibtestTerminalResult::Benchmarked,
131            ..
132        }
133    )
134}
135
136#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
137#[serde(rename_all = "camelCase", deny_unknown_fields)]
138pub struct RustCargoRunnerInvocation {
139    pub result: SupervisedResult,
140    pub started_at_ms: i64,
141    pub ended_at_ms: i64,
142    pub stdout: Vec<u8>,
143    pub stderr: Vec<u8>,
144    pub background_transport: RustTransportRead,
145}
146
147#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
148#[serde(rename_all = "camelCase", deny_unknown_fields)]
149pub struct RustCargoRunnerUnit {
150    pub version: u32,
151    pub run_id: String,
152    pub invocation_ordinal: u64,
153    pub runner: RustCargoRunnerKind,
154    #[serde(skip_serializing_if = "Option::is_none")]
155    pub runner_run_id: Option<String>,
156    #[serde(skip_serializing_if = "Option::is_none")]
157    pub runner_version: Option<String>,
158    #[serde(skip_serializing_if = "Option::is_none")]
159    pub runner_binary_id: Option<String>,
160    pub target: String,
161    pub artifact: PathBuf,
162    pub arguments: Vec<String>,
163    pub invocation: RustCargoRunnerInvocation,
164    pub attempts: Vec<RustCargoRunnerAttempt>,
165    /// Deterministic join-bounded quarantine notes: one per thread phase whose
166    /// lifetime escaped its creating test in this invocation's transport.
167    pub thread_scope_limitations: BTreeSet<String>,
168}
169
170fn validate_persisted_runner_transport(
171    unit: &RustCargoRunnerUnit,
172) -> Result<(), RustCompilerTestError> {
173    let mut roots = BTreeSet::new();
174    let mut combined = unit.invocation.background_transport.clone();
175    for attempt in &unit.attempts {
176        if matches!(attempt.context_id, 0 | u64::MAX) || !roots.insert(attempt.context_id) {
177            return Err(RustCompilerTestError::Context(format!(
178                "Cargo runner unit {} has a reserved or duplicate test context",
179                unit.invocation_ordinal
180            )));
181        }
182        combined
183            .observations
184            .extend(attempt.transport.observations.iter().cloned());
185        combined
186            .ordinal_hits
187            .extend(attempt.transport.ordinal_hits.iter().copied());
188        combined
189            .phases
190            .extend(attempt.transport.phases.iter().cloned());
191        combined
192            .thread_phases
193            .extend(attempt.transport.thread_phases.iter().copied());
194        combined
195            .thread_ends
196            .extend(attempt.transport.thread_ends.iter().copied());
197        combined
198            .test_boundaries
199            .extend(attempt.transport.test_boundaries.iter().copied());
200        combined.committed = combined
201            .committed
202            .checked_add(attempt.transport.committed)
203            .ok_or_else(|| {
204                RustCompilerTestError::Context(
205                    "Cargo runner committed transport count overflowed u64".into(),
206                )
207            })?;
208        combined.incomplete = combined
209            .incomplete
210            .checked_add(attempt.transport.incomplete)
211            .ok_or_else(|| {
212                RustCompilerTestError::Context(
213                    "Cargo runner incomplete transport count overflowed u64".into(),
214                )
215            })?;
216        combined.dropped = combined
217            .dropped
218            .checked_add(attempt.transport.dropped)
219            .ok_or_else(|| {
220                RustCompilerTestError::Context(
221                    "Cargo runner dropped transport count overflowed u64".into(),
222                )
223            })?;
224        combined.attachments = combined
225            .attachments
226            .checked_add(attempt.transport.attachments)
227            .ok_or_else(|| {
228                RustCompilerTestError::Context(
229                    "Cargo runner transport attachment count overflowed u64".into(),
230                )
231            })?;
232    }
233    let repartitioned =
234        partition_rust_transport_by_test_contexts(&combined, &roots).map_err(|error| {
235            RustCompilerTestError::Context(format!(
236                "Cargo runner unit {} has invalid persisted attribution: {error}",
237                unit.invocation_ordinal
238            ))
239        })?;
240    if repartitioned.background != unit.invocation.background_transport
241        || repartitioned.thread_scope_limitations != unit.thread_scope_limitations
242        || unit.attempts.iter().any(|attempt| {
243            repartitioned.attributed.get(&attempt.context_id) != Some(&attempt.transport)
244        })
245    {
246        return Err(RustCompilerTestError::Context(format!(
247            "Cargo runner unit {} does not preserve its exact transport partition",
248            unit.invocation_ordinal
249        )));
250    }
251    Ok(())
252}
253
254#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
255#[serde(rename_all = "camelCase", deny_unknown_fields)]
256struct RustCargoRunnerFailure {
257    version: u32,
258    run_id: String,
259    invocation_ordinal: u64,
260    target: Option<String>,
261    artifact: Option<PathBuf>,
262    error: String,
263}
264
265#[derive(Debug, Clone, PartialEq, Eq)]
266pub struct RustCargoRunnerExecution {
267    pub exit_code: i32,
268    pub unit_path: Option<PathBuf>,
269}
270
271#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
272#[serde(rename_all = "camelCase", deny_unknown_fields)]
273pub struct RustCompilerRunRequest {
274    pub project_root: PathBuf,
275    pub command: Vec<String>,
276    pub run_id: String,
277    pub generated_at: String,
278    pub wrapper_path: PathBuf,
279    pub companion_candidates: Vec<PathBuf>,
280    pub require_public_capabilities: bool,
281    pub cargo_runner_plan: RustCargoRunnerPlan,
282    pub watchdog_program: Option<PathBuf>,
283}
284
285impl RustCompilerRunRequest {
286    fn build_request(&self) -> RustCompilerBuildRequest {
287        RustCompilerBuildRequest {
288            project_root: self.project_root.clone(),
289            command: self.command.clone(),
290            run_id: self.run_id.clone(),
291            wrapper_path: self.wrapper_path.clone(),
292            companion_candidates: self.companion_candidates.clone(),
293            require_public_capabilities: self.require_public_capabilities,
294            cargo_runner_plan: self.cargo_runner_plan.clone(),
295        }
296    }
297}
298
299#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
300#[serde(rename_all = "camelCase")]
301pub struct RustCompilerTransportHealthRecord {
302    pub scope_id: String,
303    pub scope_kind: String,
304    pub status: String,
305    pub transport: RustCompilerTransportHealth,
306    /// Join-bounded thread quarantine notes for this scope's transport. Work
307    /// under an escaped thread phase is deterministic background evidence.
308    #[serde(skip_serializing_if = "BTreeSet::is_empty")]
309    pub thread_scope_limitations: BTreeSet<String>,
310}
311
312#[derive(Debug, Clone, PartialEq)]
313pub struct RustCompilerFrontendRun {
314    pub selection: crate::rust_compiler_selection::SelectedRustCompilerCompanion,
315    pub declaration: FrontendRunDeclaration,
316    pub request: CoverageReportRequest,
317    pub exit_code: i32,
318    pub artifacts: usize,
319    pub artifact_files: Vec<PathBuf>,
320    pub transport_health: Vec<RustCompilerTransportHealthRecord>,
321    pub build_ms: f64,
322    pub execution_ms: f64,
323}
324
325impl RustCompilerFrontendRun {
326    pub fn archive_entries(&self) -> Result<Vec<EvidenceArchiveEntry>, serde_json::Error> {
327        let model = PersistedCoverageModel::from_declaration(
328            self.request
329                .coverage_model
330                .as_ref()
331                .expect("Rust compiler frontend always declares a coverage model"),
332        )
333        .expect("Rust compiler coverage model is contract-valid");
334        let mut entries = vec![
335            EvidenceArchiveEntry {
336                path: "coverage-model.json".into(),
337                contents: serde_json::to_vec(&model)?,
338            },
339            EvidenceArchiveEntry {
340                path: "frontend.json".into(),
341                contents: serde_json::to_vec(&self.declaration)?,
342            },
343            EvidenceArchiveEntry {
344                path: "manifest.json".into(),
345                contents: serde_json::to_vec(&self.request.manifest)?,
346            },
347        ];
348        for (index, result) in self.request.raw_results.iter().enumerate() {
349            entries.push(EvidenceArchiveEntry {
350                path: format!("results/{index:08}/mcdc.json"),
351                contents: serde_json::to_vec(result)?,
352            });
353        }
354        entries.push(EvidenceArchiveEntry {
355            path: "rust/transport-health.json".into(),
356            contents: serde_json::to_vec(&self.transport_health)?,
357        });
358        Ok(entries)
359    }
360}
361
362#[derive(Debug)]
363pub enum RustCompilerTestError {
364    Build(String),
365    Io { path: PathBuf, reason: String },
366    UnsafeArtifact(String),
367    List { artifact: PathBuf, reason: String },
368    Context(String),
369    DuplicateTest(String),
370    Random(String),
371    Launch { test: String, reason: String },
372    Transport { test: String, reason: String },
373    DroppedEvidence { test: String, dropped: u64 },
374    Projection { test: String, reason: String },
375    UnsupportedCommand(String),
376    UnverifiedExecution { code: i32, reason: String },
377    Interrupted { code: i32, signal: String },
378}
379
380impl std::fmt::Display for RustCompilerTestError {
381    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
382        match self {
383            Self::Build(reason) => write!(formatter, "Rust compiler build failed: {reason}"),
384            Self::Io { path, reason } => write!(formatter, "{}: {reason}", path.display()),
385            Self::UnsafeArtifact(reason) => {
386                write!(formatter, "unsafe Rust test artifact: {reason}")
387            }
388            Self::List { artifact, reason } => write!(
389                formatter,
390                "could not enumerate tests in {}: {reason}",
391                artifact.display()
392            ),
393            Self::Context(reason) => write!(formatter, "invalid Rust test context: {reason}"),
394            Self::DuplicateTest(test) => write!(formatter, "duplicate Rust test identity: {test}"),
395            Self::Random(reason) => {
396                write!(formatter, "could not authenticate Rust evidence: {reason}")
397            }
398            Self::Launch { test, reason } => {
399                write!(formatter, "could not launch Rust test {test}: {reason}")
400            }
401            Self::Transport { test, reason } => {
402                write!(formatter, "invalid Rust transport for {test}: {reason}")
403            }
404            Self::DroppedEvidence { test, dropped } => write!(
405                formatter,
406                "Rust transport dropped {dropped} record(s) for {test}; refusing partial coverage"
407            ),
408            Self::Projection { test, reason } => {
409                write!(formatter, "invalid Rust evidence for {test}: {reason}")
410            }
411            Self::UnsupportedCommand(reason) => formatter.write_str(reason),
412            Self::UnverifiedExecution { code, reason } => write!(
413                formatter,
414                "Rust test command exited {code}, but Supercov could not authenticate complete coverage evidence: {reason}"
415            ),
416            Self::Interrupted { signal, .. } => {
417                write!(formatter, "Rust test run was interrupted by {signal}")
418            }
419        }
420    }
421}
422
423impl std::error::Error for RustCompilerTestError {}
424
425fn io_error(path: &Path, error: impl std::fmt::Display) -> RustCompilerTestError {
426    RustCompilerTestError::Io {
427        path: path.to_path_buf(),
428        reason: error.to_string(),
429    }
430}
431
432#[derive(Debug, Clone)]
433struct TestArtifact {
434    executable: PathBuf,
435    runner_argument: Option<OsString>,
436    package: String,
437    target_key: String,
438    kind: String,
439    source: String,
440    test_harness: bool,
441}
442
443#[derive(Debug, Clone)]
444struct ProcessTask {
445    ordinal: usize,
446    artifact_index: usize,
447    artifact: TestArtifact,
448    test: String,
449    test_id: String,
450    context_id: u64,
451    retry: usize,
452    total_attempts: usize,
453    runner_attempt_id: String,
454    runner: RustCargoRunnerKind,
455    transport: PathBuf,
456    libtest_events: PathBuf,
457    test_arguments: Vec<OsString>,
458    underlying_runner: Option<RustCargoResolvedRunner>,
459}
460
461#[derive(Debug)]
462struct ProcessOutcome {
463    task: ProcessTask,
464    output: SupervisedOutput,
465    read: RustTransportRead,
466    attempt_outcome: RustCargoRunnerAttemptOutcome,
467    started_at_ms: i64,
468    ended_at_ms: i64,
469}
470
471struct StockLibtestExecution {
472    output: SupervisedOutput,
473    events: RustLibtestRunEvents,
474    partition: RustTransportPartition,
475    started_at_ms: i64,
476    ended_at_ms: i64,
477}
478
479struct RemoveFileOnDrop(Option<PathBuf>);
480
481impl Drop for RemoveFileOnDrop {
482    fn drop(&mut self) {
483        if let Some(path) = self.0.take() {
484            let _ = fs::remove_file(path);
485        }
486    }
487}
488
489fn stock_libtest_transport_reason(
490    reason: impl std::fmt::Display,
491    output: &SupervisedOutput,
492) -> String {
493    const LIMIT: usize = 16 * 1024;
494    fn tail(bytes: &[u8]) -> String {
495        let start = bytes.len().saturating_sub(LIMIT);
496        String::from_utf8_lossy(&bytes[start..]).into_owned()
497    }
498
499    let mut message = format!(
500        "{reason}; stock libtest process exit={} ",
501        output.result.exit_code()
502    );
503    if !output.stdout.is_empty() {
504        message.push_str("\nstdout tail:\n");
505        message.push_str(&tail(&output.stdout));
506    }
507    if !output.stderr.is_empty() {
508        message.push_str("\nstderr tail:\n");
509        message.push_str(&tail(&output.stderr));
510    }
511    message
512}
513
514fn epoch_ms() -> Result<i64, RustCompilerTestError> {
515    let millis = SystemTime::now()
516        .duration_since(UNIX_EPOCH)
517        .map_err(|error| RustCompilerTestError::Random(error.to_string()))?
518        .as_millis();
519    i64::try_from(millis).map_err(|error| RustCompilerTestError::Random(error.to_string()))
520}
521
522fn relative_source(root: &Path, source: &Path) -> Result<String, RustCompilerTestError> {
523    let source = canonicalize_simplified(source).map_err(|error| io_error(source, error))?;
524    let relative = source
525        .strip_prefix(root)
526        .map_err(|_| RustCompilerTestError::UnsafeArtifact(source.display().to_string()))?;
527    if relative.as_os_str().is_empty()
528        || relative
529            .components()
530            .any(|component| !matches!(component, Component::Normal(_)))
531    {
532        return Err(RustCompilerTestError::UnsafeArtifact(
533            source.display().to_string(),
534        ));
535    }
536    Ok(relative.to_string_lossy().replace('\\', "/"))
537}
538
539fn normalize_artifacts(
540    project_root: &Path,
541    target_directory: &Path,
542    artifacts: &[RustCompilerTestArtifact],
543) -> Result<Vec<TestArtifact>, RustCompilerTestError> {
544    let target_directory = canonicalize_simplified(target_directory)
545        .map_err(|error| io_error(target_directory, error))?;
546    artifacts
547        .iter()
548        .map(|artifact| {
549            let executable = canonicalize_simplified(&artifact.executable)
550                .map_err(|error| io_error(&artifact.executable, error))?;
551            if !executable.starts_with(&target_directory) {
552                return Err(RustCompilerTestError::UnsafeArtifact(
553                    executable.display().to_string(),
554                ));
555            }
556            let mut target_kinds = artifact.target_kinds.clone();
557            target_kinds.sort();
558            target_kinds.dedup();
559            Ok(TestArtifact {
560                executable,
561                runner_argument: None,
562                package: artifact.package.clone(),
563                target_key: format!("{}:{}", target_kinds.join("+"), artifact.target_name),
564                kind: if artifact.target_kinds.iter().any(|kind| kind == "test") {
565                    "integration".into()
566                } else {
567                    "unit".into()
568                },
569                source: relative_source(project_root, &artifact.source_path)?,
570                test_harness: artifact.test_harness,
571            })
572        })
573        .collect()
574}
575
576fn libtest_id(compilation_target: &str, artifact: &TestArtifact, test: &str) -> String {
577    format!(
578        "rust:libtest:{compilation_target}:{}:{}:{}::{test}",
579        artifact.package, artifact.target_key, artifact.source,
580    )
581}
582
583fn custom_harness_id(compilation_target: &str, artifact: &TestArtifact) -> String {
584    format!(
585        "rust:custom-harness:{compilation_target}:{}:{}:{}",
586        artifact.package, artifact.target_key, artifact.source,
587    )
588}
589
590fn list_tests(
591    project_root: &Path,
592    artifact: &TestArtifact,
593    selection_arguments: &[String],
594    underlying_runner: Option<&RustCargoResolvedRunner>,
595    supervisor: &ProcessSupervisor,
596    options: SupervisionOptions,
597    event_path: &Path,
598) -> Result<Vec<String>, RustCompilerTestError> {
599    let mut test_arguments = selection_arguments
600        .iter()
601        .map(OsString::from)
602        .collect::<Vec<_>>();
603    test_arguments.extend(["--list".into(), "--format".into(), "terse".into()]);
604    let (program, arguments) = artifact_command(artifact, underlying_runner, test_arguments);
605    let mut event_token = [0_u8; supercov_contracts::RUST_LIBTEST_EVENT_TOKEN_SIZE];
606    getrandom::fill(&mut event_token).map_err(|error| {
607        RustCompilerTestError::Random(format!("libtest listing token: {error}"))
608    })?;
609    create_rust_libtest_event_file(event_path, event_token).map_err(|error| {
610        RustCompilerTestError::List {
611            artifact: artifact.executable.clone(),
612            reason: error.to_string(),
613        }
614    })?;
615    let mut event_cleanup = RemoveFileOnDrop(Some(event_path.to_owned()));
616    let output = supervisor
617        .supervise_captured(
618            &CommandSpec {
619                program,
620                arguments,
621                cwd: project_root.to_owned(),
622                environment: Some(inherited_environment([
623                    (
624                        OsString::from(RUST_LIBTEST_EVENTS_ENV),
625                        event_path.as_os_str().to_owned(),
626                    ),
627                    (
628                        OsString::from(RUST_LIBTEST_TOKEN_ENV),
629                        OsString::from(token_hex(&event_token)),
630                    ),
631                ])),
632                captured_output: None,
633            },
634            options,
635            &mut io::sink(),
636        )
637        .map_err(|error| RustCompilerTestError::List {
638            artifact: artifact.executable.clone(),
639            reason: error.to_string(),
640        })?;
641    if output.result.exit_code() != 0 {
642        return Err(RustCompilerTestError::List {
643            artifact: artifact.executable.clone(),
644            reason: format!(
645                "{}{}",
646                String::from_utf8_lossy(&output.stderr),
647                String::from_utf8_lossy(&output.stdout)
648            )
649            .trim()
650            .to_owned(),
651        });
652    }
653    let mut tests = String::from_utf8_lossy(&output.stdout)
654        .lines()
655        .filter_map(|line| {
656            line.strip_suffix(": test")
657                .or_else(|| line.strip_suffix(": benchmark"))
658        })
659        .map(str::to_owned)
660        .collect::<Vec<_>>();
661    tests.sort();
662    tests.dedup();
663    let events = read_rust_libtest_events(event_path, &event_token).map_err(|error| {
664        RustCompilerTestError::List {
665            artifact: artifact.executable.clone(),
666            reason: error.to_string(),
667        }
668    })?;
669    if !matches!(
670        events.as_slice(),
671        [RustLibtestEvent::FilteredOut { .. }, RustLibtestEvent::Filtered { count, .. }]
672            if *count == tests.len() as u64
673    ) {
674        return Err(RustCompilerTestError::List {
675            artifact: artifact.executable.clone(),
676            reason: "authenticated libtest listing events disagree with terse output".into(),
677        });
678    }
679    fs::remove_file(event_path).map_err(|error| io_error(event_path, error))?;
680    event_cleanup.0 = None;
681    Ok(tests)
682}
683
684fn artifact_command(
685    artifact: &TestArtifact,
686    underlying_runner: Option<&RustCargoResolvedRunner>,
687    test_arguments: Vec<OsString>,
688) -> (OsString, Vec<OsString>) {
689    match underlying_runner {
690        Some(runner) => {
691            let mut arguments = runner
692                .arguments
693                .iter()
694                .map(OsString::from)
695                .collect::<Vec<_>>();
696            arguments.push(
697                artifact
698                    .runner_argument
699                    .clone()
700                    .unwrap_or_else(|| artifact.executable.clone().into_os_string()),
701            );
702            arguments.extend(test_arguments);
703            (runner.program.clone().into_os_string(), arguments)
704        }
705        None => (artifact.executable.clone().into_os_string(), test_arguments),
706    }
707}
708
709fn token_hex<const N: usize>(token: &[u8; N]) -> String {
710    token.iter().map(|byte| format!("{byte:02x}")).collect()
711}
712
713fn phase_id(run_id: &str, attempt_id: &str) -> String {
714    let mut digest = Sha256::new();
715    digest.update((run_id.len() as u64).to_be_bytes());
716    digest.update(run_id.as_bytes());
717    digest.update((attempt_id.len() as u64).to_be_bytes());
718    digest.update(attempt_id.as_bytes());
719    let hex = format!("{:x}", digest.finalize());
720    format!("rust-test:{}", &hex[..40])
721}
722
723fn snapshot_has_evidence(snapshot: &RuntimeSnapshot) -> bool {
724    !snapshot.hits.is_empty() || !snapshot.decisions.is_empty() || !snapshot.events.is_empty()
725}
726
727fn inherited_environment(
728    overrides: impl IntoIterator<Item = (OsString, OsString)>,
729) -> Vec<(OsString, OsString)> {
730    let mut environment = std::env::vars_os().collect::<BTreeMap<_, _>>();
731    environment.extend(overrides);
732    environment.into_iter().collect()
733}
734
735fn run_process(
736    project_root: &Path,
737    task: &ProcessTask,
738    supervisor: &ProcessSupervisor,
739    options: SupervisionOptions,
740) -> Result<ProcessOutcome, String> {
741    let mut token = [0_u8; TOKEN_BYTES];
742    getrandom::fill(&mut token).map_err(|error| error.to_string())?;
743    create_rust_transport(
744        &task.transport,
745        token,
746        DEFAULT_DESCRIPTOR_CAPACITY,
747        DEFAULT_PAYLOAD_CAPACITY,
748    )
749    .map_err(|error| error.to_string())?;
750    let mut transport_cleanup = RemoveFileOnDrop(Some(task.transport.clone()));
751    let event_token = if task.runner == RustCargoRunnerKind::CargoCustomHarness {
752        None
753    } else {
754        let mut token = [0_u8; supercov_contracts::RUST_LIBTEST_EVENT_TOKEN_SIZE];
755        getrandom::fill(&mut token).map_err(|error| error.to_string())?;
756        create_rust_libtest_event_file(&task.libtest_events, token)
757            .map_err(|error| error.to_string())?;
758        Some(token)
759    };
760    let mut event_cleanup =
761        RemoveFileOnDrop(event_token.is_some().then(|| task.libtest_events.clone()));
762    let started_at_ms = epoch_ms().map_err(|error| error.to_string())?;
763    let (program, arguments) = artifact_command(
764        &task.artifact,
765        task.underlying_runner.as_ref(),
766        task.test_arguments.clone(),
767    );
768    let mut environment = vec![
769        (
770            OsString::from(RUST_TRANSPORT_ENV),
771            task.transport.clone().into_os_string(),
772        ),
773        (
774            OsString::from(RUST_TRANSPORT_TOKEN_ENV),
775            OsString::from(token_hex(&token)),
776        ),
777        (
778            OsString::from(RUST_CONTEXT_ENV),
779            OsString::from(format!("{:016x}", task.context_id)),
780        ),
781    ];
782    if let Some(event_token) = &event_token {
783        environment.extend([
784            (
785                OsString::from(RUST_LIBTEST_EVENTS_ENV),
786                task.libtest_events.clone().into_os_string(),
787            ),
788            (
789                OsString::from(RUST_LIBTEST_TOKEN_ENV),
790                OsString::from(token_hex(event_token)),
791            ),
792        ]);
793    }
794    let output = supervisor
795        .supervise_captured(
796            &CommandSpec {
797                program,
798                arguments,
799                cwd: project_root.to_owned(),
800                environment: Some(inherited_environment(environment)),
801                captured_output: None,
802            },
803            options,
804            &mut io::sink(),
805        )
806        .map_err(|error| error.to_string())?;
807    let ended_at_ms = epoch_ms().map_err(|error| error.to_string())?;
808    let read = read_rust_transport(&task.transport, &token).map_err(|error| error.to_string())?;
809    let attempt_outcome = if let Some(event_token) = &event_token {
810        let events = read_rust_libtest_events(&task.libtest_events, event_token)
811            .map_err(|error| error.to_string())?;
812        let joined = validate_rust_libtest_run_events(&events, [task.test.clone()])
813            .map_err(|error| error.to_string())?;
814        let [attempt] = joined.attempts.as_slice() else {
815            return Err("exact libtest attempt did not produce one terminal event".into());
816        };
817        if !joined.unstarted.is_empty()
818            || (matches!(
819                attempt.result,
820                RustLibtestTerminalResult::Passed
821                    | RustLibtestTerminalResult::Ignored
822                    | RustLibtestTerminalResult::Benchmarked
823            ) != (output.result.exit_code() == 0))
824        {
825            return Err("exact libtest terminal event disagrees with process status".into());
826        }
827        RustCargoRunnerAttemptOutcome::Libtest {
828            result: attempt.result,
829            timed_out: attempt.timed_out,
830        }
831    } else {
832        RustCargoRunnerAttemptOutcome::OpaqueProcess
833    };
834    fs::remove_file(&task.transport).map_err(|error| error.to_string())?;
835    transport_cleanup.0 = None;
836    if event_token.is_some() {
837        fs::remove_file(&task.libtest_events).map_err(|error| error.to_string())?;
838        event_cleanup.0 = None;
839    }
840    Ok(ProcessOutcome {
841        task: task.clone(),
842        output,
843        read,
844        attempt_outcome,
845        started_at_ms,
846        ended_at_ms,
847    })
848}
849
850#[allow(clippy::too_many_arguments)]
851fn run_stock_libtest_artifact(
852    project_root: &Path,
853    artifact: &TestArtifact,
854    underlying_runner: Option<&RustCargoResolvedRunner>,
855    arguments: Vec<OsString>,
856    selected_tests: &[String],
857    contexts: &BTreeMap<String, u64>,
858    transport_path: &Path,
859    event_path: &Path,
860    supervisor: &ProcessSupervisor,
861    options: SupervisionOptions,
862) -> Result<StockLibtestExecution, RustCompilerTestError> {
863    let mut token = [0_u8; TOKEN_BYTES];
864    getrandom::fill(&mut token)
865        .map_err(|error| RustCompilerTestError::Random(error.to_string()))?;
866    create_rust_transport(
867        transport_path,
868        token,
869        DEFAULT_DESCRIPTOR_CAPACITY,
870        DEFAULT_PAYLOAD_CAPACITY,
871    )
872    .map_err(|error| RustCompilerTestError::Transport {
873        test: artifact.target_key.clone(),
874        reason: error.to_string(),
875    })?;
876    let mut transport_cleanup = RemoveFileOnDrop(Some(transport_path.to_owned()));
877    let mut event_token = [0_u8; supercov_contracts::RUST_LIBTEST_EVENT_TOKEN_SIZE];
878    getrandom::fill(&mut event_token)
879        .map_err(|error| RustCompilerTestError::Random(error.to_string()))?;
880    create_rust_libtest_event_file(event_path, event_token).map_err(|error| {
881        RustCompilerTestError::Transport {
882            test: artifact.target_key.clone(),
883            reason: error.to_string(),
884        }
885    })?;
886    let mut event_cleanup = RemoveFileOnDrop(Some(event_path.to_owned()));
887    let (program, arguments) = artifact_command(artifact, underlying_runner, arguments);
888    let started_at_ms = epoch_ms()?;
889    let output = supervisor
890        .supervise_captured(
891            &CommandSpec {
892                program,
893                arguments,
894                cwd: project_root.to_owned(),
895                environment: Some(inherited_environment([
896                    (
897                        OsString::from(RUST_TRANSPORT_ENV),
898                        transport_path.as_os_str().to_owned(),
899                    ),
900                    (
901                        OsString::from(RUST_TRANSPORT_TOKEN_ENV),
902                        OsString::from(token_hex(&token)),
903                    ),
904                    (
905                        OsString::from(RUST_CONTEXT_ENV),
906                        OsString::from("0000000000000000"),
907                    ),
908                    (
909                        OsString::from(RUST_LIBTEST_EVENTS_ENV),
910                        event_path.as_os_str().to_owned(),
911                    ),
912                    (
913                        OsString::from(RUST_LIBTEST_TOKEN_ENV),
914                        OsString::from(token_hex(&event_token)),
915                    ),
916                ])),
917                captured_output: None,
918            },
919            options,
920            &mut io::sink(),
921        )
922        .map_err(|error| RustCompilerTestError::Launch {
923            test: artifact.target_key.clone(),
924            reason: error.to_string(),
925        })?;
926    let ended_at_ms = epoch_ms()?;
927    let read = read_rust_transport(transport_path, &token).map_err(|error| {
928        RustCompilerTestError::Transport {
929            test: artifact.target_key.clone(),
930            reason: error.to_string(),
931        }
932    })?;
933    let events = read_rust_libtest_events(event_path, &event_token).map_err(|error| {
934        RustCompilerTestError::Transport {
935            test: artifact.target_key.clone(),
936            reason: stock_libtest_transport_reason(error, &output),
937        }
938    })?;
939    let events = validate_rust_libtest_run_events(&events, selected_tests.iter().cloned())
940        .map_err(|error| RustCompilerTestError::Transport {
941            test: artifact.target_key.clone(),
942            reason: stock_libtest_transport_reason(error, &output),
943        })?;
944    let terminal_failure = events
945        .attempts
946        .iter()
947        .any(|attempt| attempt.result == RustLibtestTerminalResult::Failed);
948    let expected_success = !terminal_failure && events.unstarted.is_empty();
949    if expected_success != (output.result.exit_code() == 0) {
950        return Err(RustCompilerTestError::UnverifiedExecution {
951            code: output.result.exit_code(),
952            reason: "stock libtest process status disagrees with authenticated terminal and fail-fast events"
953                .into(),
954        });
955    }
956    let roots = contexts.values().copied().collect::<BTreeSet<_>>();
957    let partition = partition_rust_transport_by_test_contexts(&read, &roots).map_err(|error| {
958        RustCompilerTestError::Transport {
959            test: artifact.target_key.clone(),
960            reason: error.to_string(),
961        }
962    })?;
963    fs::remove_file(transport_path).map_err(|error| io_error(transport_path, error))?;
964    transport_cleanup.0 = None;
965    fs::remove_file(event_path).map_err(|error| io_error(event_path, error))?;
966    event_cleanup.0 = None;
967    Ok(StockLibtestExecution {
968        output,
969        events,
970        partition,
971        started_at_ms,
972        ended_at_ms,
973    })
974}
975
976fn execute_process_tasks(
977    project_root: &Path,
978    tasks: &[ProcessTask],
979    requested_workers: usize,
980    supervisor: &ProcessSupervisor,
981    options: SupervisionOptions,
982) -> Result<Vec<ProcessOutcome>, RustCompilerTestError> {
983    let workers = requested_workers.min(tasks.len());
984    let next = AtomicUsize::new(0);
985    let outcomes = Mutex::new(Vec::<Result<ProcessOutcome, String>>::with_capacity(
986        tasks.len(),
987    ));
988    std::thread::scope(|scope| {
989        for _ in 0..workers {
990            scope.spawn(|| {
991                loop {
992                    let index = next.fetch_add(1, Ordering::Relaxed);
993                    let Some(task) = tasks.get(index) else { break };
994                    outcomes
995                        .lock()
996                        .expect("Rust compiler result lock poisoned")
997                        .push(run_process(project_root, task, supervisor, options));
998                }
999            });
1000        }
1001    });
1002    let mut outcomes = outcomes
1003        .into_inner()
1004        .map_err(|_| RustCompilerTestError::Context("Rust compiler result lock poisoned".into()))?
1005        .into_iter()
1006        .collect::<Result<Vec<_>, _>>()
1007        .map_err(|reason| RustCompilerTestError::Launch {
1008            test: "unknown attempt".into(),
1009            reason,
1010        })?;
1011    outcomes.sort_by_key(|outcome| outcome.task.ordinal);
1012    Ok(outcomes)
1013}
1014
1015fn regular_directory(path: &Path) -> Result<PathBuf, RustCompilerTestError> {
1016    let metadata = fs::symlink_metadata(path).map_err(|error| io_error(path, error))?;
1017    if !metadata.file_type().is_dir() {
1018        return Err(RustCompilerTestError::UnsafeArtifact(
1019            path.display().to_string(),
1020        ));
1021    }
1022    canonicalize_simplified(path).map_err(|error| io_error(path, error))
1023}
1024
1025#[cfg(unix)]
1026fn sync_directory(path: &Path) -> Result<(), RustCompilerTestError> {
1027    fs::File::open(path)
1028        .and_then(|directory| directory.sync_all())
1029        .map_err(|error| io_error(path, error))
1030}
1031
1032#[cfg(not(unix))]
1033fn sync_directory(_path: &Path) -> Result<(), RustCompilerTestError> {
1034    Ok(())
1035}
1036
1037fn write_cargo_runner_unit(
1038    output_directory: &Path,
1039    unit: &RustCargoRunnerUnit,
1040) -> Result<PathBuf, RustCompilerTestError> {
1041    let artifact = unit.artifact.to_str().ok_or_else(|| {
1042        RustCompilerTestError::Context("Cargo test artifact path is not UTF-8".into())
1043    })?;
1044    let mut identity = Sha256::new();
1045    identity.update((unit.target.len() as u64).to_be_bytes());
1046    identity.update(unit.target.as_bytes());
1047    identity.update((artifact.len() as u64).to_be_bytes());
1048    identity.update(artifact.as_bytes());
1049    let digest = format!("{:x}", identity.finalize());
1050    let destination = output_directory.join(format!(
1051        "libtest-{:016}-{}.json",
1052        unit.invocation_ordinal,
1053        &digest[..24]
1054    ));
1055    let partial = output_directory.join(format!(
1056        ".libtest-{:016}-{}-{}.partial",
1057        unit.invocation_ordinal,
1058        &digest[..24],
1059        std::process::id()
1060    ));
1061    let bytes = serde_json::to_vec(unit)
1062        .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1063    let mut options = OpenOptions::new();
1064    options.write(true).create_new(true);
1065    #[cfg(unix)]
1066    {
1067        use std::os::unix::fs::OpenOptionsExt as _;
1068        options.mode(0o600);
1069    }
1070    let mut file = options
1071        .open(&partial)
1072        .map_err(|error| io_error(&partial, error))?;
1073    let write_result = (|| {
1074        file.write_all(&bytes)
1075            .map_err(|error| io_error(&partial, error))?;
1076        file.sync_all().map_err(|error| io_error(&partial, error))?;
1077        drop(file);
1078        fs::rename(&partial, &destination).map_err(|error| io_error(&destination, error))?;
1079        sync_directory(output_directory)
1080    })();
1081    if write_result.is_err() {
1082        let _ = fs::remove_file(&partial);
1083    }
1084    write_result.map(|()| destination)
1085}
1086
1087fn write_cargo_runner_failure(
1088    output_directory: &Path,
1089    failure: &RustCargoRunnerFailure,
1090) -> Result<PathBuf, RustCompilerTestError> {
1091    let destination =
1092        output_directory.join(format!("failure-{:016}.json", failure.invocation_ordinal));
1093    let partial = output_directory.join(format!(
1094        ".failure-{:016}-{}.partial",
1095        failure.invocation_ordinal,
1096        std::process::id()
1097    ));
1098    let bytes = serde_json::to_vec(failure)
1099        .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1100    let mut options = OpenOptions::new();
1101    options.write(true).create_new(true);
1102    #[cfg(unix)]
1103    {
1104        use std::os::unix::fs::OpenOptionsExt as _;
1105        options.mode(0o600);
1106    }
1107    let mut file = options
1108        .open(&partial)
1109        .map_err(|error| io_error(&partial, error))?;
1110    let write_result = (|| {
1111        file.write_all(&bytes)
1112            .map_err(|error| io_error(&partial, error))?;
1113        file.sync_all().map_err(|error| io_error(&partial, error))?;
1114        drop(file);
1115        fs::rename(&partial, &destination).map_err(|error| io_error(&destination, error))?;
1116        sync_directory(output_directory)
1117    })();
1118    if write_result.is_err() {
1119        let _ = fs::remove_file(&partial);
1120    }
1121    write_result.map(|()| destination)
1122}
1123
1124fn reserve_cargo_runner_ordinal(output_directory: &Path) -> Result<u64, RustCompilerTestError> {
1125    for ordinal in 0..1_000_000_u64 {
1126        let reservation = output_directory.join(format!(".sequence-{ordinal:016}.reserved"));
1127        let mut options = OpenOptions::new();
1128        options.write(true).create_new(true);
1129        #[cfg(unix)]
1130        {
1131            use std::os::unix::fs::OpenOptionsExt as _;
1132            options.mode(0o600);
1133        }
1134        match options.open(&reservation) {
1135            Ok(file) => {
1136                file.sync_all()
1137                    .map_err(|error| io_error(&reservation, error))?;
1138                sync_directory(output_directory)?;
1139                return Ok(ordinal);
1140            }
1141            Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}
1142            Err(error) => return Err(io_error(&reservation, error)),
1143        }
1144    }
1145    Err(RustCompilerTestError::Context(
1146        "Cargo runner invocation ordinal space is exhausted".into(),
1147    ))
1148}
1149
1150fn run_nextest_list_passthrough(
1151    current_directory: &Path,
1152    artifact: &TestArtifact,
1153    underlying_runner: Option<&RustCargoResolvedRunner>,
1154    arguments: Vec<OsString>,
1155    watchdog_program: Option<&Path>,
1156    stdout: &mut dyn Write,
1157    stderr: &mut dyn Write,
1158) -> Result<RustCargoRunnerExecution, RustCompilerTestError> {
1159    let supervisor = watchdog_program
1160        .map_or_else(ProcessSupervisor::new, ProcessSupervisor::new_crash_safe)
1161        .map_err(|error| RustCompilerTestError::Launch {
1162            test: "nextest list".into(),
1163            reason: error.to_string(),
1164        })?;
1165    let options =
1166        SupervisionOptions::from_environment().map_err(|error| RustCompilerTestError::Launch {
1167            test: "nextest list".into(),
1168            reason: error.to_string(),
1169        })?;
1170    let (program, arguments) = artifact_command(artifact, underlying_runner, arguments);
1171    let output = supervisor
1172        .supervise_captured(
1173            &CommandSpec {
1174                program,
1175                arguments,
1176                cwd: current_directory.to_owned(),
1177                environment: Some(inherited_environment([])),
1178                captured_output: None,
1179            },
1180            options,
1181            &mut io::sink(),
1182        )
1183        .map_err(|error| RustCompilerTestError::Launch {
1184            test: "nextest list".into(),
1185            reason: error.to_string(),
1186        })?;
1187    stdout
1188        .write_all(&output.stdout)
1189        .map_err(|error| io_error(current_directory, error))?;
1190    stderr
1191        .write_all(&output.stderr)
1192        .map_err(|error| io_error(current_directory, error))?;
1193    Ok(RustCargoRunnerExecution {
1194        exit_code: output.result.exit_code(),
1195        unit_path: None,
1196    })
1197}
1198
1199pub fn run_cargo_libtest_runner(
1200    config_path: &Path,
1201    arguments: Vec<OsString>,
1202    watchdog_program: Option<PathBuf>,
1203    stdout: &mut dyn Write,
1204    stderr: &mut dyn Write,
1205) -> Result<RustCargoRunnerExecution, RustCompilerTestError> {
1206    let config_metadata =
1207        fs::symlink_metadata(config_path).map_err(|error| io_error(config_path, error))?;
1208    if !config_metadata.file_type().is_file() {
1209        return Err(RustCompilerTestError::UnsafeArtifact(
1210            config_path.display().to_string(),
1211        ));
1212    }
1213    let config: RustCargoRunnerConfig = serde_json::from_slice(
1214        &fs::read(config_path).map_err(|error| io_error(config_path, error))?,
1215    )
1216    .map_err(|error| {
1217        RustCompilerTestError::Context(format!("invalid Cargo runner config: {error}"))
1218    })?;
1219    if config.version != RUST_CARGO_RUNNER_VERSION
1220        || !config.run_id.starts_with("run_")
1221        || config.run_id.len() != 20
1222        || !config.run_id[4..]
1223            .bytes()
1224            .all(|byte| byte.is_ascii_hexdigit())
1225    {
1226        return Err(RustCompilerTestError::Context(
1227            "Cargo runner config has an unsupported version or invalid run ID".into(),
1228        ));
1229    }
1230    let target_directory = regular_directory(&config.target_directory)?;
1231    let output_directory = regular_directory(&config.output_directory)?;
1232    let run_root = target_directory
1233        .parent()
1234        .ok_or_else(|| RustCompilerTestError::Context("Cargo target has no run root".into()))?;
1235    if !output_directory.starts_with(run_root) || output_directory == target_directory {
1236        return Err(RustCompilerTestError::UnsafeArtifact(
1237            output_directory.display().to_string(),
1238        ));
1239    }
1240    let mut configured_targets = BTreeSet::new();
1241    if config.target_runners.is_empty()
1242        || config
1243            .target_runners
1244            .iter()
1245            .any(|target| target.target.is_empty() || !configured_targets.insert(&target.target))
1246    {
1247        return Err(RustCompilerTestError::Context(
1248            "Cargo runner config has empty or duplicate target identities".into(),
1249        ));
1250    }
1251    let mut configured_artifacts = BTreeSet::new();
1252    if config.artifacts.iter().any(|artifact| {
1253        !configured_artifacts.insert(&artifact.executable)
1254            || !artifact.executable.starts_with(&target_directory)
1255    }) {
1256        return Err(RustCompilerTestError::Context(
1257            "Cargo runner config has duplicate or out-of-target artifacts".into(),
1258        ));
1259    }
1260    let runner_identity = classify_rust_runner_environment()
1261        .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1262    let run_id = config.run_id.clone();
1263    let failure_target = arguments
1264        .first()
1265        .and_then(|target| target.clone().into_string().ok());
1266    let failure_artifact = arguments.get(1).map(PathBuf::from);
1267    let mut runner_arguments = arguments.into_iter();
1268    let target = runner_arguments
1269        .next()
1270        .ok_or_else(|| {
1271            RustCompilerTestError::Context("Cargo runner received no target identity".into())
1272        })?
1273        .into_string()
1274        .map_err(|_| {
1275            RustCompilerTestError::Context(
1276                "Cargo runner received a non-UTF-8 target identity".into(),
1277            )
1278        })?;
1279    let target_runner = config
1280        .target_runners
1281        .iter()
1282        .find(|candidate| candidate.target == target)
1283        .ok_or_else(|| {
1284            RustCompilerTestError::Context(format!(
1285                "Cargo runner received an unconfigured target identity: {target}"
1286            ))
1287        })?;
1288    let artifact_argument = runner_arguments.next().ok_or_else(|| {
1289        RustCompilerTestError::Context("Cargo runner received no artifact".into())
1290    })?;
1291    let artifact_path = PathBuf::from(&artifact_argument);
1292    let artifact =
1293        canonicalize_simplified(&artifact_path).map_err(|error| io_error(&artifact_path, error))?;
1294    if !artifact.starts_with(&target_directory)
1295        || !fs::symlink_metadata(&artifact).is_ok_and(|metadata| metadata.file_type().is_file())
1296    {
1297        return Err(RustCompilerTestError::UnsafeArtifact(
1298            artifact.display().to_string(),
1299        ));
1300    }
1301    let test_harness = config
1302        .artifacts
1303        .iter()
1304        .find(|candidate| candidate.executable == artifact)
1305        .map(|candidate| candidate.test_harness);
1306    if test_harness.is_none()
1307        && !matches!(
1308            runner_identity,
1309            RustRunnerInvocationIdentity::NextestList(_)
1310        )
1311    {
1312        return Err(RustCompilerTestError::Context(format!(
1313            "Cargo runner executed an unclassified artifact: {}",
1314            artifact.display()
1315        )));
1316    }
1317    let arguments = runner_arguments.collect::<Vec<_>>();
1318    let current_directory = std::env::current_dir()
1319        .and_then(fs::canonicalize)
1320        .map_err(|error| io_error(Path::new("."), error))?;
1321    let test_artifact = TestArtifact {
1322        executable: artifact.clone(),
1323        runner_argument: Some(artifact_argument),
1324        package: "cargo-pending".into(),
1325        target_key: "cargo-pending".into(),
1326        kind: "cargo-pending".into(),
1327        source: "cargo-pending".into(),
1328        test_harness: test_harness.unwrap_or(true),
1329    };
1330    let underlying_runner = target_runner.underlying_runner.clone();
1331    if matches!(
1332        runner_identity,
1333        RustRunnerInvocationIdentity::NextestList(_)
1334    ) {
1335        return run_nextest_list_passthrough(
1336            &current_directory,
1337            &test_artifact,
1338            underlying_runner.as_ref(),
1339            arguments,
1340            watchdog_program.as_deref(),
1341            stdout,
1342            stderr,
1343        );
1344    }
1345
1346    let invocation_ordinal = reserve_cargo_runner_ordinal(&output_directory)?;
1347    let result = (|| {
1348        let utf8_arguments = arguments
1349            .iter()
1350            .cloned()
1351            .map(|argument| {
1352                argument.into_string().map_err(|_| {
1353                    RustCompilerTestError::Context(
1354                        "Cargo runner received a non-UTF-8 libtest argument".into(),
1355                    )
1356                })
1357            })
1358            .collect::<Result<Vec<_>, _>>()?;
1359        let invocation = crate::rust_test_runner::CargoTestInvocation {
1360            program: "cargo".into(),
1361            kind: crate::rust_test_runner::RustCargoCommandKind::CargoTest,
1362            arguments: vec!["test".into()],
1363            runner_arguments: utf8_arguments.clone(),
1364        };
1365        let artifact_digest = format!(
1366            "{:x}",
1367            Sha256::digest(artifact.as_os_str().as_encoded_bytes())
1368        );
1369        let transport_directory = output_directory.join("attempts").join(format!(
1370            "{invocation_ordinal:016}-{}",
1371            &artifact_digest[..24]
1372        ));
1373        fs::create_dir_all(&transport_directory)
1374            .map_err(|error| io_error(&transport_directory, error))?;
1375        let transport_directory = regular_directory(&transport_directory)?;
1376        let supervisor = watchdog_program
1377            .as_deref()
1378            .map_or_else(ProcessSupervisor::new, ProcessSupervisor::new_crash_safe)
1379            .map_err(|error| RustCompilerTestError::Launch {
1380                test: "Cargo runner".into(),
1381                reason: error.to_string(),
1382            })?;
1383        let options = SupervisionOptions::from_environment().map_err(|error| {
1384            RustCompilerTestError::Launch {
1385                test: "Cargo runner".into(),
1386                reason: error.to_string(),
1387            }
1388        })?;
1389        let (runner, runner_run_id, runner_version, runner_binary_id, requested_workers, attempts) =
1390            match &runner_identity {
1391                RustRunnerInvocationIdentity::CargoSingleAttempt if test_artifact.test_harness => {
1392                    let selection = rust_libtest_selection(&invocation).map_err(|error| {
1393                        RustCompilerTestError::UnsupportedCommand(error.to_string())
1394                    })?;
1395                    let tests = list_tests(
1396                        &current_directory,
1397                        &test_artifact,
1398                        &selection.list_arguments,
1399                        underlying_runner.as_ref(),
1400                        &supervisor,
1401                        options,
1402                        &transport_directory.join("list.events"),
1403                    )?;
1404                    let contexts = preflight_rust_test_contexts(tests.clone())
1405                        .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1406                    let stock = run_stock_libtest_artifact(
1407                        &current_directory,
1408                        &test_artifact,
1409                        underlying_runner.as_ref(),
1410                        arguments.clone(),
1411                        &tests,
1412                        &contexts,
1413                        &transport_directory.join("artifact.mmap"),
1414                        &transport_directory.join("artifact.libtest-events"),
1415                        &supervisor,
1416                        options,
1417                    )?;
1418                    let StockLibtestExecution {
1419                        output,
1420                        events,
1421                        mut partition,
1422                        started_at_ms,
1423                        ended_at_ms,
1424                    } = stock;
1425                    let exit_code = output.result.exit_code();
1426                    stdout
1427                        .write_all(&output.stdout)
1428                        .and_then(|()| stderr.write_all(&output.stderr))
1429                        .map_err(|error| io_error(&output_directory, error))?;
1430                    let mut attempts = Vec::with_capacity(tests.len());
1431                    for attempt in events.attempts {
1432                        let context_id = contexts[&attempt.name];
1433                        let index = attempts.len();
1434                        attempts.push(RustCargoRunnerAttempt {
1435                            test: attempt.name,
1436                            context_id,
1437                            retry: 0,
1438                            total_attempts: 1,
1439                            runner_attempt_id: format!(
1440                                "{}:cargo:{invocation_ordinal:016}:{index:08}",
1441                                config.run_id
1442                            ),
1443                            outcome: RustCargoRunnerAttemptOutcome::Libtest {
1444                                result: attempt.result,
1445                                timed_out: attempt.timed_out,
1446                            },
1447                            transport: partition
1448                                .attributed
1449                                .remove(&context_id)
1450                                .expect("every selected test context was partitioned"),
1451                        });
1452                    }
1453                    for test in events.unstarted {
1454                        let context_id = contexts[&test];
1455                        let index = attempts.len();
1456                        attempts.push(RustCargoRunnerAttempt {
1457                            test,
1458                            context_id,
1459                            retry: 0,
1460                            total_attempts: 1,
1461                            runner_attempt_id: format!(
1462                                "{}:cargo:{invocation_ordinal:016}:{index:08}",
1463                                config.run_id
1464                            ),
1465                            outcome: RustCargoRunnerAttemptOutcome::Unstarted,
1466                            transport: partition
1467                                .attributed
1468                                .remove(&context_id)
1469                                .expect("every unstarted test context was partitioned"),
1470                        });
1471                    }
1472                    if !partition.attributed.is_empty() {
1473                        return Err(RustCompilerTestError::Context(
1474                            "stock libtest event join left selected test contexts unclaimed".into(),
1475                        ));
1476                    }
1477                    let unit = RustCargoRunnerUnit {
1478                        version: RUST_CARGO_RUNNER_VERSION,
1479                        run_id: run_id.clone(),
1480                        invocation_ordinal,
1481                        runner: RustCargoRunnerKind::CargoTest,
1482                        runner_run_id: None,
1483                        runner_version: None,
1484                        runner_binary_id: None,
1485                        target,
1486                        artifact,
1487                        arguments: utf8_arguments,
1488                        invocation: RustCargoRunnerInvocation {
1489                            result: output.result,
1490                            started_at_ms,
1491                            ended_at_ms,
1492                            stdout: output.stdout,
1493                            stderr: output.stderr,
1494                            background_transport: partition.background,
1495                        },
1496                        attempts,
1497                        thread_scope_limitations: partition.thread_scope_limitations,
1498                    };
1499                    let unit_path = write_cargo_runner_unit(&output_directory, &unit)?;
1500                    fs::remove_dir(&transport_directory)
1501                        .map_err(|error| io_error(&transport_directory, error))?;
1502                    return Ok(RustCargoRunnerExecution {
1503                        exit_code,
1504                        unit_path: Some(unit_path),
1505                    });
1506                }
1507                RustRunnerInvocationIdentity::CargoSingleAttempt => (
1508                    RustCargoRunnerKind::CargoCustomHarness,
1509                    None,
1510                    None,
1511                    None,
1512                    1,
1513                    vec![(
1514                        "custom-harness".into(),
1515                        0,
1516                        1,
1517                        format!("{}:cargo:{invocation_ordinal:016}:00000000", config.run_id),
1518                        arguments.clone(),
1519                    )],
1520                ),
1521                RustRunnerInvocationIdentity::NextestAttempt(NextestAttemptIdentity {
1522                    invocation,
1523                    test_name,
1524                    retry,
1525                    total_attempts,
1526                    runner_attempt_id,
1527                }) => {
1528                    if !test_artifact.test_harness {
1529                        return Err(RustCompilerTestError::Context(
1530                            "nextest attempted to execute a custom Cargo harness".into(),
1531                        ));
1532                    }
1533                    if !utf8_arguments
1534                        .windows(2)
1535                        .any(|pair| pair == ["--exact", test_name])
1536                    {
1537                        return Err(RustCompilerTestError::Context(
1538                            "nextest target-runner arguments do not select NEXTEST_TEST_NAME exactly"
1539                                .into(),
1540                        ));
1541                    }
1542                    (
1543                        RustCargoRunnerKind::Nextest,
1544                        Some(invocation.run_id.clone()),
1545                        Some(invocation.version.clone()),
1546                        Some(invocation.binary_id.clone()),
1547                        1,
1548                        vec![(
1549                            test_name.clone(),
1550                            *retry,
1551                            *total_attempts,
1552                            runner_attempt_id.clone(),
1553                            arguments.clone(),
1554                        )],
1555                    )
1556                }
1557                RustRunnerInvocationIdentity::NextestList(_) => unreachable!("handled above"),
1558            };
1559        let tests = attempts
1560            .iter()
1561            .map(|(test, _, _, _, _)| test.clone())
1562            .collect::<Vec<_>>();
1563        let contexts = preflight_rust_test_contexts(tests.clone())
1564            .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
1565        let tasks = attempts
1566            .into_iter()
1567            .enumerate()
1568            .map(
1569                |(index, (test, retry, total_attempts, runner_attempt_id, test_arguments))| {
1570                    ProcessTask {
1571                        ordinal: index,
1572                        artifact_index: 0,
1573                        artifact: test_artifact.clone(),
1574                        test: test.clone(),
1575                        test_id: format!("rust:cargo-runner:{}::{test}", &artifact_digest[..24]),
1576                        context_id: contexts[&test],
1577                        retry,
1578                        total_attempts,
1579                        runner_attempt_id,
1580                        runner,
1581                        transport: transport_directory.join(format!("{index:08}.mmap")),
1582                        libtest_events: transport_directory
1583                            .join(format!("{index:08}.libtest-events")),
1584                        test_arguments,
1585                        underlying_runner: underlying_runner.clone(),
1586                    }
1587                },
1588            )
1589            .collect::<Vec<_>>();
1590        let mut outcomes = execute_process_tasks(
1591            &current_directory,
1592            &tasks,
1593            requested_workers,
1594            &supervisor,
1595            options,
1596        )?;
1597        let [outcome] = outcomes.as_mut_slice() else {
1598            return Err(RustCompilerTestError::Context(
1599                "custom-harness and nextest runner invocations must own exactly one process".into(),
1600            ));
1601        };
1602        stdout
1603            .write_all(&outcome.output.stdout)
1604            .and_then(|()| stderr.write_all(&outcome.output.stderr))
1605            .map_err(|error| io_error(&output_directory, error))?;
1606        let exit_code = outcome.output.result.exit_code();
1607        let roots = BTreeSet::from([outcome.task.context_id]);
1608        let mut partition = partition_rust_transport_by_test_contexts(&outcome.read, &roots)
1609            .map_err(|error| RustCompilerTestError::Transport {
1610                test: outcome.task.test.clone(),
1611                reason: error.to_string(),
1612            })?;
1613        let attempt_outcome = match runner {
1614            RustCargoRunnerKind::CargoCustomHarness => RustCargoRunnerAttemptOutcome::OpaqueProcess,
1615            RustCargoRunnerKind::Nextest => match outcome.attempt_outcome {
1616                RustCargoRunnerAttemptOutcome::Libtest { result, timed_out } => {
1617                    RustCargoRunnerAttemptOutcome::Libtest { result, timed_out }
1618                }
1619                _ => {
1620                    return Err(RustCompilerTestError::Context(
1621                        "nextest process has no authenticated libtest terminal event".into(),
1622                    ));
1623                }
1624            },
1625            RustCargoRunnerKind::CargoTest => unreachable!("stock libtest returned above"),
1626        };
1627        let attempts = vec![RustCargoRunnerAttempt {
1628            test: outcome.task.test.clone(),
1629            context_id: outcome.task.context_id,
1630            retry: outcome.task.retry,
1631            total_attempts: outcome.task.total_attempts,
1632            runner_attempt_id: outcome.task.runner_attempt_id.clone(),
1633            outcome: attempt_outcome,
1634            transport: partition
1635                .attributed
1636                .remove(&outcome.task.context_id)
1637                .expect("the one attempt context was partitioned"),
1638        }];
1639        let unit = RustCargoRunnerUnit {
1640            version: RUST_CARGO_RUNNER_VERSION,
1641            run_id: run_id.clone(),
1642            invocation_ordinal,
1643            runner,
1644            runner_run_id,
1645            runner_version,
1646            runner_binary_id,
1647            target,
1648            artifact,
1649            arguments: utf8_arguments,
1650            invocation: RustCargoRunnerInvocation {
1651                result: outcome.output.result.clone(),
1652                started_at_ms: outcome.started_at_ms,
1653                ended_at_ms: outcome.ended_at_ms,
1654                stdout: outcome.output.stdout.clone(),
1655                stderr: outcome.output.stderr.clone(),
1656                background_transport: partition.background,
1657            },
1658            attempts,
1659            thread_scope_limitations: partition.thread_scope_limitations,
1660        };
1661        let unit_path = write_cargo_runner_unit(&output_directory, &unit)?;
1662        fs::remove_dir(&transport_directory)
1663            .map_err(|error| io_error(&transport_directory, error))?;
1664        Ok(RustCargoRunnerExecution {
1665            exit_code,
1666            unit_path: Some(unit_path),
1667        })
1668    })();
1669    if let Err(error) = &result {
1670        let failure = RustCargoRunnerFailure {
1671            version: RUST_CARGO_RUNNER_VERSION,
1672            run_id,
1673            invocation_ordinal,
1674            target: failure_target,
1675            artifact: failure_artifact,
1676            error: error.to_string(),
1677        };
1678        if let Err(publication_error) = write_cargo_runner_failure(&output_directory, &failure) {
1679            return Err(RustCompilerTestError::Context(format!(
1680                "{error}; Cargo runner also could not publish its failure: {publication_error}"
1681            )));
1682        }
1683    }
1684    result
1685}
1686
1687pub fn read_cargo_runner_units(
1688    output_directory: &Path,
1689    run_id: &str,
1690    expected_targets: &[String],
1691) -> Result<Vec<RustCargoRunnerUnit>, RustCompilerTestError> {
1692    let output_directory = regular_directory(output_directory)?;
1693    let expected_target_count = expected_targets.len();
1694    let expected_targets = expected_targets.iter().collect::<BTreeSet<_>>();
1695    if expected_targets.is_empty() || expected_targets.len() != expected_target_count {
1696        return Err(RustCompilerTestError::Context(
1697            "Cargo runner expected-target set is empty or duplicated".into(),
1698        ));
1699    }
1700    let mut reservations = BTreeSet::new();
1701    let mut units = Vec::new();
1702    let mut failures = Vec::new();
1703    let mut retained_attempt_state = false;
1704    for entry in
1705        fs::read_dir(&output_directory).map_err(|error| io_error(&output_directory, error))?
1706    {
1707        let entry = entry.map_err(|error| io_error(&output_directory, error))?;
1708        let path = entry.path();
1709        let metadata = fs::symlink_metadata(&path).map_err(|error| io_error(&path, error))?;
1710        let name = entry.file_name();
1711        let name = name.to_str().ok_or_else(|| {
1712            RustCompilerTestError::Context("Cargo runner output name is not UTF-8".into())
1713        })?;
1714        if name == "attempts" && metadata.file_type().is_dir() {
1715            retained_attempt_state = fs::read_dir(&path)
1716                .map_err(|error| io_error(&path, error))?
1717                .next()
1718                .transpose()
1719                .map_err(|error| io_error(&path, error))?
1720                .is_some();
1721            continue;
1722        }
1723        if name.starts_with(".sequence-") && name.ends_with(".reserved") {
1724            if !metadata.file_type().is_file() || metadata.len() != 0 {
1725                return Err(RustCompilerTestError::UnsafeArtifact(
1726                    path.display().to_string(),
1727                ));
1728            }
1729            let ordinal = name
1730                .strip_prefix(".sequence-")
1731                .and_then(|name| name.strip_suffix(".reserved"))
1732                .filter(|value| {
1733                    value.len() == 16 && value.bytes().all(|byte| byte.is_ascii_digit())
1734                })
1735                .and_then(|value| value.parse::<u64>().ok())
1736                .ok_or_else(|| {
1737                    RustCompilerTestError::Context("malformed Cargo runner reservation".into())
1738                })?;
1739            if !reservations.insert(ordinal) {
1740                return Err(RustCompilerTestError::Context(
1741                    "duplicate Cargo runner reservation".into(),
1742                ));
1743            }
1744            continue;
1745        }
1746        if name.starts_with("failure-") && name.ends_with(".json") {
1747            if !metadata.file_type().is_file() {
1748                return Err(RustCompilerTestError::UnsafeArtifact(
1749                    path.display().to_string(),
1750                ));
1751            }
1752            let failure: RustCargoRunnerFailure =
1753                serde_json::from_slice(&fs::read(&path).map_err(|error| io_error(&path, error))?)
1754                    .map_err(|error| {
1755                    RustCompilerTestError::Context(format!(
1756                        "invalid Cargo runner failure unit: {error}"
1757                    ))
1758                })?;
1759            if failure.version != RUST_CARGO_RUNNER_VERSION || failure.run_id != run_id {
1760                return Err(RustCompilerTestError::Context(
1761                    "Cargo runner failure unit has incompatible identity".into(),
1762                ));
1763            }
1764            failures.push(failure);
1765            continue;
1766        }
1767        if !name.starts_with("libtest-")
1768            || !name.ends_with(".json")
1769            || !metadata.file_type().is_file()
1770        {
1771            return Err(RustCompilerTestError::UnsafeArtifact(
1772                path.display().to_string(),
1773            ));
1774        }
1775        let unit: RustCargoRunnerUnit =
1776            serde_json::from_slice(&fs::read(&path).map_err(|error| io_error(&path, error))?)
1777                .map_err(|error| {
1778                    RustCompilerTestError::Context(format!("invalid Cargo runner unit: {error}"))
1779                })?;
1780        if unit.version != RUST_CARGO_RUNNER_VERSION || unit.run_id != run_id {
1781            return Err(RustCompilerTestError::Context(
1782                "Cargo runner unit has incompatible identity".into(),
1783            ));
1784        }
1785        if !expected_targets.contains(&unit.target) {
1786            return Err(RustCompilerTestError::Context(format!(
1787                "Cargo runner unit has an unselected target identity: {}",
1788                unit.target
1789            )));
1790        }
1791        validate_persisted_runner_transport(&unit)?;
1792        units.push(unit);
1793    }
1794    units.sort_by_key(|unit| unit.invocation_ordinal);
1795    failures.sort_by_key(|failure| failure.invocation_ordinal);
1796    let mut publications = BTreeSet::new();
1797    for ordinal in units
1798        .iter()
1799        .map(|unit| unit.invocation_ordinal)
1800        .chain(failures.iter().map(|failure| failure.invocation_ordinal))
1801    {
1802        if !reservations.contains(&ordinal) || !publications.insert(ordinal) {
1803            return Err(RustCompilerTestError::Context(
1804                "Cargo runner invocation publications are malformed or duplicated".into(),
1805            ));
1806        }
1807    }
1808    if reservations.len() != publications.len()
1809        || reservations
1810            .iter()
1811            .enumerate()
1812            .any(|(expected, ordinal)| *ordinal != expected as u64)
1813    {
1814        return Err(RustCompilerTestError::Context(
1815            "Cargo runner reserved an invocation without publishing its unit".into(),
1816        ));
1817    }
1818    if let Some(failure) = failures.first() {
1819        return Err(RustCompilerTestError::Context(format!(
1820            "Cargo runner invocation {} failed for {}: {}",
1821            failure.invocation_ordinal,
1822            failure.artifact.as_ref().map_or_else(
1823                || "an unknown artifact".into(),
1824                |path| path.display().to_string()
1825            ),
1826            failure.error
1827        )));
1828    }
1829    if retained_attempt_state {
1830        return Err(RustCompilerTestError::Context(
1831            "Cargo runner retained attempt transport state".into(),
1832        ));
1833    }
1834    let runner_kinds = units
1835        .iter()
1836        .map(|unit| unit.runner)
1837        .collect::<BTreeSet<_>>();
1838    if runner_kinds.contains(&RustCargoRunnerKind::Nextest) && runner_kinds.len() > 1 {
1839        return Err(RustCompilerTestError::Context(
1840            "Cargo runner mixed standard Cargo and nextest units".into(),
1841        ));
1842    }
1843    let mut attempt_ids = BTreeSet::new();
1844    if units.iter().flat_map(|unit| &unit.attempts).any(|attempt| {
1845        attempt.runner_attempt_id.is_empty()
1846            || !attempt_ids.insert(attempt.runner_attempt_id.clone())
1847    }) {
1848        return Err(RustCompilerTestError::Context(
1849            "Cargo runner attempt identity is empty or duplicated".into(),
1850        ));
1851    }
1852    if runner_kinds.contains(&RustCargoRunnerKind::Nextest) {
1853        let mut nextest_identity = None;
1854        let mut logical_attempts =
1855            BTreeMap::<(String, PathBuf, String), Vec<&RustCargoRunnerAttempt>>::new();
1856        for unit in &units {
1857            let identity = (unit.runner_run_id.as_ref(), unit.runner_version.as_ref());
1858            if identity.0.is_none()
1859                || identity.1.is_none()
1860                || unit.runner_binary_id.is_none()
1861                || unit.attempts.len() != 1
1862            {
1863                return Err(RustCompilerTestError::Context(
1864                    "nextest runner unit lacks exact invocation or attempt identity".into(),
1865                ));
1866            }
1867            match &nextest_identity {
1868                Some(expected) if *expected != identity => {
1869                    return Err(RustCompilerTestError::Context(
1870                        "nextest runner units belong to different runs or versions".into(),
1871                    ));
1872                }
1873                None => nextest_identity = Some(identity),
1874                _ => {}
1875            }
1876            let attempt = &unit.attempts[0];
1877            logical_attempts
1878                .entry((
1879                    unit.target.clone(),
1880                    unit.artifact.clone(),
1881                    attempt.test.clone(),
1882                ))
1883                .or_default()
1884                .push(attempt);
1885        }
1886        for attempts in logical_attempts.values_mut() {
1887            attempts.sort_by_key(|attempt| attempt.retry);
1888            let total_attempts = attempts[0].total_attempts;
1889            for (expected_retry, attempt) in attempts.iter().enumerate() {
1890                if attempt.retry != expected_retry
1891                    || attempt.total_attempts != total_attempts
1892                    || attempt.retry >= total_attempts
1893                    || (expected_retry + 1 < attempts.len()
1894                        && attempt_outcome_succeeded(&attempt.outcome))
1895                {
1896                    return Err(RustCompilerTestError::Context(
1897                        "nextest retry sequence is noncontiguous, inconsistent, or continues after success"
1898                            .into(),
1899                    ));
1900                }
1901            }
1902        }
1903    } else {
1904        let mut artifacts = BTreeSet::new();
1905        if units.iter().any(|unit| {
1906            unit.runner_run_id.is_some()
1907                || unit.runner_version.is_some()
1908                || unit.runner_binary_id.is_some()
1909                || !artifacts.insert((unit.target.clone(), unit.artifact.clone()))
1910                || unit
1911                    .attempts
1912                    .iter()
1913                    .any(|attempt| attempt.retry != 0 || attempt.total_attempts != 1)
1914                || (unit.runner == RustCargoRunnerKind::CargoCustomHarness
1915                    && (unit.attempts.len() != 1 || unit.attempts[0].test != "custom-harness"))
1916        }) {
1917            return Err(RustCompilerTestError::Context(
1918                "Cargo runner units violate the single-attempt artifact identity contract".into(),
1919            ));
1920        }
1921    }
1922    Ok(units)
1923}
1924
1925fn rust_compiler_coverage_model() -> CoverageModelDeclaration {
1926    CoverageModelDeclaration {
1927        language: "rust".into(),
1928        variant: "rustc-mir-owned-v1".into(),
1929        name: "supercov-rust-compiler-v1".into(),
1930        completeness_meaning: "Every compiler-derived obligation in the frozen owned-source denominator was observed; explicit compiler limitations identify Rust surfaces not yet measured.".into(),
1931        measured: vec![
1932            "compiler-derived owned Rust statement and function-entry obligations".into(),
1933            "compiler-derived control-flow alternatives and atomic decision vectors".into(),
1934            "macro-expanded and generated owned code with exact compiler provenance".into(),
1935            "exact stock-libtest in-process context attribution".into(),
1936            "exact process-per-custom-harness-invocation attribution".into(),
1937            "exact assertion-phase attribution for supported assertion macros".into(),
1938        ],
1939        not_measured: vec![
1940            "capabilities explicitly listed in the compiler manifest limitations".into(),
1941            "all input values, semantic partitions, paths, or concurrency interleavings".into(),
1942            "mutation score or assertion fault-detection strength".into(),
1943        ],
1944    }
1945}
1946
1947fn runner_declaration() -> FrontendRunnerDeclaration {
1948    FrontendRunnerDeclaration {
1949        runner: "rust-libtest".into(),
1950        execution_model: ExecutionModel::ParallelContextPropagated,
1951        attribution: FrontendAttribution {
1952            run: AttributionPrecision::Exact,
1953            worker: AttributionPrecision::Exact,
1954            test: AttributionPrecision::Exact,
1955            retry: AttributionPrecision::Exact,
1956            phase: AttributionPrecision::Exact,
1957            action: AttributionPrecision::Unavailable,
1958            assertion: AttributionPrecision::Exact,
1959        },
1960        limitations: vec![FrontendLimitation {
1961            id: "rust-action-linkage-unavailable".into(),
1962            scopes: vec![FrontendLimitationScope::Action],
1963            reason: "Rust libtest exposes no general application-action lifecycle".into(),
1964        }],
1965    }
1966}
1967
1968fn nextest_runner_declaration() -> FrontendRunnerDeclaration {
1969    let mut declaration = runner_declaration();
1970    declaration.runner = "rust-nextest".into();
1971    declaration.execution_model = ExecutionModel::ProcessPerTest;
1972    declaration.limitations[0].reason =
1973        "nextest exposes no general application-action lifecycle".into();
1974    declaration
1975}
1976
1977fn custom_harness_runner_declaration() -> FrontendRunnerDeclaration {
1978    let mut declaration = runner_declaration();
1979    declaration.runner = "rust-custom-harness".into();
1980    declaration.execution_model = ExecutionModel::ProcessPerTest;
1981    declaration.limitations[0].id = "rust-custom-harness-action-linkage-unavailable".into();
1982    declaration.limitations[0].reason =
1983        "A custom Cargo harness exposes no general application-action lifecycle".into();
1984    declaration.limitations.push(FrontendLimitation {
1985        id: "rust-custom-harness-internal-tests-opaque".into(),
1986        scopes: vec![FrontendLimitationScope::Test],
1987        reason: "Cargo exposes a custom harness as one target invocation; Supercov attributes that invocation exactly without inventing internal test-case identities".into(),
1988    });
1989    declaration
1990}
1991
1992fn compiler_runner_declaration() -> FrontendRunnerDeclaration {
1993    FrontendRunnerDeclaration {
1994        runner: "rustc".into(),
1995        execution_model: ExecutionModel::ProcessPerTest,
1996        attribution: FrontendAttribution {
1997            run: AttributionPrecision::Exact,
1998            worker: AttributionPrecision::Exact,
1999            test: AttributionPrecision::Exact,
2000            retry: AttributionPrecision::Exact,
2001            phase: AttributionPrecision::Exact,
2002            action: AttributionPrecision::Unavailable,
2003            assertion: AttributionPrecision::Unavailable,
2004        },
2005        limitations: vec![
2006            FrontendLimitation {
2007                id: "rust-ctfe-action-linkage-unavailable".into(),
2008                scopes: vec![FrontendLimitationScope::Action],
2009                reason: "Compile-time evaluation is build execution, not an application action"
2010                    .into(),
2011            },
2012            FrontendLimitation {
2013                id: "rust-ctfe-assertion-linkage-unavailable".into(),
2014                scopes: vec![FrontendLimitationScope::Assertion],
2015                reason: "Compile-time execution has no user-test assertion lifecycle".into(),
2016            },
2017        ],
2018    }
2019}
2020
2021fn rustdoc_runner_declaration() -> FrontendRunnerDeclaration {
2022    FrontendRunnerDeclaration {
2023        runner: "rustdoc".into(),
2024        execution_model: ExecutionModel::ParallelContextPropagated,
2025        attribution: FrontendAttribution {
2026            run: AttributionPrecision::Exact,
2027            worker: AttributionPrecision::Exact,
2028            test: AttributionPrecision::Exact,
2029            retry: AttributionPrecision::Exact,
2030            phase: AttributionPrecision::Exact,
2031            action: AttributionPrecision::Unavailable,
2032            assertion: AttributionPrecision::Exact,
2033        },
2034        limitations: vec![FrontendLimitation {
2035            id: "rustdoc-action-linkage-unavailable".into(),
2036            scopes: vec![FrontendLimitationScope::Action],
2037            reason: "Rust doctests expose assertions but no general application-action lifecycle"
2038                .into(),
2039        }],
2040    }
2041}
2042
2043fn doctest_raw_results(
2044    run_id: &str,
2045    resolution: &RustdocOutcomeResolution,
2046    started_at_ms: i64,
2047    ended_at_ms: i64,
2048    normalized: &NormalizedRustCompilerManifest,
2049) -> Result<(Vec<RawTestResult>, Vec<RustCompilerTransportHealthRecord>), RustCompilerTestError> {
2050    let mut results = Vec::new();
2051    let mut health = Vec::new();
2052    for group in &resolution.groups {
2053        if group.transport.dropped != 0 {
2054            return Err(RustCompilerTestError::DroppedEvidence {
2055                test: format!("rustdoc:{}", group.group),
2056                dropped: group.transport.dropped,
2057            });
2058        }
2059        let worker_id = format!("rustdoc-{}", &group.invocation_id[..16]);
2060        let mut accounted_committed = 0_u64;
2061        for joined in &group.entries {
2062            let entry = &joined.catalog;
2063            let test_id = format!("rust:doctest:{}:{}:{}", group.group, entry.file, entry.line);
2064            let attempt_id = format!(
2065                "{run_id}:doctest:{}:{}",
2066                group.invocation_id, joined.catalog_index
2067            );
2068            let (status, error, started, completed) = match &joined.state {
2069                RustdocJoinedOutcomeState::Completed { outcome } => (
2070                    match outcome.status {
2071                        RustdocOutcomeStatus::Passed => "passed",
2072                        RustdocOutcomeStatus::Failed => "failed",
2073                        RustdocOutcomeStatus::Ignored => "skipped",
2074                    },
2075                    (outcome.status == RustdocOutcomeStatus::Failed).then(|| {
2076                        outcome
2077                            .message
2078                            .as_deref()
2079                            .or(outcome.reason.as_deref())
2080                            .unwrap_or("rustdoc reported a failed doctest")
2081                            .to_owned()
2082                    }),
2083                    true,
2084                    true,
2085                ),
2086                RustdocJoinedOutcomeState::UnfinishedStarted => (
2087                    "unknown",
2088                    Some("rustdoc fail-fast ended after this doctest started".into()),
2089                    true,
2090                    false,
2091                ),
2092                RustdocJoinedOutcomeState::Unstarted => (
2093                    "unknown",
2094                    Some("rustdoc fail-fast ended before this doctest started".into()),
2095                    false,
2096                    false,
2097                ),
2098                RustdocJoinedOutcomeState::FilteredOut => ("skipped", None, false, false),
2099                RustdocJoinedOutcomeState::NotRunAmbiguous => (
2100                    "unknown",
2101                    Some(
2102                        "rustdoc did not identify whether this doctest was filtered or left unstarted by fail-fast"
2103                            .into(),
2104                    ),
2105                    false,
2106                    false,
2107                ),
2108            };
2109            let mut phases = started
2110                .then(|| CoveragePhase {
2111                    id: phase_id(run_id, &attempt_id),
2112                    kind: "test".into(),
2113                    operation: format!("Rust doctest {}", entry.name),
2114                    source: Some(entry.file.clone()),
2115                    caused_by_phase_id: None,
2116                    // Pinned libtest reports duration but no wall-clock
2117                    // boundaries. The authenticated rustdoc invocation is the
2118                    // narrowest non-invented interval available; an unfinished
2119                    // test deliberately has no terminal timestamp.
2120                    started_at_ms,
2121                    ended_at_ms: completed.then_some(ended_at_ms),
2122                    status: Some(status.into()),
2123                    error: error.clone(),
2124                })
2125                .into_iter()
2126                .collect::<Vec<_>>();
2127            let (base_context, transport) =
2128                group.attributed_transport(joined).map_err(|error| {
2129                    RustCompilerTestError::Projection {
2130                        test: test_id.clone(),
2131                        reason: error.to_string(),
2132                    }
2133                })?;
2134            accounted_committed = accounted_committed
2135                .checked_add(transport.committed)
2136                .ok_or_else(|| RustCompilerTestError::Projection {
2137                    test: test_id.clone(),
2138                    reason: "rustdoc committed evidence count overflow".into(),
2139                })?;
2140            if !started && transport.committed != 0 {
2141                return Err(RustCompilerTestError::Projection {
2142                    test: test_id,
2143                    reason: "a filtered or unstarted doctest emitted runtime evidence".into(),
2144                });
2145            }
2146            let runtime = if let Some(base_phase) = phases.first() {
2147                let projection = project_rust_compiler_evidence(
2148                    base_context,
2149                    base_phase,
2150                    &transport,
2151                    normalized,
2152                )
2153                .map_err(|error| RustCompilerTestError::Projection {
2154                    test: test_id.clone(),
2155                    reason: error.to_string(),
2156                })?;
2157                if snapshot_has_evidence(&projection.background) {
2158                    return Err(RustCompilerTestError::Projection {
2159                        test: test_id.clone(),
2160                        reason: "doctest context partition retained background evidence".into(),
2161                    });
2162                }
2163                phases.extend(projection.assertion_phases);
2164                vec![projection.attributed]
2165            } else {
2166                Vec::new()
2167            };
2168            results.push(RawTestResult {
2169                test_id: Some(test_id.clone()),
2170                scope: Some(ExecutionScope {
2171                    version: 1,
2172                    run_id: run_id.into(),
2173                    worker_id: worker_id.clone(),
2174                    test_id: test_id.clone(),
2175                    test_key: test_id.clone(),
2176                    retry: 0,
2177                    attempt_id,
2178                }),
2179                test: test_id,
2180                test_file: Some(entry.file.clone()),
2181                title: Some(entry.name.clone()),
2182                retry: Some(0),
2183                status: Some(status.into()),
2184                expected_status: Some("passed".into()),
2185                flaky: false,
2186                provenance: TestProvenance {
2187                    runner: "rustdoc".into(),
2188                    kind: "doctest".into(),
2189                    project: Some(group.group.clone()),
2190                    source: "supercov-rustdoc-outcome".into(),
2191                },
2192                role: "test".into(),
2193                attribution: crate::coverage_report::ATTRIBUTION_EXACT.into(),
2194                phases,
2195                runtime,
2196                browser: Vec::new(),
2197                server: Vec::new(),
2198            });
2199        }
2200
2201        let background =
2202            group
2203                .background_transport()
2204                .map_err(|error| RustCompilerTestError::Projection {
2205                    test: format!("rustdoc:{}", group.group),
2206                    reason: error.to_string(),
2207                })?;
2208        accounted_committed = accounted_committed
2209            .checked_add(background.committed)
2210            .ok_or_else(|| RustCompilerTestError::Projection {
2211                test: format!("rustdoc:{}", group.group),
2212                reason: "rustdoc background evidence count overflow".into(),
2213            })?;
2214        if accounted_committed != group.transport.committed {
2215            return Err(RustCompilerTestError::Projection {
2216                test: format!("rustdoc:{}", group.group),
2217                reason: format!(
2218                    "rustdoc context partition accounted for {accounted_committed} of {} committed records",
2219                    group.transport.committed
2220                ),
2221            });
2222        }
2223        if background.committed != 0 {
2224            let background_id = format!("background:rustdoc:{}", group.invocation_id);
2225            let background_phase = CoveragePhase {
2226                id: phase_id(run_id, &background_id),
2227                kind: "setup".into(),
2228                operation: format!("Background while running Rust doctests for {}", group.group),
2229                source: None,
2230                caused_by_phase_id: None,
2231                started_at_ms,
2232                ended_at_ms: Some(ended_at_ms),
2233                status: Some("passed".into()),
2234                error: None,
2235            };
2236            // The rustdoc background partition may hold join-bounded
2237            // quarantined thread evidence with real contexts; flatten every
2238            // record to context zero so escaped-thread work can never become
2239            // test or phase attributed downstream.
2240            let mut flattened = background.clone();
2241            for observation in &mut flattened.observations {
2242                observation.context_id = 0;
2243            }
2244            for hit in &mut flattened.ordinal_hits {
2245                hit.context_id = 0;
2246            }
2247            flattened.phases.clear();
2248            flattened.thread_phases.clear();
2249            flattened.thread_ends.clear();
2250            flattened.test_boundaries.clear();
2251            let projection =
2252                project_rust_compiler_evidence(1, &background_phase, &flattened, normalized)
2253                    .map_err(|error| RustCompilerTestError::Projection {
2254                        test: background_id.clone(),
2255                        reason: error.to_string(),
2256                    })?;
2257            if snapshot_has_evidence(&projection.attributed)
2258                || !projection.assertion_phases.is_empty()
2259            {
2260                return Err(RustCompilerTestError::Projection {
2261                    test: background_id,
2262                    reason: "context-zero doctest evidence became test-attributed".into(),
2263                });
2264            }
2265            results.push(RawTestResult {
2266                test_id: Some(background_id.clone()),
2267                scope: Some(ExecutionScope {
2268                    version: 1,
2269                    run_id: run_id.into(),
2270                    worker_id: worker_id.clone(),
2271                    test_id: background_id.clone(),
2272                    test_key: background_id.clone(),
2273                    retry: 0,
2274                    attempt_id: format!("{run_id}:doctest:{}:background", group.invocation_id),
2275                }),
2276                test: background_id,
2277                test_file: None,
2278                title: Some(format!("Background Rust doctest work for {}", group.group)),
2279                retry: Some(0),
2280                status: Some("passed".into()),
2281                expected_status: Some("passed".into()),
2282                flaky: false,
2283                provenance: TestProvenance {
2284                    runner: "rustdoc".into(),
2285                    kind: "doctest".into(),
2286                    project: Some(group.group.clone()),
2287                    source: "supercov-rustdoc-context-zero".into(),
2288                },
2289                role: "background".into(),
2290                attribution: crate::coverage_report::ATTRIBUTION_EXACT.into(),
2291                phases: Vec::new(),
2292                runtime: vec![projection.background],
2293                browser: Vec::new(),
2294                server: Vec::new(),
2295            });
2296        }
2297        health.push(RustCompilerTransportHealthRecord {
2298            scope_id: format!("rustdoc:{}", group.invocation_id),
2299            scope_kind: "runner-invocation".into(),
2300            status: if group.transport.dropped == 0 && group.transport.incomplete == 0 {
2301                "passed".into()
2302            } else {
2303                "unknown".into()
2304            },
2305            transport: RustCompilerTransportHealth {
2306                committed: group.transport.committed,
2307                incomplete: group.transport.incomplete,
2308                dropped: group.transport.dropped,
2309                attachments: group.transport.attachments,
2310            },
2311            thread_scope_limitations: group.thread_scope_limitations().map_err(|error| {
2312                RustCompilerTestError::Projection {
2313                    test: format!("rustdoc:{}", group.group),
2314                    reason: error.to_string(),
2315                }
2316            })?,
2317        });
2318    }
2319    Ok((results, health))
2320}
2321
2322fn doctest_command_failed(resolution: &RustdocOutcomeResolution) -> bool {
2323    resolution.groups.iter().any(|group| {
2324        group.entries.iter().any(|joined| match &joined.state {
2325            RustdocJoinedOutcomeState::Completed { outcome } => {
2326                outcome.status == RustdocOutcomeStatus::Failed
2327            }
2328            RustdocJoinedOutcomeState::UnfinishedStarted | RustdocJoinedOutcomeState::Unstarted => {
2329                true
2330            }
2331            RustdocJoinedOutcomeState::FilteredOut => false,
2332            RustdocJoinedOutcomeState::NotRunAmbiguous => true,
2333        }) || group.ambiguous_unstarted_tests != 0
2334    })
2335}
2336
2337fn ctfe_raw_results(
2338    run_id: &str,
2339    units: Vec<RustCompilerCtfeUnit>,
2340    started_at_ms: i64,
2341    ended_at_ms: i64,
2342) -> Vec<RawTestResult> {
2343    units
2344        .into_iter()
2345        .enumerate()
2346        .map(|(index, unit)| {
2347            let worker_id = format!("rustc-{index:04}");
2348            let test_id = format!("rust:build:ctfe:{}:{index:04}", unit.crate_name);
2349            let attempt_id = format!("{run_id}:ctfe:{index:04}");
2350            let phase_id = phase_id(run_id, &attempt_id);
2351            let mut snapshot = unit.snapshot;
2352            for event in &mut snapshot.events {
2353                event.phase_id = Some(phase_id.clone());
2354            }
2355            RawTestResult {
2356                test_id: Some(test_id.clone()),
2357                scope: Some(ExecutionScope {
2358                    version: 1,
2359                    run_id: run_id.into(),
2360                    worker_id,
2361                    test_id: test_id.clone(),
2362                    test_key: test_id.clone(),
2363                    retry: 0,
2364                    attempt_id,
2365                }),
2366                test: test_id.clone(),
2367                test_file: None,
2368                title: Some(format!(
2369                    "Compile-time evaluation for {} ({})",
2370                    unit.crate_name, unit.identity
2371                )),
2372                retry: Some(0),
2373                status: Some("passed".into()),
2374                expected_status: Some("passed".into()),
2375                flaky: false,
2376                provenance: TestProvenance {
2377                    runner: "rustc".into(),
2378                    kind: "build".into(),
2379                    project: Some(unit.crate_name),
2380                    source: "supercov-rustc-ctfe".into(),
2381                },
2382                role: "setup".into(),
2383                attribution: crate::coverage_report::ATTRIBUTION_EXACT.into(),
2384                phases: vec![CoveragePhase {
2385                    id: phase_id,
2386                    kind: "setup".into(),
2387                    operation: "Rust constant evaluation".into(),
2388                    source: None,
2389                    caused_by_phase_id: None,
2390                    started_at_ms,
2391                    ended_at_ms: Some(ended_at_ms),
2392                    status: Some("passed".into()),
2393                    error: None,
2394                }],
2395                runtime: vec![snapshot],
2396                browser: Vec::new(),
2397                server: Vec::new(),
2398            }
2399        })
2400        .collect()
2401}
2402
2403fn status(
2404    outcome: &RustCargoRunnerAttemptOutcome,
2405    output: &SupervisedOutput,
2406) -> (&'static str, i32) {
2407    match outcome {
2408        RustCargoRunnerAttemptOutcome::Libtest {
2409            result: RustLibtestTerminalResult::Passed | RustLibtestTerminalResult::Benchmarked,
2410            ..
2411        } => ("passed", 0),
2412        RustCargoRunnerAttemptOutcome::Libtest {
2413            result: RustLibtestTerminalResult::Ignored,
2414            ..
2415        } => ("skipped", 0),
2416        RustCargoRunnerAttemptOutcome::Libtest {
2417            result: RustLibtestTerminalResult::Failed,
2418            ..
2419        } => ("failed", 101),
2420        RustCargoRunnerAttemptOutcome::Unstarted => ("unstarted", 0),
2421        RustCargoRunnerAttemptOutcome::OpaqueProcess => {
2422            let exit = output.result.exit_code();
2423            (if exit == 0 { "passed" } else { "failed" }, exit)
2424        }
2425    }
2426}
2427
2428fn raw_result(
2429    run_id: &str,
2430    task: &ProcessTask,
2431    status: &str,
2432    base_phase: CoveragePhase,
2433    projection: RustCompilerEvidenceProjection,
2434) -> (RawTestResult, RustCompilerTransportHealthRecord) {
2435    let worker_id = format!("artifact-{:04}", task.artifact_index);
2436    let attempt_id = task.runner_attempt_id.clone();
2437    let runner = match task.runner {
2438        RustCargoRunnerKind::CargoTest => "rust-libtest",
2439        RustCargoRunnerKind::CargoCustomHarness => "rust-custom-harness",
2440        RustCargoRunnerKind::Nextest => "rust-nextest",
2441    };
2442    let scope = ExecutionScope {
2443        version: 1,
2444        run_id: run_id.into(),
2445        worker_id: worker_id.clone(),
2446        test_id: task.test_id.clone(),
2447        test_key: task.test_id.clone(),
2448        retry: task.retry,
2449        attempt_id: attempt_id.clone(),
2450    };
2451    let mut phases = vec![base_phase];
2452    phases.extend(projection.assertion_phases);
2453    let result = RawTestResult {
2454        test_id: Some(task.test_id.clone()),
2455        scope: Some(scope),
2456        test: task.test_id.clone(),
2457        test_file: Some(task.artifact.source.clone()),
2458        title: Some(if task.runner == RustCargoRunnerKind::CargoCustomHarness {
2459            task.artifact.target_key.clone()
2460        } else {
2461            task.test.clone()
2462        }),
2463        retry: Some(task.retry),
2464        status: Some(status.into()),
2465        expected_status: Some("passed".into()),
2466        flaky: false,
2467        provenance: TestProvenance {
2468            runner: runner.into(),
2469            kind: task.artifact.kind.clone(),
2470            project: Some(task.artifact.package.clone()),
2471            source: match task.runner {
2472                RustCargoRunnerKind::CargoTest => "supercov-rustc-stock-libtest-context",
2473                RustCargoRunnerKind::CargoCustomHarness => "supercov-rustc-custom-harness-process",
2474                RustCargoRunnerKind::Nextest => "supercov-rustc-nextest-process",
2475            }
2476            .into(),
2477        },
2478        role: "test".into(),
2479        attribution: crate::coverage_report::ATTRIBUTION_EXACT.into(),
2480        phases,
2481        runtime: vec![projection.attributed],
2482        browser: Vec::new(),
2483        server: Vec::new(),
2484    };
2485    let health = RustCompilerTransportHealthRecord {
2486        scope_id: task.test_id.clone(),
2487        scope_kind: "test-attempt".into(),
2488        status: status.into(),
2489        transport: projection.health,
2490        thread_scope_limitations: BTreeSet::new(),
2491    };
2492    (result, health)
2493}
2494
2495fn cargo_runner_background_result(
2496    run_id: &str,
2497    artifact_index: usize,
2498    artifact: &TestArtifact,
2499    unit: &RustCargoRunnerUnit,
2500    normalized: &NormalizedRustCompilerManifest,
2501) -> Result<(Option<RawTestResult>, RustCompilerTransportHealthRecord), RustCompilerTestError> {
2502    let transport = &unit.invocation.background_transport;
2503    let background_id = format!("background:rust-runner:{:016}", unit.invocation_ordinal);
2504    let invocation_status = if unit.invocation.result.exit_code() == 0 {
2505        "passed"
2506    } else {
2507        "failed"
2508    };
2509    let base_phase = CoveragePhase {
2510        id: phase_id(run_id, &background_id),
2511        kind: "setup".into(),
2512        operation: format!("Background while running {}", artifact.target_key),
2513        source: Some(artifact.source.clone()),
2514        caused_by_phase_id: None,
2515        started_at_ms: unit.invocation.started_at_ms,
2516        ended_at_ms: Some(unit.invocation.ended_at_ms),
2517        status: Some(invocation_status.into()),
2518        error: None,
2519    };
2520    // The persisted background partition holds context-zero records plus
2521    // join-bounded quarantined thread evidence that keeps its real contexts.
2522    // Projection deliberately flattens every record to context zero under a
2523    // non-reserved synthetic base: the shared projector still validates probe
2524    // identities while escaped-thread work can never become test or phase
2525    // attributed downstream.
2526    let mut flattened = transport.clone();
2527    for observation in &mut flattened.observations {
2528        observation.context_id = 0;
2529    }
2530    for hit in &mut flattened.ordinal_hits {
2531        hit.context_id = 0;
2532    }
2533    flattened.phases.clear();
2534    flattened.thread_phases.clear();
2535    flattened.thread_ends.clear();
2536    flattened.test_boundaries.clear();
2537    let projection = project_rust_compiler_evidence(1, &base_phase, &flattened, normalized)
2538        .map_err(|error| RustCompilerTestError::Projection {
2539            test: background_id.clone(),
2540            reason: error.to_string(),
2541        })?;
2542    if snapshot_has_evidence(&projection.attributed) || !projection.assertion_phases.is_empty() {
2543        return Err(RustCompilerTestError::Projection {
2544            test: background_id,
2545            reason: "context-zero Cargo evidence became test-attributed".into(),
2546        });
2547    }
2548    let runner = match unit.runner {
2549        RustCargoRunnerKind::CargoTest => "rust-libtest",
2550        RustCargoRunnerKind::CargoCustomHarness => "rust-custom-harness",
2551        RustCargoRunnerKind::Nextest => "rust-nextest",
2552    };
2553    let result = snapshot_has_evidence(&projection.background).then(|| RawTestResult {
2554        test_id: Some(background_id.clone()),
2555        scope: Some(ExecutionScope {
2556            version: 1,
2557            run_id: run_id.into(),
2558            worker_id: format!("artifact-{artifact_index:04}"),
2559            test_id: background_id.clone(),
2560            test_key: background_id.clone(),
2561            retry: 0,
2562            attempt_id: format!("{run_id}:cargo:{:016}:background", unit.invocation_ordinal),
2563        }),
2564        test: background_id.clone(),
2565        test_file: Some(artifact.source.clone()),
2566        title: Some(format!("Background Rust work for {}", artifact.target_key)),
2567        retry: Some(0),
2568        status: Some(invocation_status.into()),
2569        expected_status: Some("passed".into()),
2570        flaky: false,
2571        provenance: TestProvenance {
2572            runner: runner.into(),
2573            kind: artifact.kind.clone(),
2574            project: Some(artifact.package.clone()),
2575            source: "supercov-rustc-context-zero".into(),
2576        },
2577        role: "background".into(),
2578        attribution: crate::coverage_report::ATTRIBUTION_EXACT.into(),
2579        phases: Vec::new(),
2580        runtime: vec![projection.background],
2581        browser: Vec::new(),
2582        server: Vec::new(),
2583    });
2584    Ok((
2585        result,
2586        RustCompilerTransportHealthRecord {
2587            scope_id: background_id,
2588            scope_kind: "runner-invocation".into(),
2589            status: invocation_status.into(),
2590            transport: projection.health,
2591            thread_scope_limitations: unit.thread_scope_limitations.clone(),
2592        },
2593    ))
2594}
2595
2596pub fn run_rust_compiler_frontend(
2597    request: &RustCompilerRunRequest,
2598    diagnostics: &mut dyn Write,
2599) -> Result<RustCompilerFrontendRun, RustCompilerTestError> {
2600    let supervisor = request
2601        .watchdog_program
2602        .as_deref()
2603        .map_or_else(ProcessSupervisor::new, ProcessSupervisor::new_crash_safe)
2604        .map_err(|error| RustCompilerTestError::Build(error.to_string()))?;
2605    let options = SupervisionOptions::from_environment()
2606        .map_err(|error| RustCompilerTestError::Build(error.to_string()))?;
2607    let build = crate::rust_compiler_orchestration::build_with_rust_compiler_companion_supervised(
2608        &request.build_request(),
2609        &supervisor,
2610        options,
2611        diagnostics,
2612    )
2613    .map_err(|error| {
2614        match error {
2615        crate::rust_compiler_orchestration::RustCompilerOrchestrationError::Interrupted {
2616            code,
2617            signal,
2618        } => RustCompilerTestError::Interrupted { code, signal },
2619        crate::rust_compiler_orchestration::RustCompilerOrchestrationError::UnverifiedExecution {
2620            code,
2621            reason,
2622        } => RustCompilerTestError::UnverifiedExecution { code, reason },
2623        error => RustCompilerTestError::Build(error.to_string()),
2624    }
2625    })?;
2626    execute_compiler_build(request, build, diagnostics)
2627}
2628
2629fn execute_compiler_build(
2630    request: &RustCompilerRunRequest,
2631    build: RustCompilerBuild,
2632    diagnostics: &mut dyn Write,
2633) -> Result<RustCompilerFrontendRun, RustCompilerTestError> {
2634    let project_root = canonicalize_simplified(&request.project_root)
2635        .map_err(|error| io_error(&request.project_root, error))?;
2636    let artifacts = normalize_artifacts(&project_root, &build.target_directory, &build.artifacts)?;
2637    let artifact_by_path = artifacts
2638        .iter()
2639        .enumerate()
2640        .map(|(index, artifact)| (artifact.executable.clone(), (index, artifact.clone())))
2641        .collect::<BTreeMap<_, _>>();
2642    let nextest_version = match build.command_kind {
2643        crate::rust_test_runner::RustCargoCommandKind::NextestRun => {
2644            Some(build.nextest_version.as_deref().ok_or_else(|| {
2645                RustCompilerTestError::Context(
2646                    "nextest execution lacks its authenticated version handshake".into(),
2647                )
2648            })?)
2649        }
2650        crate::rust_test_runner::RustCargoCommandKind::CargoTest => {
2651            if build.nextest_version.is_some() || build.nextest_catalog.is_some() {
2652                return Err(RustCompilerTestError::Context(
2653                    "standard Cargo execution carries foreign nextest preflight state".into(),
2654                ));
2655            }
2656            None
2657        }
2658    };
2659    let mut nextest_selected =
2660        BTreeMap::<(PathBuf, String), (String, usize, TestArtifact, String)>::new();
2661    let mut nextest_selected_ids = BTreeSet::new();
2662    let mut nextest_binary_by_artifact = BTreeMap::<PathBuf, String>::new();
2663    if let Some(catalog) = &build.nextest_catalog {
2664        if build.command_kind != crate::rust_test_runner::RustCargoCommandKind::NextestRun {
2665            return Err(RustCompilerTestError::Context(
2666                "a nextest catalog was attached to a non-nextest build".into(),
2667            ));
2668        }
2669        for (binary_id, suite) in &catalog.rust_suites {
2670            if suite.status != RustTestSuiteStatusSummary::LISTED && !suite.test_cases.is_empty() {
2671                return Err(RustCompilerTestError::Context(format!(
2672                    "nextest skipped suite {binary_id} contains test cases"
2673                )));
2674            }
2675            let executable = canonicalize_simplified(suite.binary.binary_path.as_std_path())
2676                .map_err(|error| io_error(suite.binary.binary_path.as_std_path(), error))?;
2677            let (artifact_index, artifact) =
2678                artifact_by_path.get(&executable).ok_or_else(|| {
2679                    RustCompilerTestError::Context(format!(
2680                        "nextest catalog contains an unknown artifact: {}",
2681                        executable.display()
2682                    ))
2683                })?;
2684            if nextest_binary_by_artifact
2685                .insert(executable.clone(), binary_id.to_string())
2686                .is_some()
2687            {
2688                return Err(RustCompilerTestError::Context(
2689                    "nextest catalog aliases two binary identities to one artifact".into(),
2690                ));
2691            }
2692            let compilation_target = match suite.binary.build_platform {
2693                BuildPlatform::Host => catalog
2694                    .rust_build_meta
2695                    .platforms
2696                    .as_ref()
2697                    .map(|platforms| platforms.host.platform.triple.as_str()),
2698                BuildPlatform::Target => {
2699                    catalog
2700                        .rust_build_meta
2701                        .platforms
2702                        .as_ref()
2703                        .and_then(|platforms| {
2704                            if platforms.targets.len() > 1 {
2705                                None
2706                            } else {
2707                                platforms
2708                                    .targets
2709                                    .first()
2710                                    .map(|target| target.platform.triple.as_str())
2711                                    .or(Some(platforms.host.platform.triple.as_str()))
2712                            }
2713                        })
2714                }
2715            }
2716            .ok_or_else(|| {
2717                RustCompilerTestError::Context(format!(
2718                    "nextest binary {binary_id} lacks one exact compilation target"
2719                ))
2720            })?
2721            .to_owned();
2722            if !request
2723                .cargo_runner_plan
2724                .targets
2725                .iter()
2726                .any(|target| target.target == compilation_target)
2727            {
2728                return Err(RustCompilerTestError::Context(format!(
2729                    "nextest binary {binary_id} uses unselected target {compilation_target}"
2730                )));
2731            }
2732            for (test, summary) in &suite.test_cases {
2733                if summary.kind.is_none() {
2734                    return Err(RustCompilerTestError::Context(format!(
2735                        "nextest catalog test {binary_id}::{test} lacks a test kind"
2736                    )));
2737                }
2738                if summary.filter_match == FilterMatch::Matches {
2739                    let test = test.to_string();
2740                    let test_id = libtest_id(&compilation_target, artifact, &test);
2741                    if !nextest_selected_ids.insert(test_id.clone()) {
2742                        return Err(RustCompilerTestError::DuplicateTest(test_id));
2743                    }
2744                    if nextest_selected
2745                        .insert(
2746                            (executable.clone(), test.clone()),
2747                            (
2748                                compilation_target.clone(),
2749                                *artifact_index,
2750                                artifact.clone(),
2751                                test,
2752                            ),
2753                        )
2754                        .is_some()
2755                    {
2756                        return Err(RustCompilerTestError::DuplicateTest(test_id));
2757                    }
2758                }
2759            }
2760        }
2761    } else if build.command_kind == crate::rust_test_runner::RustCargoCommandKind::NextestRun {
2762        return Err(RustCompilerTestError::Context(
2763            "nextest execution lacks its exact selected-test catalog".into(),
2764        ));
2765    }
2766    let mut outcomes = Vec::new();
2767    let mut identities = BTreeSet::new();
2768    let mut attempt_ids = BTreeSet::new();
2769    let mut nextest_attempted = BTreeSet::new();
2770    for unit in &build.cargo_runner_units {
2771        let (artifact_index, artifact) = artifact_by_path.get(&unit.artifact).ok_or_else(|| {
2772            RustCompilerTestError::Context(format!(
2773                "Cargo runner executed an unknown artifact: {}",
2774                unit.artifact.display()
2775            ))
2776        })?;
2777        let tests = unit
2778            .attempts
2779            .iter()
2780            .map(|attempt| attempt.test.clone())
2781            .collect::<Vec<_>>();
2782        let contexts = preflight_rust_test_contexts(tests)
2783            .map_err(|error| RustCompilerTestError::Context(error.to_string()))?;
2784        for (test_index, attempt) in unit.attempts.iter().enumerate() {
2785            if contexts[&attempt.test] != attempt.context_id {
2786                return Err(RustCompilerTestError::Context(format!(
2787                    "Cargo runner context changed for {}",
2788                    attempt.test
2789                )));
2790            }
2791            let test_id = match unit.runner {
2792                RustCargoRunnerKind::CargoCustomHarness => {
2793                    custom_harness_id(&unit.target, artifact)
2794                }
2795                RustCargoRunnerKind::CargoTest | RustCargoRunnerKind::Nextest => {
2796                    libtest_id(&unit.target, artifact, &attempt.test)
2797                }
2798            };
2799            if unit.runner == RustCargoRunnerKind::Nextest {
2800                if unit.runner_version.as_deref() != nextest_version {
2801                    return Err(RustCompilerTestError::Context(format!(
2802                        "nextest target-runner version disagrees with the authenticated outer handshake for {test_id}"
2803                    )));
2804                }
2805                let expected_binary =
2806                    nextest_binary_by_artifact
2807                        .get(&unit.artifact)
2808                        .ok_or_else(|| {
2809                            RustCompilerTestError::Context(format!(
2810                                "nextest executed an uncatalogued artifact: {}",
2811                                unit.artifact.display()
2812                            ))
2813                        })?;
2814                if unit.runner_binary_id.as_deref() != Some(expected_binary.as_str()) {
2815                    return Err(RustCompilerTestError::Context(format!(
2816                        "nextest runner binary identity disagrees with the selected-test catalog for {}",
2817                        unit.artifact.display()
2818                    )));
2819                }
2820                if !nextest_selected.contains_key(&(unit.artifact.clone(), attempt.test.clone())) {
2821                    return Err(RustCompilerTestError::Context(format!(
2822                        "nextest executed a test excluded by its machine-readable catalog: {test_id}"
2823                    )));
2824                }
2825                nextest_attempted.insert(test_id.clone());
2826            }
2827            if !identities.insert((test_id.clone(), attempt.retry)) {
2828                return Err(RustCompilerTestError::DuplicateTest(format!(
2829                    "{test_id} retry {}",
2830                    attempt.retry
2831                )));
2832            }
2833            if !attempt_ids.insert(attempt.runner_attempt_id.clone()) {
2834                return Err(RustCompilerTestError::Context(format!(
2835                    "Cargo runner attempt ID is duplicated: {}",
2836                    attempt.runner_attempt_id
2837                )));
2838            }
2839            let task = ProcessTask {
2840                ordinal: outcomes.len(),
2841                artifact_index: *artifact_index,
2842                artifact: artifact.clone(),
2843                test: attempt.test.clone(),
2844                test_id,
2845                context_id: attempt.context_id,
2846                retry: attempt.retry,
2847                total_attempts: attempt.total_attempts,
2848                runner_attempt_id: attempt.runner_attempt_id.clone(),
2849                runner: unit.runner,
2850                transport: build.compiler_output_directory.join(format!(
2851                    "cargo-runner/libtest-{:04}-{test_index:08}.json",
2852                    unit.invocation_ordinal
2853                )),
2854                libtest_events: build.compiler_output_directory.join(format!(
2855                    "cargo-runner/libtest-{:04}-{test_index:08}.events",
2856                    unit.invocation_ordinal
2857                )),
2858                test_arguments: Vec::new(),
2859                underlying_runner: None,
2860            };
2861            outcomes.push(ProcessOutcome {
2862                task,
2863                output: SupervisedOutput {
2864                    result: unit.invocation.result.clone(),
2865                    stdout: unit.invocation.stdout.clone(),
2866                    stderr: unit.invocation.stderr.clone(),
2867                },
2868                read: attempt.transport.clone(),
2869                attempt_outcome: attempt.outcome.clone(),
2870                started_at_ms: unit.invocation.started_at_ms,
2871                ended_at_ms: unit.invocation.ended_at_ms,
2872            });
2873        }
2874    }
2875    let standard_cargo_units = build
2876        .cargo_runner_units
2877        .iter()
2878        .filter(|unit| {
2879            matches!(
2880                unit.runner,
2881                RustCargoRunnerKind::CargoTest | RustCargoRunnerKind::CargoCustomHarness
2882            )
2883        })
2884        .count();
2885    if build.execution_exit_code == 0
2886        && build.run_libtests
2887        && standard_cargo_units != 0
2888        && standard_cargo_units != artifacts.len()
2889    {
2890        return Err(RustCompilerTestError::Context(format!(
2891            "Cargo completed successfully but published {} runner unit(s) for {} artifact(s)",
2892            standard_cargo_units,
2893            artifacts.len()
2894        )));
2895    }
2896    let mut nextest_attempt_groups = BTreeMap::<String, Vec<&ProcessOutcome>>::new();
2897    for outcome in &outcomes {
2898        if outcome.task.runner == RustCargoRunnerKind::Nextest {
2899            nextest_attempt_groups
2900                .entry(outcome.task.test_id.clone())
2901                .or_default()
2902                .push(outcome);
2903        }
2904    }
2905    for attempts in nextest_attempt_groups.values_mut() {
2906        attempts.sort_by_key(|outcome| outcome.task.retry);
2907    }
2908    let nextest_terminal_failure = nextest_attempt_groups.values().any(|attempts| {
2909        attempts
2910            .last()
2911            .is_some_and(|outcome| outcome.output.result.exit_code() != 0)
2912    });
2913    let nextest_flaky = nextest_attempt_groups.values().any(|attempts| {
2914        attempts
2915            .last()
2916            .is_some_and(|outcome| outcome.output.result.exit_code() == 0)
2917            && attempts
2918                .iter()
2919                .take(attempts.len().saturating_sub(1))
2920                .any(|outcome| outcome.output.result.exit_code() != 0)
2921    });
2922    let nextest_unstarted = nextest_selected
2923        .values()
2924        .filter(|(target, _, artifact, test)| {
2925            !nextest_attempted.contains(&libtest_id(target, artifact, test))
2926        })
2927        .cloned()
2928        .collect::<Vec<_>>();
2929
2930    let mut raw_results = ctfe_raw_results(
2931        &request.run_id,
2932        build.ctfe_units.clone(),
2933        build.build_started_at_ms,
2934        build.build_ended_at_ms,
2935    );
2936    let (doctest_results, mut transport_health) = doctest_raw_results(
2937        &request.run_id,
2938        &build.doctest_outcomes,
2939        build.build_started_at_ms,
2940        build.build_ended_at_ms,
2941        &build.normalized,
2942    )?;
2943    raw_results.extend(doctest_results);
2944    for unit in &build.cargo_runner_units {
2945        let (artifact_index, artifact) = artifact_by_path.get(&unit.artifact).ok_or_else(|| {
2946            RustCompilerTestError::Context(format!(
2947                "Cargo runner published background evidence for an unknown artifact: {}",
2948                unit.artifact.display()
2949            ))
2950        })?;
2951        let (background, health) = cargo_runner_background_result(
2952            &request.run_id,
2953            *artifact_index,
2954            artifact,
2955            unit,
2956            &build.normalized,
2957        )?;
2958        if let Some(background) = background {
2959            raw_results.push(background);
2960        }
2961        transport_health.push(health);
2962    }
2963    let overall_exit = build.execution_exit_code;
2964    if overall_exit != 0 {
2965        diagnostics
2966            .write_all(&build.execution_stdout)
2967            .and_then(|_| diagnostics.write_all(&build.execution_stderr))
2968            .map_err(|error| RustCompilerTestError::Io {
2969                path: build.compiler_output_directory.clone(),
2970                reason: error.to_string(),
2971            })?;
2972    }
2973    match build.command_kind {
2974        crate::rust_test_runner::RustCargoCommandKind::CargoTest => {
2975            let authenticated_failure = doctest_command_failed(&build.doctest_outcomes)
2976                || outcomes
2977                    .iter()
2978                    .any(|outcome| outcome.output.result.exit_code() != 0);
2979            if (overall_exit != 0) != authenticated_failure {
2980                return Err(RustCompilerTestError::Context(
2981                    "Cargo exit status disagrees with authenticated libtest/doctest outcomes"
2982                        .into(),
2983                ));
2984            }
2985        }
2986        crate::rust_test_runner::RustCargoCommandKind::NextestRun => match overall_exit {
2987            NextestExitCode::OK => {
2988                if nextest_terminal_failure || !nextest_unstarted.is_empty() {
2989                    return Err(RustCompilerTestError::Context(
2990                        "nextest exited successfully without terminal successful attempts for every selected test"
2991                            .into(),
2992                    ));
2993                }
2994            }
2995            NextestExitCode::TEST_RUN_FAILED => {
2996                if !nextest_terminal_failure && !nextest_flaky {
2997                    return Err(RustCompilerTestError::Context(
2998                        "nextest reported test failure without an authenticated terminal failure or flaky attempt sequence"
2999                            .into(),
3000                    ));
3001                }
3002            }
3003            NextestExitCode::NO_TESTS_RUN if nextest_selected.is_empty() => {}
3004            code => {
3005                return Err(RustCompilerTestError::UnverifiedExecution {
3006                    code,
3007                    reason: "nextest returned an infrastructure/status code that is not authenticated by test attempts"
3008                        .into(),
3009                });
3010            }
3011        },
3012    }
3013    for outcome in outcomes {
3014        let (test_status, exit) = status(&outcome.attempt_outcome, &outcome.output);
3015        if outcome.read.dropped != 0 {
3016            return Err(RustCompilerTestError::DroppedEvidence {
3017                test: outcome.task.test_id,
3018                dropped: outcome.read.dropped,
3019            });
3020        }
3021        let attempt_id = outcome.task.runner_attempt_id.clone();
3022        let base_phase = CoveragePhase {
3023            id: phase_id(&request.run_id, &attempt_id),
3024            kind: "test".into(),
3025            operation: format!(
3026                "{} {}",
3027                match outcome.task.runner {
3028                    RustCargoRunnerKind::CargoTest => "Rust libtest",
3029                    RustCargoRunnerKind::CargoCustomHarness => "Rust custom harness",
3030                    RustCargoRunnerKind::Nextest => "Rust nextest test",
3031                },
3032                if outcome.task.runner == RustCargoRunnerKind::CargoCustomHarness {
3033                    &outcome.task.artifact.target_key
3034                } else {
3035                    &outcome.task.test
3036                }
3037            ),
3038            source: Some(outcome.task.artifact.source.clone()),
3039            caused_by_phase_id: None,
3040            started_at_ms: outcome.started_at_ms,
3041            ended_at_ms: Some(outcome.ended_at_ms),
3042            status: Some(test_status.into()),
3043            error: (exit != 0).then(|| {
3044                String::from_utf8_lossy(&outcome.output.stderr)
3045                    .trim()
3046                    .to_owned()
3047            }),
3048        };
3049        let projection = project_rust_compiler_evidence(
3050            outcome.task.context_id,
3051            &base_phase,
3052            &outcome.read,
3053            &build.normalized,
3054        )
3055        .map_err(|error| RustCompilerTestError::Projection {
3056            test: outcome.task.test_id.clone(),
3057            reason: error.to_string(),
3058        })?;
3059        if snapshot_has_evidence(&projection.background) {
3060            return Err(RustCompilerTestError::Projection {
3061                test: outcome.task.test_id,
3062                reason: "a persisted test partition retained context-zero evidence".into(),
3063            });
3064        }
3065        let (result, health) = raw_result(
3066            &request.run_id,
3067            &outcome.task,
3068            test_status,
3069            base_phase,
3070            projection,
3071        );
3072        raw_results.push(result);
3073        transport_health.push(health);
3074    }
3075    for (target, _artifact_index, artifact, test) in nextest_unstarted {
3076        let test_id = libtest_id(&target, &artifact, &test);
3077        raw_results.push(RawTestResult {
3078            test_id: Some(test_id.clone()),
3079            scope: None,
3080            test: test_id,
3081            test_file: Some(artifact.source.clone()),
3082            title: Some(test),
3083            retry: None,
3084            status: Some("unstarted".into()),
3085            expected_status: Some("passed".into()),
3086            flaky: false,
3087            provenance: TestProvenance {
3088                runner: "rust-nextest".into(),
3089                kind: artifact.kind,
3090                project: Some(artifact.package),
3091                source: "nextest-selected-but-not-started".into(),
3092            },
3093            role: "test".into(),
3094            attribution: crate::coverage_report::ATTRIBUTION_EXACT.into(),
3095            phases: Vec::new(),
3096            runtime: Vec::new(),
3097            browser: Vec::new(),
3098            server: Vec::new(),
3099        });
3100    }
3101
3102    let mut structural_limitations = build
3103        .normalized
3104        .manifest
3105        .limitations
3106        .iter()
3107        .filter_map(|limitation| {
3108            limitation
3109                .get("id")
3110                .and_then(|id| id.as_str())
3111                .map(str::to_owned)
3112        })
3113        .collect::<Vec<_>>();
3114    if !build.doctest_outcomes.is_fully_catalogued() {
3115        structural_limitations.push("rust-doctest-outcome-catalog-incomplete".into());
3116    }
3117    if build.doctest_outcomes.has_ambiguous_outcomes() {
3118        structural_limitations.push("rust-doctest-filter-fail-fast-identity-ambiguous".into());
3119    }
3120    structural_limitations.sort();
3121    structural_limitations.dedup();
3122    let observed_runners = raw_results
3123        .iter()
3124        .map(|result| result.provenance.runner.as_str())
3125        .collect::<BTreeSet<_>>();
3126    if observed_runners.iter().any(|runner| {
3127        !matches!(
3128            *runner,
3129            "rustc" | "rust-libtest" | "rust-custom-harness" | "rust-nextest" | "rustdoc"
3130        )
3131    }) {
3132        return Err(RustCompilerTestError::Context(
3133            "Rust compiler run produced an unknown runner identity".into(),
3134        ));
3135    }
3136    // The frontend contract declares capabilities actually present in this
3137    // run. In particular, `cargo test --doc` must not advertise an unobserved
3138    // libtest runner, and an explicit non-doc target must not advertise
3139    // rustdoc. The shared analyzer deliberately rejects such declarations.
3140    let runners = [
3141        ("rustc", compiler_runner_declaration()),
3142        ("rust-libtest", runner_declaration()),
3143        ("rust-custom-harness", custom_harness_runner_declaration()),
3144        ("rust-nextest", nextest_runner_declaration()),
3145        ("rustdoc", rustdoc_runner_declaration()),
3146    ]
3147    .into_iter()
3148    .filter_map(|(name, declaration)| observed_runners.contains(name).then_some(declaration))
3149    .collect();
3150    let declaration = FrontendRunDeclaration {
3151        protocol_version: LANGUAGE_FRONTEND_PROTOCOL_VERSION,
3152        frontend_id: "rust".into(),
3153        frontend_version: "rust-compiler-v1".into(),
3154        language: "rust".into(),
3155        structural_source: StructuralSource::OwnedProbes,
3156        runners,
3157        structural_limitations,
3158    };
3159    Ok(RustCompilerFrontendRun {
3160        selection: build.selection,
3161        declaration,
3162        request: CoverageReportRequest {
3163            run_id: request.run_id.clone(),
3164            manifest: build.normalized.manifest,
3165            raw_results,
3166            generated_at: request.generated_at.clone(),
3167            coverage_model: Some(rust_compiler_coverage_model()),
3168            integrity: None,
3169            test_exit_code: ExitCodeInput::Present(Some(overall_exit)),
3170        },
3171        exit_code: overall_exit,
3172        artifacts: artifacts.len(),
3173        artifact_files: artifacts
3174            .into_iter()
3175            .map(|artifact| artifact.executable)
3176            .collect(),
3177        transport_health,
3178        build_ms: build.build_ms,
3179        execution_ms: build.execution_ms,
3180    })
3181}
3182
3183#[cfg(test)]
3184mod tests {
3185    use super::*;
3186    use crate::{
3187        coverage_analysis::PointKind,
3188        coverage_report::{CoverageManifest, PointMeta},
3189        rust_doctest::{
3190            RustdocDoctestAttributes, RustdocDoctestCode, RustdocDoctestIgnore,
3191            RustdocDoctestWrapper, RustdocExtractedDoctest, RustdocJoinedOutcome,
3192            RustdocMergedEntry, RustdocOutcomeGroupJoin, RustdocTestOutcome,
3193        },
3194        rust_probe_transport::RustOrdinalHit,
3195    };
3196
3197    fn test_transport() -> RustTransportRead {
3198        RustTransportRead::empty()
3199    }
3200
3201    fn test_invocation(status: i32) -> RustCargoRunnerInvocation {
3202        RustCargoRunnerInvocation {
3203            result: SupervisedResult {
3204                status: Some(status),
3205                signal: None,
3206                timed_out: false,
3207                interrupted_signal: None,
3208            },
3209            started_at_ms: 0,
3210            ended_at_ms: 1,
3211            stdout: Vec::new(),
3212            stderr: Vec::new(),
3213            background_transport: test_transport(),
3214        }
3215    }
3216
3217    fn test_runner_unit() -> RustCargoRunnerUnit {
3218        RustCargoRunnerUnit {
3219            version: RUST_CARGO_RUNNER_VERSION,
3220            run_id: "run_0123456789abcdef".into(),
3221            invocation_ordinal: 0,
3222            runner: RustCargoRunnerKind::CargoTest,
3223            runner_run_id: None,
3224            runner_version: None,
3225            runner_binary_id: None,
3226            target: "aarch64-apple-darwin".into(),
3227            artifact: PathBuf::from("target/test-artifact"),
3228            arguments: Vec::new(),
3229            invocation: test_invocation(0),
3230            attempts: Vec::new(),
3231            thread_scope_limitations: BTreeSet::new(),
3232        }
3233    }
3234
3235    #[test]
3236    fn persisted_runner_transport_rejects_cross_partition_and_count_tampering() {
3237        let mut unit = test_runner_unit();
3238        unit.attempts.push(RustCargoRunnerAttempt {
3239            test: "tests::one".into(),
3240            context_id: 7,
3241            retry: 0,
3242            total_attempts: 1,
3243            runner_attempt_id: "attempt-one".into(),
3244            outcome: RustCargoRunnerAttemptOutcome::Libtest {
3245                result: RustLibtestTerminalResult::Passed,
3246                timed_out: false,
3247            },
3248            transport: test_transport(),
3249        });
3250        validate_persisted_runner_transport(&unit).unwrap();
3251
3252        unit.attempts[0]
3253            .transport
3254            .ordinal_hits
3255            .push(RustOrdinalHit {
3256                process_id: 1,
3257                context_id: 0,
3258                ordinal: 9,
3259            });
3260        unit.attempts[0].transport.committed = 1;
3261        assert!(
3262            validate_persisted_runner_transport(&unit)
3263                .unwrap_err()
3264                .to_string()
3265                .contains("exact transport partition")
3266        );
3267
3268        unit.attempts[0].transport.ordinal_hits[0].context_id = 7;
3269        unit.attempts[0].transport.committed = 2;
3270        assert!(
3271            validate_persisted_runner_transport(&unit)
3272                .unwrap_err()
3273                .to_string()
3274                .contains("persisted attribution")
3275        );
3276    }
3277
3278    #[test]
3279    fn empty_selected_suite_publishes_invocation_background_without_a_test_attempt() {
3280        let point_id = "rs:statement:111111111111111111111111";
3281        let mut unit = test_runner_unit();
3282        unit.invocation.background_transport = RustTransportRead {
3283            ordinal_hits: vec![RustOrdinalHit {
3284                process_id: 1,
3285                context_id: 0,
3286                ordinal: 9,
3287            }],
3288            committed: 1,
3289            attachments: 1,
3290            ..RustTransportRead::empty()
3291        };
3292        validate_persisted_runner_transport(&unit).unwrap();
3293        let normalized = NormalizedRustCompilerManifest {
3294            manifest: CoverageManifest {
3295                unmeasured: Vec::new(),
3296                decisions: Vec::new(),
3297                points: vec![PointMeta {
3298                    id: point_id.into(),
3299                    kind: PointKind::Statement,
3300                    file: "src/lib.rs".into(),
3301                    line: 1,
3302                    column: 1,
3303                    source: "setup();".into(),
3304                    label: None,
3305                }],
3306                branches: Vec::new(),
3307                limitations: Vec::new(),
3308                scope: None,
3309            },
3310            hit_obligations_by_ordinal: BTreeMap::from([(9, vec![point_id.into()])]),
3311            internal_ordinals: BTreeSet::new(),
3312            decision_outcome_obligations: BTreeMap::new(),
3313            decision_loop_obligations: BTreeMap::new(),
3314            decision_logical_selection_obligations: BTreeMap::new(),
3315        };
3316        let artifact = TestArtifact {
3317            executable: unit.artifact.clone(),
3318            runner_argument: None,
3319            package: "package:.".into(),
3320            target_key: "lib:fixture".into(),
3321            kind: "unit".into(),
3322            source: "src/lib.rs".into(),
3323            test_harness: true,
3324        };
3325        let (result, health) =
3326            cargo_runner_background_result(&unit.run_id, 0, &artifact, &unit, &normalized).unwrap();
3327        let result = result.expect("context-zero evidence must remain queryable");
3328        assert_eq!(result.role, "background");
3329        assert!(result.runtime[0].hits.iter().any(|hit| hit == point_id));
3330        assert_eq!(health.scope_kind, "runner-invocation");
3331        assert_eq!(health.transport.committed, 1);
3332    }
3333
3334    #[test]
3335    fn tokens_and_phase_ids_are_fixed_width_and_domain_separated() {
3336        assert_eq!(token_hex(&[0xab; TOKEN_BYTES]), "ab".repeat(TOKEN_BYTES));
3337        let first = phase_id("run-a", "attempt");
3338        assert_eq!(first.len(), "rust-test:".len() + 40);
3339        assert_ne!(first, phase_id("run-b", "attempt"));
3340        assert_ne!(first, phase_id("run-a", "attempt-b"));
3341    }
3342
3343    #[test]
3344    fn test_identities_include_runner_package_target_and_workspace_source() {
3345        let artifact = |package: &str, target_key: &str| TestArtifact {
3346            executable: PathBuf::from("test-artifact"),
3347            runner_argument: None,
3348            package: package.into(),
3349            target_key: target_key.into(),
3350            kind: "unit".into(),
3351            source: "shared/src/lib.rs".into(),
3352            test_harness: true,
3353        };
3354        let root = artifact("package:.", "lib:same");
3355        let sibling = artifact("package:crates/sibling", "lib:same");
3356        let integration = artifact("package:.", "test:same");
3357        assert_eq!(
3358            libtest_id("aarch64-apple-darwin", &root, "tests::same_name"),
3359            "rust:libtest:aarch64-apple-darwin:package:.:lib:same:shared/src/lib.rs::tests::same_name"
3360        );
3361        assert_ne!(
3362            libtest_id("aarch64-apple-darwin", &root, "tests::same_name"),
3363            libtest_id("aarch64-apple-darwin", &sibling, "tests::same_name")
3364        );
3365        assert_ne!(
3366            libtest_id("aarch64-apple-darwin", &root, "tests::same_name"),
3367            libtest_id("aarch64-apple-darwin", &integration, "tests::same_name")
3368        );
3369        assert_ne!(
3370            libtest_id("aarch64-apple-darwin", &root, "tests::same_name"),
3371            libtest_id("x86_64-apple-darwin", &root, "tests::same_name")
3372        );
3373        assert_eq!(
3374            custom_harness_id("aarch64-apple-darwin", &integration),
3375            "rust:custom-harness:aarch64-apple-darwin:package:.:test:same:shared/src/lib.rs"
3376        );
3377        assert_ne!(
3378            custom_harness_id("aarch64-apple-darwin", &integration),
3379            libtest_id("aarch64-apple-darwin", &integration, "custom-harness")
3380        );
3381    }
3382
3383    #[test]
3384    fn cargo_runner_failure_units_are_atomic_and_diagnostic() {
3385        let root = std::env::temp_dir().join(format!(
3386            "supercov-cargo-runner-failure-{}-{}",
3387            std::process::id(),
3388            SystemTime::now()
3389                .duration_since(UNIX_EPOCH)
3390                .unwrap()
3391                .as_nanos()
3392        ));
3393        fs::create_dir(&root).unwrap();
3394        let ordinal = reserve_cargo_runner_ordinal(&root).unwrap();
3395        assert_eq!(ordinal, 0);
3396        let failure = RustCargoRunnerFailure {
3397            version: RUST_CARGO_RUNNER_VERSION,
3398            run_id: "run_0123456789abcdef".into(),
3399            invocation_ordinal: ordinal,
3400            target: Some("aarch64-apple-darwin".into()),
3401            artifact: Some(PathBuf::from("target/test-artifact")),
3402            error: "deliberate runner failure".into(),
3403        };
3404        let published = write_cargo_runner_failure(&root, &failure).unwrap();
3405        assert_eq!(
3406            published.file_name().unwrap(),
3407            "failure-0000000000000000.json"
3408        );
3409        assert!(fs::read_dir(&root).unwrap().all(|entry| {
3410            !entry
3411                .unwrap()
3412                .file_name()
3413                .to_string_lossy()
3414                .ends_with(".partial")
3415        }));
3416        let error = read_cargo_runner_units(
3417            &root,
3418            "run_0123456789abcdef",
3419            &["aarch64-apple-darwin".into()],
3420        )
3421        .unwrap_err();
3422        let error = error.to_string();
3423        assert!(error.contains("invocation 0 failed"), "{error}");
3424        assert!(error.contains("deliberate runner failure"), "{error}");
3425        fs::remove_dir_all(root).unwrap();
3426    }
3427
3428    #[test]
3429    fn cargo_runner_process_death_is_distinct_from_an_internal_failure() {
3430        let root = std::env::temp_dir().join(format!(
3431            "supercov-cargo-runner-death-{}-{}",
3432            std::process::id(),
3433            SystemTime::now()
3434                .duration_since(UNIX_EPOCH)
3435                .unwrap()
3436                .as_nanos()
3437        ));
3438        fs::create_dir(&root).unwrap();
3439        reserve_cargo_runner_ordinal(&root).unwrap();
3440        let error = read_cargo_runner_units(
3441            &root,
3442            "run_0123456789abcdef",
3443            &["aarch64-apple-darwin".into()],
3444        )
3445        .unwrap_err()
3446        .to_string();
3447        assert!(error.contains("without publishing its unit"), "{error}");
3448        fs::remove_dir_all(root).unwrap();
3449    }
3450
3451    #[test]
3452    fn cargo_runner_units_are_bound_to_the_selected_target_set() {
3453        let root = std::env::temp_dir().join(format!(
3454            "supercov-cargo-runner-target-{}-{}",
3455            std::process::id(),
3456            SystemTime::now()
3457                .duration_since(UNIX_EPOCH)
3458                .unwrap()
3459                .as_nanos()
3460        ));
3461        fs::create_dir(&root).unwrap();
3462        let ordinal = reserve_cargo_runner_ordinal(&root).unwrap();
3463        write_cargo_runner_unit(
3464            &root,
3465            &RustCargoRunnerUnit {
3466                version: RUST_CARGO_RUNNER_VERSION,
3467                run_id: "run_0123456789abcdef".into(),
3468                invocation_ordinal: ordinal,
3469                runner: RustCargoRunnerKind::CargoTest,
3470                runner_run_id: None,
3471                runner_version: None,
3472                runner_binary_id: None,
3473                target: "aarch64-apple-darwin".into(),
3474                artifact: PathBuf::from("target/test-artifact"),
3475                arguments: Vec::new(),
3476                invocation: test_invocation(0),
3477                attempts: Vec::new(),
3478                thread_scope_limitations: BTreeSet::new(),
3479            },
3480        )
3481        .unwrap();
3482        let second_ordinal = reserve_cargo_runner_ordinal(&root).unwrap();
3483        write_cargo_runner_unit(
3484            &root,
3485            &RustCargoRunnerUnit {
3486                version: RUST_CARGO_RUNNER_VERSION,
3487                run_id: "run_0123456789abcdef".into(),
3488                invocation_ordinal: second_ordinal,
3489                runner: RustCargoRunnerKind::CargoTest,
3490                runner_run_id: None,
3491                runner_version: None,
3492                runner_binary_id: None,
3493                target: "x86_64-unknown-linux-gnu".into(),
3494                artifact: PathBuf::from("target/test-artifact"),
3495                arguments: Vec::new(),
3496                invocation: test_invocation(0),
3497                attempts: Vec::new(),
3498                thread_scope_limitations: BTreeSet::new(),
3499            },
3500        )
3501        .unwrap();
3502        assert_eq!(
3503            read_cargo_runner_units(
3504                &root,
3505                "run_0123456789abcdef",
3506                &[
3507                    "aarch64-apple-darwin".into(),
3508                    "x86_64-unknown-linux-gnu".into(),
3509                ],
3510            )
3511            .unwrap()
3512            .len(),
3513            2
3514        );
3515        let error = read_cargo_runner_units(
3516            &root,
3517            "run_0123456789abcdef",
3518            &["x86_64-unknown-linux-gnu".into()],
3519        )
3520        .unwrap_err()
3521        .to_string();
3522        assert!(error.contains("unselected target identity"), "{error}");
3523        fs::remove_dir_all(root).unwrap();
3524    }
3525
3526    #[test]
3527    fn nextest_retry_units_preserve_each_exact_attempt_and_reject_gaps() {
3528        let root = std::env::temp_dir().join(format!(
3529            "supercov-nextest-retries-{}-{}",
3530            std::process::id(),
3531            SystemTime::now()
3532                .duration_since(UNIX_EPOCH)
3533                .unwrap()
3534                .as_nanos()
3535        ));
3536        fs::create_dir(&root).unwrap();
3537        let unit = |ordinal: u64, retry: usize, status: i32| RustCargoRunnerUnit {
3538            version: RUST_CARGO_RUNNER_VERSION,
3539            run_id: "run_0123456789abcdef".into(),
3540            invocation_ordinal: ordinal,
3541            runner: RustCargoRunnerKind::Nextest,
3542            runner_run_id: Some("2ae19189-240a-433a-a31d-acc411fe8e1f".into()),
3543            runner_version: Some("0.9.140".into()),
3544            runner_binary_id: Some("fixture".into()),
3545            target: "aarch64-apple-darwin".into(),
3546            artifact: PathBuf::from("target/test-artifact"),
3547            arguments: vec!["--exact".into(), "tests::flaky".into()],
3548            invocation: RustCargoRunnerInvocation {
3549                started_at_ms: retry as i64,
3550                ended_at_ms: retry as i64 + 1,
3551                ..test_invocation(status)
3552            },
3553            attempts: vec![RustCargoRunnerAttempt {
3554                test: "tests::flaky".into(),
3555                context_id: 7,
3556                retry,
3557                total_attempts: 2,
3558                runner_attempt_id: format!(
3559                    "2ae19189-240a-433a-a31d-acc411fe8e1f:fixture$tests::flaky{}",
3560                    if retry == 0 {
3561                        String::new()
3562                    } else {
3563                        format!("#{}", retry + 1)
3564                    }
3565                ),
3566                outcome: RustCargoRunnerAttemptOutcome::Libtest {
3567                    result: if status == 0 {
3568                        RustLibtestTerminalResult::Passed
3569                    } else {
3570                        RustLibtestTerminalResult::Failed
3571                    },
3572                    timed_out: false,
3573                },
3574                transport: test_transport(),
3575            }],
3576            thread_scope_limitations: BTreeSet::new(),
3577        };
3578
3579        let first = reserve_cargo_runner_ordinal(&root).unwrap();
3580        write_cargo_runner_unit(&root, &unit(first, 0, 101)).unwrap();
3581        let second = reserve_cargo_runner_ordinal(&root).unwrap();
3582        write_cargo_runner_unit(&root, &unit(second, 1, 0)).unwrap();
3583        let read = read_cargo_runner_units(
3584            &root,
3585            "run_0123456789abcdef",
3586            &["aarch64-apple-darwin".into()],
3587        )
3588        .unwrap();
3589        assert_eq!(read.len(), 2);
3590        assert_eq!(read[0].attempts[0].retry, 0);
3591        assert_eq!(read[1].attempts[0].retry, 1);
3592        fs::remove_dir_all(&root).unwrap();
3593
3594        let gap_root = root.with_extension("gap");
3595        fs::create_dir(&gap_root).unwrap();
3596        let first = reserve_cargo_runner_ordinal(&gap_root).unwrap();
3597        write_cargo_runner_unit(&gap_root, &unit(first, 1, 0)).unwrap();
3598        let error = read_cargo_runner_units(
3599            &gap_root,
3600            "run_0123456789abcdef",
3601            &["aarch64-apple-darwin".into()],
3602        )
3603        .unwrap_err()
3604        .to_string();
3605        assert!(error.contains("retry sequence"), "{error}");
3606        fs::remove_dir_all(gap_root).unwrap();
3607    }
3608
3609    #[cfg(unix)]
3610    #[test]
3611    fn nextest_list_passthrough_preserves_output_and_publishes_no_unit() {
3612        let artifact = TestArtifact {
3613            executable: PathBuf::from("/bin/echo"),
3614            runner_argument: None,
3615            package: "fixture".into(),
3616            target_key: "lib:fixture".into(),
3617            kind: "unit".into(),
3618            source: "src/lib.rs".into(),
3619            test_harness: true,
3620        };
3621        let mut stdout = Vec::new();
3622        let mut stderr = Vec::new();
3623        let execution = run_nextest_list_passthrough(
3624            Path::new("/tmp"),
3625            &artifact,
3626            None,
3627            vec![OsString::from("--list"), OsString::from("--format=terse")],
3628            None,
3629            &mut stdout,
3630            &mut stderr,
3631        )
3632        .unwrap();
3633        assert_eq!(execution.exit_code, 0);
3634        assert!(execution.unit_path.is_none());
3635        assert_eq!(stdout, b"--list --format=terse\n");
3636        assert!(stderr.is_empty());
3637    }
3638
3639    #[test]
3640    fn compiler_runner_declares_exact_assertion_but_not_action_causality() {
3641        let runner = runner_declaration();
3642        assert_eq!(runner.attribution.assertion, AttributionPrecision::Exact);
3643        assert_eq!(runner.attribution.action, AttributionPrecision::Unavailable);
3644        assert_eq!(runner.limitations.len(), 1);
3645        let compiler = compiler_runner_declaration();
3646        assert_eq!(compiler.attribution.phase, AttributionPrecision::Exact);
3647        assert_eq!(
3648            compiler.attribution.assertion,
3649            AttributionPrecision::Unavailable
3650        );
3651        let rustdoc = rustdoc_runner_declaration();
3652        assert_eq!(
3653            rustdoc.execution_model,
3654            ExecutionModel::ParallelContextPropagated
3655        );
3656        assert_eq!(rustdoc.attribution.test, AttributionPrecision::Exact);
3657        assert_eq!(rustdoc.attribution.assertion, AttributionPrecision::Exact);
3658    }
3659
3660    #[test]
3661    fn doctest_outcomes_project_exact_status_identity_and_fail_fast_state() {
3662        let entry = |module: &str, line: u64| RustdocMergedEntry {
3663            module: module.into(),
3664            display_name: format!("src/lib.rs - (line {line})"),
3665            path: "src/lib.rs".into(),
3666            line,
3667            ignored: false,
3668            no_run: false,
3669            should_panic: false,
3670        };
3671        let catalog = |line: u64| RustdocExtractedDoctest {
3672            file: "src/lib.rs".into(),
3673            line,
3674            doctest_attributes: RustdocDoctestAttributes {
3675                original: String::new(),
3676                should_panic: false,
3677                no_run: false,
3678                ignore: RustdocDoctestIgnore::None,
3679                rust: true,
3680                test_harness: false,
3681                compile_fail: false,
3682                standalone_crate: false,
3683                error_codes: Vec::new(),
3684                edition: None,
3685                added_css_classes: Vec::new(),
3686                unknown: Vec::new(),
3687            },
3688            original_code: "assert!(true);".into(),
3689            doctest_code: Some(RustdocDoctestCode {
3690                crate_level: String::new(),
3691                code: "assert!(true);".into(),
3692                wrapper: Some(RustdocDoctestWrapper {
3693                    before: "fn main() {".into(),
3694                    after: "}".into(),
3695                    returns_result: false,
3696                }),
3697            }),
3698            name: format!("src/lib.rs - (line {line})"),
3699        };
3700        let completed = |catalog_index, entry: RustdocMergedEntry, status| RustdocJoinedOutcome {
3701            catalog_index,
3702            catalog: catalog(entry.line),
3703            merged_entry: Some(entry.clone()),
3704            state: RustdocJoinedOutcomeState::Completed {
3705                outcome: RustdocTestOutcome {
3706                    display_name: entry.display_name,
3707                    status,
3708                    execution_seconds: Some(0.1),
3709                    stdout: None,
3710                    message: None,
3711                    reason: None,
3712                    timeout_warning: false,
3713                },
3714            },
3715        };
3716        let resolution = RustdocOutcomeResolution {
3717            groups: vec![RustdocOutcomeGroupJoin {
3718                invocation_id: "1".repeat(64),
3719                group: "fixture".into(),
3720                companion_build_id: "2".repeat(64),
3721                raw_catalog_sha256: "4".repeat(64),
3722                raw_events_sha256: "3".repeat(64),
3723                transport_sha256: "5".repeat(64),
3724                join: None,
3725                transport: RustTransportRead::empty(),
3726                entries: vec![
3727                    completed(0, entry("__doctest_0", 3), RustdocOutcomeStatus::Passed),
3728                    RustdocJoinedOutcome {
3729                        catalog_index: 1,
3730                        catalog: catalog(10),
3731                        merged_entry: Some(entry("__doctest_1", 10)),
3732                        state: RustdocJoinedOutcomeState::Unstarted,
3733                    },
3734                ],
3735                ambiguous_filtered_out: 0,
3736                ambiguous_unstarted_tests: 0,
3737            }],
3738            unmatched_maps: Vec::new(),
3739        };
3740        let normalized = NormalizedRustCompilerManifest {
3741            manifest: crate::coverage_report::CoverageManifest {
3742                unmeasured: Vec::new(),
3743                decisions: Vec::new(),
3744                points: Vec::new(),
3745                branches: Vec::new(),
3746                limitations: Vec::new(),
3747                scope: None,
3748            },
3749            hit_obligations_by_ordinal: std::collections::BTreeMap::new(),
3750            internal_ordinals: BTreeSet::new(),
3751            decision_outcome_obligations: std::collections::BTreeMap::new(),
3752            decision_loop_obligations: std::collections::BTreeMap::new(),
3753            decision_logical_selection_obligations: std::collections::BTreeMap::new(),
3754        };
3755        let (results, health) =
3756            doctest_raw_results("run", &resolution, 10, 20, &normalized).unwrap();
3757        assert_eq!(results.len(), 2);
3758        assert_eq!(health.len(), 1);
3759        assert_eq!(results[0].status.as_deref(), Some("passed"));
3760        assert_eq!(results[1].status.as_deref(), Some("unknown"));
3761        assert_eq!(results[0].retry, Some(0));
3762        assert_eq!(results[0].test_file.as_deref(), Some("src/lib.rs"));
3763        assert_eq!(
3764            results[0].test_id.as_deref(),
3765            Some("rust:doctest:fixture:src/lib.rs:3")
3766        );
3767        assert_eq!(results[0].scope.as_ref().unwrap().test_id, results[0].test);
3768        assert!(doctest_command_failed(&resolution));
3769        assert!(results[1].phases.is_empty());
3770        assert!(resolution.is_fully_catalogued());
3771    }
3772}