Skip to main content

shipper_core/engine/
mod.rs

1use std::collections::BTreeMap;
2use std::path::Path;
3use std::thread;
4use std::time::{Duration, Instant};
5
6use anyhow::{Context, Result, bail};
7use chrono::Utc;
8
9use crate::cargo;
10use crate::plan::PlannedWorkspace;
11use crate::registry::RegistryClient;
12#[cfg(test)]
13use crate::runtime::environment;
14#[cfg(test)]
15use crate::runtime::execution::short_state;
16use crate::runtime::execution::{
17    backoff_delay, classify_cargo_failure, pkg_key, record_attempt_detail, registry_aware_backoff,
18    resolve_state_dir, retry_after_delay, retry_next_attempt_at, update_state,
19};
20use crate::state::events;
21use crate::state::execution_state as state;
22#[cfg(test)]
23use crate::types::ExecutionResult;
24use crate::types::{
25    AttemptDetail, AttemptEvidence, ErrorClass, EventType, ExecutionState, PackageProgress,
26    PackageReceipt, PackageState, PreflightReport, PublishEvent, PublishRegime, ReadinessEvidence,
27    Receipt, ReconciliationOutcome, Registry, RuntimeOptions,
28};
29#[cfg(test)]
30use crate::types::{Finishability, PreflightPackage};
31use crate::webhook::{self, WebhookEvent};
32
33mod preflight;
34mod publish;
35
36pub use preflight::PreflightRunOptions;
37
38pub trait Reporter {
39    fn info(&mut self, msg: &str);
40    fn warn(&mut self, msg: &str);
41    fn error(&mut self, msg: &str);
42
43    /// Narrate a retry-backoff wait to the operator and block until the wait
44    /// has elapsed. Called once per retry backoff site after the
45    /// [`EventType::RetryBackoffStarted`] event has been recorded. The default
46    /// implementation preserves the pre-#103 behavior (a single `warn()` line
47    /// then `thread::sleep(delay)`); richer UIs (e.g., the CLI) override it to
48    /// render a live countdown during the wait window. Overrides MUST block
49    /// for the full `delay` before returning — the engine relies on this
50    /// method to own the retry sleep. See #103 and #91.
51    #[allow(clippy::too_many_arguments)]
52    fn retry_wait(
53        &mut self,
54        pkg_name: &str,
55        pkg_version: &str,
56        attempt: u32,
57        max_attempts: u32,
58        delay: Duration,
59        reason: ErrorClass,
60        message: &str,
61    ) {
62        self.warn(&format!(
63            "{}@{}: {} ({:?}); next attempt in {} (attempt {}/{})",
64            pkg_name,
65            pkg_version,
66            message,
67            reason,
68            humantime::format_duration(delay),
69            attempt.saturating_add(1),
70            max_attempts,
71        ));
72        thread::sleep(delay);
73    }
74}
75
76pub(crate) fn policy_effects(opts: &RuntimeOptions) -> crate::runtime::policy::PolicyEffects {
77    crate::runtime::policy::policy_effects(opts)
78}
79
80fn write_reconciliation_report_best_effort(
81    state_dir: &Path,
82    ws: &PlannedWorkspace,
83    events_path: &Path,
84    reporter: &mut dyn Reporter,
85) {
86    if let Err(err) = crate::state::reconciliation::write_report_from_events(
87        state_dir,
88        &ws.plan.plan_id,
89        &ws.plan.registry,
90        events_path,
91    ) {
92        reporter.warn(&format!("failed to write reconciliation report: {err}"));
93    }
94}
95
96fn init_registry_client(registry: Registry, state_dir: &Path) -> Result<RegistryClient> {
97    let cache_dir = state_dir.join("cache");
98    RegistryClient::new(registry).map(|c| c.with_cache_dir(cache_dir))
99}
100
101/// Run preflight verification checks before publishing.
102///
103/// This is the backward-compatible read-only entry point for embedders.
104/// Call [`run_preflight_in_place`] if you want preflight to stamp the detected
105/// [`PublishRegime`] onto each `PlannedPackage` for a subsequent publish in
106/// the same process.
107///
108/// This function performs various pre-publish checks to catch issues early:
109/// - Git cleanliness (if `allow_dirty` is false)
110/// - Registry reachability
111/// - Dry-run compilation verification
112/// - Version existence check (skip already-published versions)
113/// - Ownership verification (optional, based on policy)
114///
115/// # Arguments
116///
117/// * `ws` - The planned workspace containing packages to publish
118/// * `opts` - Runtime options controlling behavior
119/// * `reporter` - A reporter for outputting progress and warnings
120///
121/// # Returns
122///
123/// Returns a [`PreflightReport`] containing:
124/// - Whether a token was detected
125/// - The finishability assessment (Proven/NotProven/Failed)
126/// - Per-package preflight results
127///
128/// # Example
129///
130/// ```ignore
131/// let ws = plan::build_plan(&spec)?;
132/// let opts = types::RuntimeOptions { /* ... */ };
133/// let mut reporter = MyReporter::default();
134/// let report = engine::run_preflight(&ws, &opts, &mut reporter)?;
135/// println!("Finishability: {:?}", report.finishability);
136/// ```
137/// Back-compat entry point preserving the historical preflight shape
138/// (appends to `events.jsonl`). Equivalent to
139/// `run_preflight_with_options(ws, opts, reporter, Default::default())`.
140pub fn run_preflight(
141    ws: &PlannedWorkspace,
142    opts: &RuntimeOptions,
143    reporter: &mut dyn Reporter,
144) -> Result<PreflightReport> {
145    let mut ws = ws.clone();
146    run_preflight_in_place_with_options(&mut ws, opts, reporter, PreflightRunOptions::default())
147}
148
149/// Run preflight checks for a planned workspace and stamp regime metadata.
150///
151/// Takes `&mut PlannedWorkspace` so preflight can stamp the detected
152/// [`PublishRegime`] onto each `PlannedPackage` once it has queried the
153/// registry (#106 PR 1). The mutation is additive: the new `regime`
154/// field defaults to `None` and is skipped in serialization when unset,
155/// so older readers of `state.json` / plan files stay compatible.
156pub fn run_preflight_in_place(
157    ws: &mut PlannedWorkspace,
158    opts: &RuntimeOptions,
159    reporter: &mut dyn Reporter,
160) -> Result<PreflightReport> {
161    run_preflight_in_place_with_options(ws, opts, reporter, PreflightRunOptions::default())
162}
163
164/// Run preflight with caller-chosen [`PreflightRunOptions`] (#100).
165///
166/// See [`run_preflight`] for the full pipeline description; this entry
167/// point additionally honors the `fresh_audit` flag, which redirects
168/// event writes to a session-scoped sidecar (see
169/// [`PreflightRunOptions::fresh_audit`] for details).
170pub fn run_preflight_with_options(
171    ws: &PlannedWorkspace,
172    opts: &RuntimeOptions,
173    reporter: &mut dyn Reporter,
174    run_opts: PreflightRunOptions,
175) -> Result<PreflightReport> {
176    let mut ws = ws.clone();
177    run_preflight_in_place_with_options(&mut ws, opts, reporter, run_opts)
178}
179
180/// Run preflight checks for a planned workspace and stamp regime metadata,
181/// with caller-chosen [`PreflightRunOptions`] (#100 + #106).
182pub fn run_preflight_in_place_with_options(
183    ws: &mut PlannedWorkspace,
184    opts: &RuntimeOptions,
185    reporter: &mut dyn Reporter,
186    run_opts: PreflightRunOptions,
187) -> Result<PreflightReport> {
188    preflight::run(ws, opts, reporter, run_opts)
189}
190
191/// Enforce the rehearsal hard gate (#97 PR 3).
192///
193/// Rules, evaluated in order:
194///
195/// 1. **Rehearsal not configured** (`opts.rehearsal_registry` is `None`) →
196///    gate is dormant; publish proceeds. Rehearsal is opt-in; existing
197///    workflows that never set a rehearsal registry are unaffected.
198///
199/// 2. **Operator override** (`opts.rehearsal_skip` is `true`) → publish
200///    proceeds with a loud warning logged to the reporter. Use sparingly
201///    (incident response, bootstrap runs). The skip decision is
202///    operator-visible in stderr; it does *not* synthesize a passing
203///    rehearsal receipt, so the audit trail still shows "no rehearsal
204///    ran."
205///
206/// 3. **No rehearsal receipt** (`rehearsal.json` is missing) → refuse.
207///    The operator needs to run `shipper rehearse` first.
208///
209/// 4. **Stale receipt** (receipt exists but `plan_id` mismatches the
210///    current workspace's plan) → refuse. A workspace change between
211///    rehearse and publish invalidates the rehearsal.
212///
213/// 5. **Failing receipt** (`passed: false`) → refuse.
214///
215/// 6. **Fresh passing receipt for current plan** → publish proceeds.
216fn enforce_rehearsal_gate(
217    ws: &PlannedWorkspace,
218    opts: &RuntimeOptions,
219    state_dir: &Path,
220    reporter: &mut dyn Reporter,
221) -> Result<()> {
222    let Some(rehearsal_name) = opts.rehearsal_registry.as_deref() else {
223        return Ok(());
224    };
225
226    if opts.rehearsal_skip {
227        reporter.warn(&format!(
228            "--skip-rehearsal was set; publish is proceeding without a rehearsal against '{rehearsal_name}'. \
229             This is an operator-authorized bypass; auditors reading events.jsonl will see no RehearsalComplete event for this plan_id."
230        ));
231        return Ok(());
232    }
233
234    let receipt = crate::state::rehearsal::load_rehearsal(state_dir)
235        .context("failed to read rehearsal receipt while enforcing hard gate")?;
236
237    let rehearsal_path = crate::state::rehearsal::rehearsal_path(state_dir);
238
239    let receipt = match receipt {
240        Some(r) => r,
241        None => bail!(
242            "rehearsal is required (rehearsal registry '{rehearsal_name}' is configured) but no rehearsal receipt was found at {}. \
243             Run `shipper rehearse --rehearsal-registry {rehearsal_name}` first, \
244             or pass --skip-rehearsal to override (not recommended).",
245            rehearsal_path.display()
246        ),
247    };
248
249    if receipt.plan_id != ws.plan.plan_id {
250        bail!(
251            "rehearsal receipt is stale: rehearsal ran for plan_id {} but the current plan_id is {}. \
252             The workspace changed between rehearse and publish; re-run `shipper rehearse` against the current plan.",
253            receipt.plan_id,
254            ws.plan.plan_id
255        );
256    }
257
258    if !receipt.passed {
259        bail!(
260            "rehearsal against '{}' did NOT pass for plan_id {}: {}. \
261             Fix the cause and re-run `shipper rehearse` before publishing.",
262            receipt.registry,
263            receipt.plan_id,
264            receipt.summary
265        );
266    }
267
268    reporter.info(&format!(
269        "rehearsal gate: passing receipt found ({} packages against '{}', plan_id {})",
270        receipt.packages_published, receipt.registry, receipt.plan_id
271    ));
272    Ok(())
273}
274
275/// Execute the publish operation for all packages in the workspace.
276///
277/// This is the main publishing function that:
278/// 1. Acquires a distributed lock to prevent concurrent publishes
279/// 2. Checks git cleanliness (if configured)
280/// 3. Initializes or resumes from existing state
281/// 4. Publishes each package in dependency order
282/// 5. Verifies visibility on the registry after each publish
283/// 6. Writes a receipt with full evidence upon completion
284///
285/// # Arguments
286///
287/// * `ws` - The planned workspace containing packages to publish
288/// * `opts` - Runtime options controlling retry, readiness, policy, etc.
289/// * `reporter` - A reporter for outputting progress and warnings
290///
291/// # Returns
292///
293/// Returns a [`Receipt`] containing:
294/// - The plan ID and registry
295/// - Start and finish timestamps
296/// - Per-package receipts with evidence
297/// - Git context and environment fingerprint
298/// - Path to the event log
299///
300/// # Behavior
301///
302/// - **Resumability**: If interrupted, the state is persisted and `run_resume` can continue
303/// - **Parallel publishing**: If `opts.parallel.enabled` is true, uses parallel publishing
304/// - **Readiness checks**: Verifies crate visibility after publishing (configurable)
305/// - **Retry logic**: Retries transient failures with exponential backoff
306///
307/// # Error Handling
308///
309/// Returns an error if:
310/// - Lock acquisition fails
311/// - Git check fails (when required)
312/// - A permanent error occurs (e.g., authentication failure)
313/// - All retry attempts are exhausted
314pub fn run_publish(
315    ws: &PlannedWorkspace,
316    opts: &RuntimeOptions,
317    reporter: &mut dyn Reporter,
318) -> Result<Receipt> {
319    let workspace_root = &ws.workspace_root;
320    publish::bootstrap::validate_resume_target(ws, opts)?;
321    let effects = policy_effects(opts);
322
323    let publish::bootstrap::PublishBootstrap {
324        state_dir,
325        _lock,
326        git_context,
327        environment,
328        auth_evidence,
329        registry: reg,
330        events_path,
331        mut event_log,
332        state: mut st,
333        run_started,
334    } = publish::bootstrap::prepare_publish_run(ws, opts, reporter)?;
335
336    let mut receipts: Vec<PackageReceipt> = Vec::new();
337
338    // Track if we've reached the resume point if one was specified
339    let mut reached_resume_point = opts.resume_from.is_none();
340
341    // Check for parallel mode
342    if opts.parallel.enabled {
343        let parallel_receipts = crate::engine::parallel::run_publish_parallel(
344            ws, opts, &mut st, &state_dir, &reg, reporter,
345        )?;
346
347        publish::finalize::record_consistency_drift(&events_path, &st, &mut event_log, reporter);
348        return publish::finalize::finish_parallel_run(
349            ws,
350            opts,
351            &state_dir,
352            &events_path,
353            &mut event_log,
354            &st,
355            parallel_receipts,
356            run_started,
357            git_context,
358            environment,
359            auth_evidence,
360        );
361    }
362
363    for p in &ws.plan.packages {
364        let key = pkg_key(&p.name, &p.version);
365        let pkg_label = format!("{}@{}", p.name, p.version);
366        let progress = st
367            .packages
368            .get(&key)
369            .context("missing package progress in state")?
370            .clone();
371
372        if matches!(
373            publish::resume::apply_resume_from_gate(
374                p,
375                &progress,
376                opts,
377                &mut reached_resume_point,
378                reporter,
379            ),
380            publish::resume::ResumeGate::Skip
381        ) {
382            if matches!(
383                progress.state,
384                PackageState::Published | PackageState::Skipped { .. }
385            ) {
386                publish::resume::record_terminal_resume_skip_event(
387                    &progress,
388                    &pkg_label,
389                    &events_path,
390                    &mut event_log,
391                )?;
392            }
393            continue;
394        }
395
396        // Track whether cargo publish already succeeded (e.g. from Uploaded state on resume)
397        let mut cargo_succeeded = false;
398
399        match progress.state.clone() {
400            PackageState::Published | PackageState::Skipped { .. } => {
401                publish::resume::record_terminal_resume_skip(
402                    p,
403                    &progress,
404                    &pkg_label,
405                    &events_path,
406                    &mut event_log,
407                    reporter,
408                )?;
409                continue;
410            }
411            PackageState::Uploaded => {
412                reporter.info(&format!(
413                    "{}@{}: resuming from uploaded (skipping cargo publish)",
414                    p.name, p.version
415                ));
416                cargo_succeeded = true;
417            }
418            PackageState::Ambiguous {
419                message: prior_reason,
420            } => {
421                // Resume-path reconciliation (#99 follow-on). A prior run left
422                // this package in Ambiguous state (reconciliation inconclusive).
423                // Before doing ANY further work, reconcile against the
424                // registry so we never re-upload a crate whose prior upload
425                // may have actually succeeded.
426                reporter.warn(&format!(
427                    "{}@{}: resume found ambiguous state ({}); reconciling against registry",
428                    p.name, p.version, prior_reason
429                ));
430
431                let readiness_config = crate::types::ReadinessConfig {
432                    enabled: effects.readiness_enabled,
433                    ..opts.readiness.clone()
434                };
435
436                event_log.record(PublishEvent {
437                    timestamp: Utc::now(),
438                    event_type: EventType::PublishReconciling {
439                        method: readiness_config.method,
440                    },
441                    package: pkg_label.clone(),
442                });
443
444                let (outcome, _evidence) =
445                    sequential_reconcile(&reg, &p.name, &p.version, &readiness_config);
446
447                event_log.record(PublishEvent {
448                    timestamp: Utc::now(),
449                    event_type: EventType::PublishReconciled {
450                        outcome: outcome.clone(),
451                    },
452                    package: pkg_label.clone(),
453                });
454                event_log.write_to_file(&events_path)?;
455                event_log.clear();
456                write_reconciliation_report_best_effort(&state_dir, ws, &events_path, reporter);
457                let reconciliation_report_path = state::reconciliation_path(&state_dir);
458
459                match outcome {
460                    ReconciliationOutcome::Published { .. } => {
461                        update_state(&mut st, &state_dir, &key, PackageState::Published)?;
462                        reporter.info(&format!(
463                            "{}@{}: reconciliation outcome: Published; action: mark published and continue without republish (evidence: {})",
464                            p.name,
465                            p.version,
466                            reconciliation_report_path.display()
467                        ));
468                        continue;
469                    }
470                    ReconciliationOutcome::NotPublished { .. } => {
471                        // Clear the Ambiguous state — the registry confirms
472                        // no prior upload succeeded, so falling through to
473                        // the normal publish flow is safe.
474                        update_state(&mut st, &state_dir, &key, PackageState::Pending)?;
475                        reporter.info(&format!(
476                            "{}@{}: reconciliation outcome: NotPublished; action: retry under publish policy (evidence: {})",
477                            p.name,
478                            p.version,
479                            reconciliation_report_path.display()
480                        ));
481                        // fall through to normal flow
482                    }
483                    ReconciliationOutcome::StillUnknown { reason, .. } => {
484                        reporter.error(&format!(
485                            "{}@{}: reconciliation outcome: StillUnknown; action: stop before blind retry; operator action required (evidence: {}): {}",
486                            p.name,
487                            p.version,
488                            reconciliation_report_path.display(),
489                            reason
490                        ));
491                        webhook::maybe_send_event(
492                            &opts.webhook,
493                            WebhookEvent::PublishFailed {
494                                plan_id: ws.plan.plan_id.clone(),
495                                package_name: p.name.clone(),
496                                package_version: p.version.clone(),
497                                error_class: format!("{:?}", ErrorClass::Ambiguous),
498                                message: format!(
499                                    "resume reconciliation still inconclusive: {reason}"
500                                ),
501                            },
502                        );
503                        bail!(
504                            "{}@{}: resume reconciliation still inconclusive; operator action required. Prior reason: {}",
505                            p.name,
506                            p.version,
507                            reason
508                        );
509                    }
510                }
511            }
512            _ => {}
513        }
514
515        // Event: PackageStarted
516        event_log.record(PublishEvent {
517            timestamp: Utc::now(),
518            event_type: EventType::PackageStarted {
519                name: p.name.clone(),
520                version: p.version.clone(),
521            },
522            package: pkg_label.clone(),
523        });
524
525        let started_at = Utc::now();
526        let start_instant = Instant::now();
527
528        // First, check if the version is already present.
529        if reg.version_exists(&p.name, &p.version)? {
530            reporter.info(&format!(
531                "{}@{}: already published (skipping)",
532                p.name, p.version
533            ));
534            let skipped = PackageState::Skipped {
535                reason: "already published".into(),
536            };
537            update_state(&mut st, &state_dir, &key, skipped)?;
538
539            // Event: PackageSkipped
540            event_log.record(PublishEvent {
541                timestamp: Utc::now(),
542                event_type: EventType::PackageSkipped {
543                    reason: "already published".to_string(),
544                },
545                package: pkg_label.clone(),
546            });
547            event_log.write_to_file(&events_path)?;
548            event_log.clear();
549
550            let progress = st
551                .packages
552                .get(&key)
553                .context("missing package progress in state for skipped package")?;
554            receipts.push(PackageReceipt {
555                name: p.name.clone(),
556                version: p.version.clone(),
557                attempts: progress.attempts,
558                state: progress.state.clone(),
559                started_at,
560                finished_at: Utc::now(),
561                duration_ms: start_instant.elapsed().as_millis(),
562                evidence: crate::types::PackageEvidence {
563                    attempts: vec![],
564                    readiness_checks: vec![],
565                },
566                compromised_at: None,
567                compromised_by: None,
568                superseded_by: None,
569            });
570            continue;
571        }
572
573        reporter.info(&format!("{}@{}: publishing...", p.name, p.version));
574
575        // Registry-aware backoff (#94 / #106 PR 1): prefer the `PublishRegime`
576        // that preflight stamped onto the `PlannedPackage`. That answer is
577        // authoritative; when present, we never re-query the registry
578        // mid-retry.
579        //
580        // `None` here means the plan predates the regime field (old
581        // state.json, legacy test harness). In that case we fall back to the
582        // historical lazy-cached behavior for backward compatibility.
583        let mut is_new_crate_cached: Option<bool> = p.regime.map(PublishRegime::is_new_crate);
584
585        let mut attempt = st
586            .packages
587            .get(&key)
588            .context("missing package progress in state for publish")?
589            .attempts;
590        let mut last_err: Option<(ErrorClass, String)> = None;
591        let mut attempt_evidence: Vec<AttemptEvidence> = Vec::new();
592        let mut readiness_evidence: Vec<ReadinessEvidence> = Vec::new();
593
594        while attempt < opts.max_attempts {
595            attempt += 1;
596            {
597                let pr = st
598                    .packages
599                    .get_mut(&key)
600                    .context("missing package progress in state during attempt")?;
601                pr.attempts = attempt;
602                pr.last_updated_at = Utc::now();
603                state::save_state(&state_dir, &st)?;
604            }
605
606            let command = format!(
607                "cargo publish -p {} --registry {}",
608                p.name, ws.plan.registry.name
609            );
610
611            reporter.info(&format!(
612                "{}@{}: attempt {}/{}",
613                p.name, p.version, attempt, opts.max_attempts
614            ));
615
616            if !cargo_succeeded {
617                // Event: PackageAttempted
618                let attempt_started_at = Utc::now();
619                event_log.record(PublishEvent {
620                    timestamp: attempt_started_at,
621                    event_type: EventType::PackageAttempted {
622                        attempt,
623                        command: command.clone(),
624                    },
625                    package: pkg_label.clone(),
626                });
627
628                let out = cargo::cargo_publish(
629                    workspace_root,
630                    &p.name,
631                    &ws.plan.registry.name,
632                    opts.allow_dirty,
633                    opts.no_verify,
634                    opts.output_lines,
635                    None, // sequential mode: no per-package timeout
636                )?;
637                let attempt_ended_at = Utc::now();
638
639                // Collect attempt evidence
640                attempt_evidence.push(AttemptEvidence {
641                    attempt_number: attempt,
642                    command: command.clone(),
643                    exit_code: out.exit_code,
644                    stdout_tail: out.stdout_tail.clone(),
645                    stderr_tail: out.stderr_tail.clone(),
646                    timestamp: attempt_ended_at,
647                    duration: out.duration,
648                });
649
650                // Event: PackageOutput
651                event_log.record(PublishEvent {
652                    timestamp: Utc::now(),
653                    event_type: EventType::PackageOutput {
654                        stdout_tail: out.stdout_tail.clone(),
655                        stderr_tail: out.stderr_tail.clone(),
656                    },
657                    package: pkg_label.clone(),
658                });
659                event_log.write_to_file(&events_path)?;
660                event_log.clear();
661
662                if out.exit_code == 0 {
663                    record_attempt_detail(
664                        &mut st,
665                        &state_dir,
666                        AttemptDetail {
667                            package: p.name.clone(),
668                            version: p.version.clone(),
669                            attempt,
670                            max_attempts: opts.max_attempts,
671                            started_at: attempt_started_at,
672                            ended_at: attempt_ended_at,
673                            error_class: None,
674                            next_attempt_at: None,
675                            redacted_message: None,
676                        },
677                    )?;
678                    cargo_succeeded = true;
679                    // Persist Uploaded state so resume skips cargo publish
680                    update_state(&mut st, &state_dir, &key, PackageState::Uploaded)?;
681                } else {
682                    let failure_output = format!("{}\n{}", out.stderr_tail, out.stdout_tail);
683                    let (class, msg) = classify_cargo_failure(&out.stderr_tail, &out.stdout_tail);
684                    last_err = Some((class.clone(), msg.clone()));
685                    let mut attempt_detail = AttemptDetail {
686                        package: p.name.clone(),
687                        version: p.version.clone(),
688                        attempt,
689                        max_attempts: opts.max_attempts,
690                        started_at: attempt_started_at,
691                        ended_at: attempt_ended_at,
692                        error_class: Some(class.clone()),
693                        next_attempt_at: None,
694                        redacted_message: Some(msg.clone()),
695                    };
696
697                    if class == ErrorClass::Ambiguous {
698                        event_log.record(PublishEvent {
699                            timestamp: Utc::now(),
700                            event_type: EventType::PackageFailed {
701                                class: class.clone(),
702                                message: msg.clone(),
703                            },
704                            package: pkg_label.clone(),
705                        });
706                        event_log.record(PublishEvent {
707                            timestamp: Utc::now(),
708                            event_type: EventType::PublishReconciling {
709                                method: opts.readiness.method,
710                            },
711                            package: pkg_label.clone(),
712                        });
713                        reporter.warn(&format!(
714                            "{}@{}: cargo exit ambiguous; reconciling against registry truth before retry",
715                            p.name, p.version
716                        ));
717
718                        let readiness_config = crate::types::ReadinessConfig {
719                            enabled: effects.readiness_enabled,
720                            ..opts.readiness.clone()
721                        };
722                        let (outcome, reconcile_evidence) =
723                            sequential_reconcile(&reg, &p.name, &p.version, &readiness_config);
724
725                        event_log.record(PublishEvent {
726                            timestamp: Utc::now(),
727                            event_type: EventType::PublishReconciled {
728                                outcome: outcome.clone(),
729                            },
730                            package: pkg_label.clone(),
731                        });
732                        event_log.write_to_file(&events_path)?;
733                        event_log.clear();
734                        write_reconciliation_report_best_effort(
735                            &state_dir,
736                            ws,
737                            &events_path,
738                            reporter,
739                        );
740                        let reconciliation_report_path = state::reconciliation_path(&state_dir);
741
742                        match outcome {
743                            ReconciliationOutcome::Published { .. } => {
744                                record_attempt_detail(&mut st, &state_dir, attempt_detail)?;
745                                reporter.info(&format!(
746                                    "{}@{}: reconciliation outcome: Published; registry shows version present; action: mark published and continue without retry (evidence: {})",
747                                    p.name,
748                                    p.version,
749                                    reconciliation_report_path.display()
750                                ));
751                                update_state(&mut st, &state_dir, &key, PackageState::Published)?;
752                                event_log.record(PublishEvent {
753                                    timestamp: Utc::now(),
754                                    event_type: EventType::PackagePublished {
755                                        duration_ms: start_instant.elapsed().as_millis() as u64,
756                                    },
757                                    package: pkg_label.clone(),
758                                });
759                                event_log.write_to_file(&events_path)?;
760                                event_log.clear();
761                                readiness_evidence = reconcile_evidence;
762                                last_err = None;
763                                break;
764                            }
765                            ReconciliationOutcome::NotPublished { .. } => {
766                                reporter.info(&format!(
767                                    "{}@{}: reconciliation outcome: NotPublished; registry still absent; action: retry under publish policy (evidence: {})",
768                                    p.name,
769                                    p.version,
770                                    reconciliation_report_path.display()
771                                ));
772                                readiness_evidence = reconcile_evidence;
773                            }
774                            ReconciliationOutcome::StillUnknown { reason, .. } => {
775                                record_attempt_detail(&mut st, &state_dir, attempt_detail)?;
776                                let ambiguous_state = PackageState::Ambiguous {
777                                    message: reason.clone(),
778                                };
779                                update_state(&mut st, &state_dir, &key, ambiguous_state)?;
780                                reporter.error(&format!(
781                                    "{}@{}: reconciliation outcome: StillUnknown; action: stop before blind retry; operator action required (evidence: {}): {}",
782                                    p.name,
783                                    p.version,
784                                    reconciliation_report_path.display(),
785                                    reason
786                                ));
787                                webhook::maybe_send_event(
788                                    &opts.webhook,
789                                    WebhookEvent::PublishFailed {
790                                        plan_id: ws.plan.plan_id.clone(),
791                                        package_name: p.name.clone(),
792                                        package_version: p.version.clone(),
793                                        error_class: format!("{:?}", ErrorClass::Ambiguous),
794                                        message: format!("reconciliation inconclusive: {reason}"),
795                                    },
796                                );
797                                bail!(
798                                    "{}@{}: reconciliation inconclusive; operator action required: {}",
799                                    p.name,
800                                    p.version,
801                                    reason
802                                );
803                            }
804                        }
805                    } else {
806                        // Even if cargo fails, the publish may have succeeded (timeouts, network splits).
807                        // Non-ambiguous failures keep the historical quick registry check; ambiguous
808                        // failures use the full reconciliation state machine above.
809                        reporter.warn(&format!(
810                            "{}@{}: cargo publish failed (exit={:?}); checking registry...",
811                            p.name, p.version, out.exit_code
812                        ));
813
814                        if reg.version_exists(&p.name, &p.version)? {
815                            reporter.info(&format!(
816                                "{}@{}: version is present on registry; treating as published",
817                                p.name, p.version
818                            ));
819                            record_attempt_detail(&mut st, &state_dir, attempt_detail)?;
820                            update_state(&mut st, &state_dir, &key, PackageState::Published)?;
821                            event_log.record(PublishEvent {
822                                timestamp: Utc::now(),
823                                event_type: EventType::PackagePublished {
824                                    duration_ms: start_instant.elapsed().as_millis() as u64,
825                                },
826                                package: pkg_label.clone(),
827                            });
828                            event_log.write_to_file(&events_path)?;
829                            event_log.clear();
830                            last_err = None;
831                            break;
832                        }
833                    }
834
835                    match class {
836                        ErrorClass::Permanent => {
837                            record_attempt_detail(&mut st, &state_dir, attempt_detail)?;
838                            let failed = PackageState::Failed {
839                                class: class.clone(),
840                                message: msg.clone(),
841                            };
842                            update_state(&mut st, &state_dir, &key, failed)?;
843
844                            // Event: PackageFailed
845                            event_log.record(PublishEvent {
846                                timestamp: Utc::now(),
847                                event_type: EventType::PackageFailed {
848                                    class,
849                                    message: msg,
850                                },
851                                package: pkg_label.clone(),
852                            });
853                            event_log.write_to_file(&events_path)?;
854                            event_log.clear();
855
856                            return Err(anyhow::anyhow!(
857                                "{}@{}: permanent failure: {}",
858                                p.name,
859                                p.version,
860                                last_err.unwrap().1
861                            ));
862                        }
863                        ErrorClass::Retryable | ErrorClass::Ambiguous => {
864                            let is_new_crate = if crate::runtime::execution::looks_like_rate_limit(
865                                &failure_output,
866                            ) {
867                                *is_new_crate_cached.get_or_insert_with(|| {
868                                    reg.check_new_crate(&p.name).unwrap_or(false)
869                                })
870                            } else {
871                                false
872                            };
873                            if attempt < opts.max_attempts {
874                                if crate::runtime::execution::looks_like_rate_limit(&failure_output)
875                                {
876                                    record_rate_limit_observed_event(
877                                        &mut event_log,
878                                        &events_path,
879                                        &pkg_label,
880                                        is_new_crate,
881                                        retry_after_delay(&failure_output),
882                                        &msg,
883                                    )?;
884                                }
885                                let delay = registry_aware_backoff(
886                                    opts.base_delay,
887                                    opts.max_delay,
888                                    attempt,
889                                    opts.retry_strategy,
890                                    opts.retry_jitter,
891                                    is_new_crate,
892                                    &failure_output,
893                                );
894                                let next_attempt_at = retry_next_attempt_at(delay);
895                                attempt_detail.next_attempt_at = Some(next_attempt_at);
896                                record_retry_backoff_event(
897                                    &mut event_log,
898                                    &events_path,
899                                    &pkg_label,
900                                    attempt,
901                                    opts.max_attempts,
902                                    delay,
903                                    next_attempt_at,
904                                    &class,
905                                    &msg,
906                                )?;
907                                record_attempt_detail(&mut st, &state_dir, attempt_detail)?;
908                                wait_after_retry(
909                                    reporter,
910                                    &p.name,
911                                    &p.version,
912                                    attempt,
913                                    opts.max_attempts,
914                                    delay,
915                                    class.clone(),
916                                    &msg,
917                                );
918                            } else {
919                                record_attempt_detail(&mut st, &state_dir, attempt_detail)?;
920                            }
921                        }
922                    }
923                    continue;
924                }
925            }
926
927            // Readiness verification (runs after first cargo success + all retries)
928            reporter.info(&format!(
929                "{}@{}: cargo publish exited successfully; verifying...",
930                p.name, p.version
931            ));
932            let readiness_config = crate::types::ReadinessConfig {
933                enabled: effects.readiness_enabled,
934                ..opts.readiness.clone()
935            };
936            let (visible, checks) = verify_published(
937                &reg,
938                &p.name,
939                &p.version,
940                &readiness_config,
941                reporter,
942                &mut event_log,
943                &events_path,
944                &pkg_label,
945            )?;
946            readiness_evidence = checks;
947            if visible {
948                update_state(&mut st, &state_dir, &key, PackageState::Published)?;
949                last_err = None;
950
951                // Event: PackagePublished
952                event_log.record(PublishEvent {
953                    timestamp: Utc::now(),
954                    event_type: EventType::PackagePublished {
955                        duration_ms: start_instant.elapsed().as_millis() as u64,
956                    },
957                    package: pkg_label.clone(),
958                });
959                event_log.write_to_file(&events_path)?;
960                event_log.clear();
961
962                // Send webhook notification: package succeeded
963                webhook::maybe_send_event(
964                    &opts.webhook,
965                    WebhookEvent::PublishSucceeded {
966                        plan_id: ws.plan.plan_id.clone(),
967                        package_name: p.name.clone(),
968                        package_version: p.version.clone(),
969                        duration_ms: start_instant.elapsed().as_millis() as u64,
970                    },
971                );
972
973                break;
974            } else {
975                let message =
976                    "published locally, but version not observed on registry within timeout";
977                last_err = Some((ErrorClass::Ambiguous, message.to_string()));
978                let delay = backoff_delay(
979                    opts.base_delay,
980                    opts.max_delay,
981                    attempt,
982                    opts.retry_strategy,
983                    opts.retry_jitter,
984                );
985                let next_attempt_at = retry_next_attempt_at(delay);
986                emit_retry_backoff_event(
987                    &mut event_log,
988                    &events_path,
989                    reporter,
990                    &pkg_label,
991                    &p.name,
992                    &p.version,
993                    attempt,
994                    opts.max_attempts,
995                    delay,
996                    next_attempt_at,
997                    ErrorClass::Ambiguous,
998                    message,
999                )?;
1000            }
1001        }
1002
1003        // If package is still Uploaded (loop didn't run or readiness never checked), force a final check
1004        if last_err.is_none() {
1005            let current_state = st.packages.get(&key).map(|p| &p.state);
1006            if matches!(current_state, Some(PackageState::Uploaded)) {
1007                if reg.version_exists(&p.name, &p.version)? {
1008                    update_state(&mut st, &state_dir, &key, PackageState::Published)?;
1009                    event_log.record(PublishEvent {
1010                        timestamp: Utc::now(),
1011                        event_type: EventType::PackagePublished {
1012                            duration_ms: start_instant.elapsed().as_millis() as u64,
1013                        },
1014                        package: pkg_label.clone(),
1015                    });
1016                    event_log.write_to_file(&events_path)?;
1017                    event_log.clear();
1018                } else {
1019                    last_err = Some((
1020                        ErrorClass::Ambiguous,
1021                        "package was uploaded but not confirmed visible on registry".into(),
1022                    ));
1023                }
1024            }
1025        }
1026
1027        let finished_at = Utc::now();
1028        let duration_ms = start_instant.elapsed().as_millis();
1029
1030        if let Some((class, msg)) = last_err {
1031            // Final chance: maybe it eventually showed up.
1032            if reg.version_exists(&p.name, &p.version)? {
1033                update_state(&mut st, &state_dir, &key, PackageState::Published)?;
1034                event_log.record(PublishEvent {
1035                    timestamp: Utc::now(),
1036                    event_type: EventType::PackagePublished {
1037                        duration_ms: start_instant.elapsed().as_millis() as u64,
1038                    },
1039                    package: pkg_label.clone(),
1040                });
1041                event_log.write_to_file(&events_path)?;
1042                event_log.clear();
1043            } else {
1044                let failed = PackageState::Failed {
1045                    class: class.clone(),
1046                    message: msg.clone(),
1047                };
1048                update_state(&mut st, &state_dir, &key, failed)?;
1049
1050                // Event: PackageFailed
1051                event_log.record(PublishEvent {
1052                    timestamp: Utc::now(),
1053                    event_type: EventType::PackageFailed {
1054                        class: class.clone(),
1055                        message: msg.clone(),
1056                    },
1057                    package: pkg_label.clone(),
1058                });
1059                event_log.write_to_file(&events_path)?;
1060                event_log.clear();
1061
1062                // Send webhook notification: package failed
1063                webhook::maybe_send_event(
1064                    &opts.webhook,
1065                    WebhookEvent::PublishFailed {
1066                        plan_id: ws.plan.plan_id.clone(),
1067                        package_name: p.name.clone(),
1068                        package_version: p.version.clone(),
1069                        error_class: format!("{:?}", class.clone()),
1070                        message: msg.clone(),
1071                    },
1072                );
1073
1074                let progress = st
1075                    .packages
1076                    .get(&key)
1077                    .context("missing package progress in state for failed package")?;
1078                receipts.push(PackageReceipt {
1079                    name: p.name.clone(),
1080                    version: p.version.clone(),
1081                    attempts: progress.attempts,
1082                    state: progress.state.clone(),
1083                    started_at,
1084                    finished_at,
1085                    duration_ms,
1086                    evidence: crate::types::PackageEvidence {
1087                        attempts: attempt_evidence,
1088                        readiness_checks: readiness_evidence,
1089                    },
1090                    compromised_at: None,
1091                    compromised_by: None,
1092                    superseded_by: None,
1093                });
1094                return Err(anyhow::anyhow!("{}@{}: failed: {}", p.name, p.version, msg));
1095            }
1096        }
1097
1098        let progress = st
1099            .packages
1100            .get(&key)
1101            .context("missing package progress in state for completed package")?;
1102        receipts.push(PackageReceipt {
1103            name: p.name.clone(),
1104            version: p.version.clone(),
1105            attempts: progress.attempts,
1106            state: progress.state.clone(),
1107            started_at,
1108            finished_at,
1109            duration_ms,
1110            evidence: crate::types::PackageEvidence {
1111                attempts: attempt_evidence,
1112                readiness_checks: readiness_evidence,
1113            },
1114            compromised_at: None,
1115            compromised_by: None,
1116            superseded_by: None,
1117        });
1118    }
1119
1120    publish::finalize::record_consistency_drift(&events_path, &st, &mut event_log, reporter);
1121    publish::finalize::finish_sequential_run(
1122        ws,
1123        opts,
1124        &state_dir,
1125        &events_path,
1126        &mut event_log,
1127        &st,
1128        receipts,
1129        run_started,
1130        git_context,
1131        environment,
1132        auth_evidence,
1133    )
1134}
1135
1136/// Resume a previously interrupted publish operation.
1137///
1138/// This function loads existing state from the state directory and continues
1139/// publishing from where it left off. It handles:
1140/// - Packages that were never attempted (Pending)
1141/// - Packages that failed and should be retried
1142/// - Packages that were uploaded but not verified (Uploaded)
1143/// - Already-successful packages (Published/Skipped) - skipped automatically
1144///
1145/// # Arguments
1146///
1147/// * `ws` - The planned workspace (should match the original plan)
1148/// * `opts` - Runtime options
1149/// * `reporter` - A reporter for outputting progress
1150///
1151/// # Returns
1152///
1153/// Returns a [`Receipt`] similar to [`run_publish`].
1154///
1155/// # Error Handling
1156///
1157/// Returns an error if:
1158/// - No existing state is found in the state directory
1159/// - The plan ID doesn't match (use `opts.force_resume` to override)
1160/// - Lock acquisition fails
1161///
1162/// # Example
1163///
1164/// ```ignore
1165/// let mut ws = plan::build_plan(&spec)?;
1166/// let opts = types::RuntimeOptions { /* ... */ };
1167/// let mut reporter = MyReporter::default();
1168/// let receipt = engine::run_resume(&ws, &opts, &mut reporter)?;
1169/// println!("Published {} packages", receipt.packages.len());
1170/// ```
1171pub fn run_resume(
1172    ws: &PlannedWorkspace,
1173    opts: &RuntimeOptions,
1174    reporter: &mut dyn Reporter,
1175) -> Result<Receipt> {
1176    let workspace_root = &ws.workspace_root;
1177    let state_dir = resolve_state_dir(workspace_root, &opts.state_dir);
1178    if state::load_state(&state_dir)?.is_none() {
1179        bail!(
1180            "no existing state found in {}; run shipper publish first",
1181            state_dir.display()
1182        );
1183    }
1184    run_publish(ws, opts, reporter)
1185}
1186
1187/// Outcome of a rehearsal run. Sufficient for callers (CLI, future hard gate)
1188/// to decide whether live dispatch is authorized without re-reading events.
1189///
1190/// #97 PR 2. The hard gate (#97 PR 3) will bind this outcome to a `plan_id`
1191/// so "rehearsal passed" can't be claimed for a different workspace state.
1192#[derive(Debug, Clone)]
1193pub struct RehearsalOutcome {
1194    pub passed: bool,
1195    pub registry_name: String,
1196    pub packages_attempted: usize,
1197    pub packages_published: usize,
1198    pub summary: String,
1199}
1200
1201/// Run a rehearsal publish against an alternate registry (#97 PR 2).
1202///
1203/// Phase-2 preflight: publish every crate in the plan to a non-live
1204/// registry, verify each is visible on that registry, and emit a
1205/// `RehearsalComplete` event summarizing the outcome.
1206///
1207/// **Contract**:
1208/// - Reads `opts.rehearsal_registry` (set via `--rehearsal-registry` or
1209///   `[rehearsal]` config). Must resolve to a [`Registry`] entry in
1210///   `opts.registries`; bails clean otherwise.
1211/// - Refuses to rehearse against the live target (`ws.plan.registry`).
1212///   Rehearsal and live must be different registries.
1213/// - Runs sequentially (no parallel yet); stops at the first failure.
1214/// - Does NOT touch `state.json`. Rehearsal is a pre-publish proof, not
1215///   an execution; it only appends to `events.jsonl` so auditors can
1216///   replay the rehearsal from the event log.
1217/// - Post-publish visibility check uses the SAME readiness mechanism as
1218///   live publish (`reg.version_exists`). A rehearsal artifact that's
1219///   not visible within the readiness window fails the rehearsal.
1220///
1221/// **Not in this PR**:
1222/// - Hard gate wiring into `run_publish` (PR 3).
1223/// - Install/smoke check against the rehearsal registry (PR 3 or 4).
1224/// - Parallel rehearsal (not planned; rehearsal is infrequent and
1225///   correctness > speed).
1226pub fn run_rehearsal(
1227    ws: &PlannedWorkspace,
1228    opts: &RuntimeOptions,
1229    reporter: &mut dyn Reporter,
1230) -> Result<RehearsalOutcome> {
1231    let rehearsal_name = opts
1232        .rehearsal_registry
1233        .as_ref()
1234        .ok_or_else(|| {
1235            anyhow::anyhow!(
1236                "no rehearsal registry configured; set --rehearsal-registry <name> \
1237             or enable [rehearsal] in .shipper.toml"
1238            )
1239        })?
1240        .clone();
1241
1242    if opts.rehearsal_skip {
1243        reporter.warn(&format!(
1244            "--skip-rehearsal set; rehearsal against '{rehearsal_name}' was requested but will not run. \
1245             Once #97 PR 3 lands, live dispatch will refuse without a prior passing rehearsal."
1246        ));
1247        return Ok(RehearsalOutcome {
1248            passed: false,
1249            registry_name: rehearsal_name,
1250            packages_attempted: 0,
1251            packages_published: 0,
1252            summary: "skipped by --skip-rehearsal".to_string(),
1253        });
1254    }
1255
1256    let rehearsal_reg = opts
1257        .registries
1258        .iter()
1259        .find(|r| r.name == rehearsal_name)
1260        .cloned()
1261        .ok_or_else(|| {
1262            anyhow::anyhow!(
1263                "rehearsal registry '{rehearsal_name}' is not configured. \
1264             Add it to [[registries]] in .shipper.toml or pass --registries."
1265            )
1266        })?;
1267
1268    if rehearsal_reg.name == ws.plan.registry.name {
1269        bail!(
1270            "rehearsal registry '{}' must differ from the live target; \
1271             pick a sandbox registry (e.g. kellnr, a fresh crates-io test account, \
1272             or a throwaway alternate-registry entry)",
1273            rehearsal_reg.name
1274        );
1275    }
1276
1277    let workspace_root = &ws.workspace_root;
1278    let state_dir = resolve_state_dir(workspace_root, &opts.state_dir);
1279    std::fs::create_dir_all(&state_dir)
1280        .with_context(|| format!("failed to create state dir {}", state_dir.display()))?;
1281    let events_path = events::events_path(&state_dir);
1282    let mut event_log = events::EventLog::new();
1283    let started_at = Utc::now();
1284
1285    reporter.info(&format!(
1286        "rehearsal starting — {} packages against '{}'",
1287        ws.plan.packages.len(),
1288        rehearsal_name
1289    ));
1290
1291    event_log.record(PublishEvent {
1292        timestamp: Utc::now(),
1293        event_type: EventType::RehearsalStarted {
1294            registry: rehearsal_name.clone(),
1295            plan_id: ws.plan.plan_id.clone(),
1296            package_count: ws.plan.packages.len(),
1297        },
1298        package: "all".to_string(),
1299    });
1300    event_log.write_to_file(&events_path)?;
1301    event_log.clear();
1302
1303    let rehearsal_client = init_registry_client(rehearsal_reg.clone(), &state_dir)?;
1304
1305    let mut packages_published: usize = 0;
1306    let mut first_failure: Option<String> = None;
1307
1308    for p in &ws.plan.packages {
1309        let pkg_label = format!("{}@{}", p.name, p.version);
1310        reporter.info(&format!("rehearsing {pkg_label} → {rehearsal_name}"));
1311        let start = Instant::now();
1312
1313        let out = cargo::cargo_publish(
1314            workspace_root,
1315            &p.name,
1316            &rehearsal_reg.name,
1317            opts.allow_dirty,
1318            opts.no_verify,
1319            opts.output_lines,
1320            None,
1321        )?;
1322
1323        if out.exit_code != 0 {
1324            let (class, msg) = classify_cargo_failure(&out.stderr_tail, &out.stdout_tail);
1325            reporter.error(&format!(
1326                "rehearsal failed for {pkg_label}: {msg}\nstderr tail:\n{}",
1327                out.stderr_tail
1328            ));
1329            event_log.record(PublishEvent {
1330                timestamp: Utc::now(),
1331                event_type: EventType::RehearsalPackageFailed {
1332                    name: p.name.clone(),
1333                    version: p.version.clone(),
1334                    class,
1335                    message: msg.clone(),
1336                },
1337                package: pkg_label.clone(),
1338            });
1339            event_log.write_to_file(&events_path)?;
1340            event_log.clear();
1341            first_failure = Some(format!("{pkg_label}: {msg}"));
1342            break;
1343        }
1344
1345        // Post-publish visibility check on the rehearsal registry. Reuse
1346        // `version_exists` — same mechanism live publish trusts.
1347        if !rehearsal_client.version_exists(&p.name, &p.version)? {
1348            let msg = format!(
1349                "rehearsal: cargo publish succeeded but {pkg_label} is not visible on '{rehearsal_name}'"
1350            );
1351            reporter.error(&msg);
1352            event_log.record(PublishEvent {
1353                timestamp: Utc::now(),
1354                event_type: EventType::RehearsalPackageFailed {
1355                    name: p.name.clone(),
1356                    version: p.version.clone(),
1357                    class: ErrorClass::Ambiguous,
1358                    message: msg.clone(),
1359                },
1360                package: pkg_label.clone(),
1361            });
1362            event_log.write_to_file(&events_path)?;
1363            event_log.clear();
1364            first_failure = Some(msg);
1365            break;
1366        }
1367
1368        let duration_ms = start.elapsed().as_millis();
1369        event_log.record(PublishEvent {
1370            timestamp: Utc::now(),
1371            event_type: EventType::RehearsalPackagePublished {
1372                name: p.name.clone(),
1373                version: p.version.clone(),
1374                duration_ms,
1375            },
1376            package: pkg_label.clone(),
1377        });
1378        event_log.write_to_file(&events_path)?;
1379        event_log.clear();
1380        packages_published += 1;
1381    }
1382
1383    // #97 PR 4 — smoke-install. Opt-in. Runs only if:
1384    //   (a) all packages in the plan were published successfully AND
1385    //   (b) the operator named a crate via --smoke-install / config.
1386    // The named crate must be in the plan; resolves its planned version
1387    // to pass through to `cargo install`.
1388    if first_failure.is_none()
1389        && let Some(ref smoke_name) = opts.rehearsal_smoke_install
1390    {
1391        match ws.plan.packages.iter().find(|p| &p.name == smoke_name) {
1392            Some(smoke_pkg) => {
1393                reporter.info(&format!(
1394                    "smoke-install: {smoke_name}@{} from '{rehearsal_name}'",
1395                    smoke_pkg.version
1396                ));
1397
1398                event_log.record(PublishEvent {
1399                    timestamp: Utc::now(),
1400                    event_type: EventType::RehearsalSmokeCheckStarted {
1401                        name: smoke_pkg.name.clone(),
1402                        version: smoke_pkg.version.clone(),
1403                        registry: rehearsal_name.clone(),
1404                    },
1405                    package: format!("{smoke_name}@{}", smoke_pkg.version),
1406                });
1407                event_log.write_to_file(&events_path)?;
1408                event_log.clear();
1409
1410                let install_root = state_dir.join("smoke-install");
1411                let _ = std::fs::remove_dir_all(&install_root);
1412                let smoke_start = Instant::now();
1413                let out = cargo::cargo_install_smoke(
1414                    workspace_root,
1415                    &smoke_pkg.name,
1416                    &smoke_pkg.version,
1417                    &rehearsal_reg.name,
1418                    &install_root,
1419                    opts.output_lines,
1420                    None,
1421                )?;
1422
1423                if out.exit_code == 0 {
1424                    let duration_ms = smoke_start.elapsed().as_millis();
1425                    event_log.record(PublishEvent {
1426                        timestamp: Utc::now(),
1427                        event_type: EventType::RehearsalSmokeCheckSucceeded {
1428                            name: smoke_pkg.name.clone(),
1429                            version: smoke_pkg.version.clone(),
1430                            duration_ms,
1431                        },
1432                        package: format!("{smoke_name}@{}", smoke_pkg.version),
1433                    });
1434                    reporter.info(&format!(
1435                        "smoke-install OK for {smoke_name}@{}",
1436                        smoke_pkg.version
1437                    ));
1438                } else {
1439                    let msg = format!(
1440                        "cargo install exited {} for {smoke_name}@{}. stderr tail:\n{}",
1441                        out.exit_code, smoke_pkg.version, out.stderr_tail
1442                    );
1443                    reporter.error(&msg);
1444                    event_log.record(PublishEvent {
1445                        timestamp: Utc::now(),
1446                        event_type: EventType::RehearsalSmokeCheckFailed {
1447                            name: smoke_pkg.name.clone(),
1448                            version: smoke_pkg.version.clone(),
1449                            message: msg.clone(),
1450                        },
1451                        package: format!("{smoke_name}@{}", smoke_pkg.version),
1452                    });
1453                    first_failure = Some(format!(
1454                        "smoke-install of {smoke_name}@{} failed: cargo exit {}",
1455                        smoke_pkg.version, out.exit_code
1456                    ));
1457                }
1458                event_log.write_to_file(&events_path)?;
1459                event_log.clear();
1460            }
1461            None => {
1462                // Operator named a crate that isn't in the plan. Warn,
1463                // don't fail — their intent is clear but the workspace
1464                // shape disagrees, and failing the whole rehearsal over
1465                // a typo would be overkill.
1466                reporter.warn(&format!(
1467                    "smoke-install target '{smoke_name}' is not in the rehearsal plan; skipping. \
1468                     Available crates: {}",
1469                    ws.plan
1470                        .packages
1471                        .iter()
1472                        .map(|p| p.name.as_str())
1473                        .collect::<Vec<_>>()
1474                        .join(", ")
1475                ));
1476            }
1477        }
1478    }
1479
1480    let passed = first_failure.is_none();
1481    let summary = if passed {
1482        format!("rehearsed {packages_published} packages against '{rehearsal_name}' successfully")
1483    } else {
1484        format!(
1485            "rehearsal stopped at {}/{}: {}",
1486            packages_published + 1,
1487            ws.plan.packages.len(),
1488            first_failure.as_deref().unwrap_or("")
1489        )
1490    };
1491
1492    let completed_at = Utc::now();
1493    event_log.record(PublishEvent {
1494        timestamp: completed_at,
1495        event_type: EventType::RehearsalComplete {
1496            passed,
1497            registry: rehearsal_name.clone(),
1498            plan_id: ws.plan.plan_id.clone(),
1499            summary: summary.clone(),
1500        },
1501        package: "all".to_string(),
1502    });
1503    event_log.write_to_file(&events_path)?;
1504
1505    // Persist the sidecar receipt so the hard gate in `run_publish`
1506    // can consult it without parsing events.jsonl. Best-effort — a
1507    // write failure here doesn't invalidate the events log, which is
1508    // the authoritative source; the gate will just act as if no
1509    // rehearsal happened and block, which is the safe default.
1510    let packages_attempted = packages_published + if passed { 0 } else { 1 };
1511    if let Err(err) = crate::state::rehearsal::save_rehearsal(
1512        &state_dir,
1513        &crate::state::rehearsal::RehearsalReceipt {
1514            schema_version: crate::state::rehearsal::CURRENT_REHEARSAL_VERSION.to_string(),
1515            plan_id: ws.plan.plan_id.clone(),
1516            registry: rehearsal_name.clone(),
1517            passed,
1518            packages_attempted,
1519            packages_published,
1520            summary: summary.clone(),
1521            started_at,
1522            completed_at,
1523        },
1524    ) {
1525        reporter.warn(&format!(
1526            "rehearsal outcome event was written, but sidecar receipt could not be persisted: {err:#}. \
1527             The hard gate may not recognize this rehearsal — check {}.",
1528            crate::state::rehearsal::rehearsal_path(&state_dir).display()
1529        ));
1530    }
1531
1532    if passed {
1533        reporter.info(&summary);
1534    } else {
1535        reporter.error(&summary);
1536    }
1537
1538    Ok(RehearsalOutcome {
1539        passed,
1540        registry_name: rehearsal_name,
1541        packages_attempted,
1542        packages_published,
1543        summary,
1544    })
1545}
1546
1547pub(crate) fn init_state(ws: &PlannedWorkspace, state_dir: &Path) -> Result<ExecutionState> {
1548    let mut packages: BTreeMap<String, PackageProgress> = BTreeMap::new();
1549    for p in &ws.plan.packages {
1550        packages.insert(
1551            pkg_key(&p.name, &p.version),
1552            PackageProgress {
1553                name: p.name.clone(),
1554                version: p.version.clone(),
1555                attempts: 0,
1556                state: PackageState::Pending,
1557                last_updated_at: Utc::now(),
1558            },
1559        );
1560    }
1561
1562    let st = ExecutionState {
1563        state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
1564        plan_id: ws.plan.plan_id.clone(),
1565        registry: ws.plan.registry.clone(),
1566        created_at: Utc::now(),
1567        updated_at: Utc::now(),
1568        attempt_history: Vec::new(),
1569        packages,
1570    };
1571
1572    state::save_state(state_dir, &st)?;
1573    Ok(st)
1574}
1575
1576/// Reconcile an ambiguous publish outcome against registry truth (sequential
1577/// path mirror of `engine::parallel::reconcile::reconcile_ambiguous_upload`).
1578///
1579/// Returns the same [`ReconciliationOutcome`] enum + accumulated
1580/// [`ReadinessEvidence`], wrapping the sequential path's registry client
1581/// (`crate::registry::RegistryClient`) rather than the parallel path's
1582/// `HttpRegistryClient`. Used by the resume-path branch that handles packages
1583/// found in `PackageState::Ambiguous` (#99 follow-on).
1584fn sequential_reconcile(
1585    reg: &RegistryClient,
1586    crate_name: &str,
1587    version: &str,
1588    config: &crate::types::ReadinessConfig,
1589) -> (
1590    crate::types::ReconciliationOutcome,
1591    Vec<crate::types::ReadinessEvidence>,
1592) {
1593    let start = Instant::now();
1594    match reg.is_version_visible_with_backoff(crate_name, version, config) {
1595        Ok((true, evidence)) => (
1596            crate::types::ReconciliationOutcome::Published {
1597                attempts: evidence.len() as u32,
1598                elapsed_ms: start.elapsed().as_millis() as u64,
1599            },
1600            evidence,
1601        ),
1602        Ok((false, evidence)) => (
1603            crate::types::ReconciliationOutcome::NotPublished {
1604                attempts: evidence.len() as u32,
1605                elapsed_ms: start.elapsed().as_millis() as u64,
1606            },
1607            evidence,
1608        ),
1609        Err(e) => (
1610            crate::types::ReconciliationOutcome::StillUnknown {
1611                attempts: 0,
1612                elapsed_ms: start.elapsed().as_millis() as u64,
1613                reason: format!("reconciliation query failed: {e}"),
1614            },
1615            Vec::new(),
1616        ),
1617    }
1618}
1619
1620/// Emit a [`EventType::RetryBackoffStarted`] event + a human-readable warn
1621/// line, then `thread::sleep(delay)`. Used at every retry-backoff site in the
1622/// sequential publish loop so operators never stare at a silent CI log during
1623/// the wait window. See #91. (The parallel path has a mirror helper in
1624/// `engine::parallel::publish::emit_retry_backoff` that handles its
1625/// `Arc<Mutex<_>>` wrapping.)
1626#[allow(clippy::too_many_arguments)]
1627fn emit_retry_backoff_event(
1628    event_log: &mut events::EventLog,
1629    events_path: &Path,
1630    reporter: &mut dyn Reporter,
1631    pkg_label: &str,
1632    pkg_name: &str,
1633    pkg_version: &str,
1634    attempt: u32,
1635    max_attempts: u32,
1636    delay: std::time::Duration,
1637    next_attempt_at: chrono::DateTime<Utc>,
1638    reason: ErrorClass,
1639    message: &str,
1640) -> Result<()> {
1641    record_retry_backoff_event(
1642        event_log,
1643        events_path,
1644        pkg_label,
1645        attempt,
1646        max_attempts,
1647        delay,
1648        next_attempt_at,
1649        &reason,
1650        message,
1651    )?;
1652    wait_after_retry(
1653        reporter,
1654        pkg_name,
1655        pkg_version,
1656        attempt,
1657        max_attempts,
1658        delay,
1659        reason,
1660        message,
1661    );
1662    Ok(())
1663}
1664
1665#[allow(clippy::too_many_arguments)]
1666fn record_retry_backoff_event(
1667    event_log: &mut events::EventLog,
1668    events_path: &Path,
1669    pkg_label: &str,
1670    attempt: u32,
1671    max_attempts: u32,
1672    delay: std::time::Duration,
1673    next_attempt_at: chrono::DateTime<Utc>,
1674    reason: &ErrorClass,
1675    message: &str,
1676) -> Result<()> {
1677    event_log.record(PublishEvent {
1678        timestamp: Utc::now(),
1679        event_type: EventType::RetryScheduled {
1680            attempt,
1681            max_attempts,
1682            delay_ms: delay.as_millis() as u64,
1683            next_attempt_at,
1684            reason: reason.clone(),
1685            message: message.to_string(),
1686        },
1687        package: pkg_label.to_string(),
1688    });
1689    record_publish_wait_event(
1690        event_log,
1691        events_path,
1692        pkg_label,
1693        delay,
1694        "retry backoff",
1695        Some(next_attempt_at),
1696    )?;
1697    event_log.record(PublishEvent {
1698        timestamp: Utc::now(),
1699        event_type: EventType::RetryBackoffStarted {
1700            attempt,
1701            max_attempts,
1702            delay_ms: delay.as_millis() as u64,
1703            next_attempt_at,
1704            reason: reason.clone(),
1705            message: message.to_string(),
1706        },
1707        package: pkg_label.to_string(),
1708    });
1709    event_log.write_to_file(events_path)?;
1710    event_log.clear();
1711    Ok(())
1712}
1713
1714fn record_rate_limit_observed_event(
1715    event_log: &mut events::EventLog,
1716    events_path: &Path,
1717    pkg_label: &str,
1718    is_new_crate: bool,
1719    retry_after: Option<std::time::Duration>,
1720    message: &str,
1721) -> Result<()> {
1722    event_log.record(PublishEvent {
1723        timestamp: Utc::now(),
1724        event_type: EventType::RateLimitObserved {
1725            is_new_crate,
1726            retry_after_ms: retry_after.map(|delay| delay.as_millis() as u64),
1727            message: message.to_string(),
1728        },
1729        package: pkg_label.to_string(),
1730    });
1731    event_log.write_to_file(events_path)?;
1732    event_log.clear();
1733    Ok(())
1734}
1735
1736fn record_publish_wait_event(
1737    event_log: &mut events::EventLog,
1738    events_path: &Path,
1739    pkg_label: &str,
1740    delay: std::time::Duration,
1741    reason: &str,
1742    until: Option<chrono::DateTime<Utc>>,
1743) -> Result<()> {
1744    event_log.record(PublishEvent {
1745        timestamp: Utc::now(),
1746        event_type: EventType::PublishWaiting {
1747            reason: reason.to_string(),
1748            delay_ms: delay.as_millis() as u64,
1749            until: until.unwrap_or_else(|| retry_next_attempt_at(delay)),
1750        },
1751        package: pkg_label.to_string(),
1752    });
1753    event_log.write_to_file(events_path)?;
1754    event_log.clear();
1755    Ok(())
1756}
1757
1758#[allow(clippy::too_many_arguments)]
1759fn wait_after_retry(
1760    reporter: &mut dyn Reporter,
1761    pkg_name: &str,
1762    pkg_version: &str,
1763    attempt: u32,
1764    max_attempts: u32,
1765    delay: std::time::Duration,
1766    reason: ErrorClass,
1767    message: &str,
1768) {
1769    // Delegate the human-facing narration AND the backoff sleep to the
1770    // Reporter so TTY-capable UIs can render a live countdown during the
1771    // wait. The default `Reporter::retry_wait` impl preserves the pre-#103
1772    // behavior (single warn line + `thread::sleep(delay)`), so reporters
1773    // that don't override it behave exactly as before.
1774    reporter.retry_wait(
1775        pkg_name,
1776        pkg_version,
1777        attempt,
1778        max_attempts,
1779        delay,
1780        reason,
1781        message,
1782    );
1783}
1784
1785fn verify_published(
1786    reg: &RegistryClient,
1787    crate_name: &str,
1788    version: &str,
1789    config: &crate::types::ReadinessConfig,
1790    reporter: &mut dyn Reporter,
1791    event_log: &mut events::EventLog,
1792    events_path: &Path,
1793    pkg_label: &str,
1794) -> Result<(bool, Vec<ReadinessEvidence>)> {
1795    reporter.info(&format!(
1796        "{}@{}: readiness check ({:?})...",
1797        crate_name, version, config.method
1798    ));
1799    let started_at = Instant::now();
1800    record_readiness_event(
1801        event_log,
1802        events_path,
1803        PublishEvent {
1804            timestamp: Utc::now(),
1805            event_type: EventType::ReadinessStarted {
1806                method: config.method,
1807            },
1808            package: pkg_label.to_string(),
1809        },
1810    )?;
1811    let mut emit_event = |event| record_readiness_event(event_log, events_path, event);
1812    let (visible, evidence) = reg.is_version_visible_with_backoff_and_events(
1813        crate_name,
1814        version,
1815        config,
1816        &mut emit_event,
1817    )?;
1818    if visible {
1819        reporter.info(&format!(
1820            "{}@{}: visible after {} checks",
1821            crate_name,
1822            version,
1823            evidence.len()
1824        ));
1825        record_readiness_event(
1826            event_log,
1827            events_path,
1828            PublishEvent {
1829                timestamp: Utc::now(),
1830                event_type: EventType::ReadinessComplete {
1831                    duration_ms: started_at.elapsed().as_millis() as u64,
1832                    attempts: evidence.len() as u32,
1833                },
1834                package: pkg_label.to_string(),
1835            },
1836        )?;
1837    } else {
1838        reporter.warn(&format!(
1839            "{}@{}: not visible after {} checks",
1840            crate_name,
1841            version,
1842            evidence.len()
1843        ));
1844        record_readiness_event(
1845            event_log,
1846            events_path,
1847            PublishEvent {
1848                timestamp: Utc::now(),
1849                event_type: EventType::ReadinessTimeout {
1850                    max_wait_ms: config.max_total_wait.as_millis() as u64,
1851                },
1852                package: pkg_label.to_string(),
1853            },
1854        )?;
1855    }
1856    Ok((visible, evidence))
1857}
1858
1859fn record_readiness_event(
1860    event_log: &mut events::EventLog,
1861    events_path: &Path,
1862    event: PublishEvent,
1863) -> Result<()> {
1864    event_log.record(event);
1865    event_log.write_to_file(events_path)?;
1866    event_log.clear();
1867    Ok(())
1868}
1869
1870#[cfg(test)]
1871mod tests {
1872    use std::fs;
1873    use std::path::{Path, PathBuf};
1874    use std::sync::{Arc, Mutex};
1875    use std::thread;
1876    use std::time::Duration;
1877
1878    use chrono::Utc;
1879    use serial_test::serial;
1880    use tempfile::tempdir;
1881    use tiny_http::{Header, Response, Server, StatusCode};
1882
1883    use super::*;
1884    use crate::plan::PlannedWorkspace;
1885    use crate::types::{AuthEvidenceMode, AuthType, PlannedPackage, Registry, ReleasePlan};
1886
1887    #[derive(Default)]
1888    struct CollectingReporter {
1889        infos: Vec<String>,
1890        warns: Vec<String>,
1891        errors: Vec<String>,
1892    }
1893
1894    impl Reporter for CollectingReporter {
1895        fn info(&mut self, msg: &str) {
1896            self.infos.push(msg.to_string());
1897        }
1898
1899        fn warn(&mut self, msg: &str) {
1900            self.warns.push(msg.to_string());
1901        }
1902
1903        fn error(&mut self, msg: &str) {
1904            self.errors.push(msg.to_string());
1905        }
1906    }
1907
1908    #[cfg(windows)]
1909    fn fake_cargo_path(bin_dir: &Path) -> PathBuf {
1910        bin_dir.join("cargo.cmd")
1911    }
1912
1913    #[cfg(not(windows))]
1914    fn fake_cargo_path(bin_dir: &Path) -> PathBuf {
1915        bin_dir.join("cargo")
1916    }
1917
1918    #[cfg(windows)]
1919    fn fake_git_path(bin_dir: &Path) -> PathBuf {
1920        bin_dir.join("git.cmd")
1921    }
1922
1923    #[cfg(not(windows))]
1924    fn fake_git_path(bin_dir: &Path) -> PathBuf {
1925        bin_dir.join("git")
1926    }
1927
1928    fn fake_program_env_vars(bin_dir: &Path) -> Vec<(&'static str, Option<String>)> {
1929        vec![
1930            (
1931                "SHIPPER_CARGO_BIN",
1932                Some(fake_cargo_path(bin_dir).to_str().expect("utf8").to_string()),
1933            ),
1934            (
1935                "SHIPPER_GIT_BIN",
1936                Some(fake_git_path(bin_dir).to_str().expect("utf8").to_string()),
1937            ),
1938        ]
1939    }
1940
1941    /// Build a combined env var list from fake programs + additional vars, then run closure.
1942    fn with_test_env<F, R>(bin_dir: &Path, extra: Vec<(&'static str, Option<String>)>, f: F) -> R
1943    where
1944        F: FnOnce() -> R,
1945    {
1946        let mut vars = fake_program_env_vars(bin_dir);
1947        vars.extend(extra);
1948        temp_env::with_vars(vars, f)
1949    }
1950
1951    fn write_fake_cargo(bin_dir: &Path) {
1952        #[cfg(windows)]
1953        {
1954            fs::write(
1955                bin_dir.join("cargo.cmd"),
1956                "@echo off\r\nif not \"%SHIPPER_CARGO_ARGS_LOG%\"==\"\" echo %*>>\"%SHIPPER_CARGO_ARGS_LOG%\"\r\nif not \"%SHIPPER_CARGO_STDOUT%\"==\"\" echo %SHIPPER_CARGO_STDOUT%\r\nif not \"%SHIPPER_CARGO_STDERR%\"==\"\" echo %SHIPPER_CARGO_STDERR% 1>&2\r\nif \"%SHIPPER_CARGO_EXIT%\"==\"\" (exit /b 0) else (exit /b %SHIPPER_CARGO_EXIT%)\r\n",
1957            )
1958            .expect("write fake cargo");
1959        }
1960
1961        #[cfg(not(windows))]
1962        {
1963            use std::os::unix::fs::PermissionsExt;
1964
1965            let path = bin_dir.join("cargo");
1966            fs::write(
1967                &path,
1968                "#!/usr/bin/env sh\nif [ -n \"$SHIPPER_CARGO_ARGS_LOG\" ]; then\n  echo \"$*\" >>\"$SHIPPER_CARGO_ARGS_LOG\"\nfi\nif [ -n \"$SHIPPER_CARGO_STDOUT\" ]; then\n  echo \"$SHIPPER_CARGO_STDOUT\"\nfi\nif [ -n \"$SHIPPER_CARGO_STDERR\" ]; then\n  echo \"$SHIPPER_CARGO_STDERR\" >&2\nfi\nexit \"${SHIPPER_CARGO_EXIT:-0}\"\n",
1969            )
1970            .expect("write fake cargo");
1971            let mut perms = fs::metadata(&path).expect("meta").permissions();
1972            perms.set_mode(0o755);
1973            fs::set_permissions(&path, perms).expect("chmod");
1974        }
1975    }
1976
1977    fn write_fake_cargo_ambiguous_then_permanent(bin_dir: &Path) {
1978        #[cfg(windows)]
1979        {
1980            fs::write(
1981                bin_dir.join("cargo.cmd"),
1982                "@echo off\r\nif not \"%SHIPPER_CARGO_ARGS_LOG%\"==\"\" echo %*>>\"%SHIPPER_CARGO_ARGS_LOG%\"\r\nset COUNT_FILE=%SHIPPER_CARGO_COUNT_FILE%\r\nif \"%COUNT_FILE%\"==\"\" set COUNT_FILE=%TEMP%\\shipper-cargo-count.txt\r\nif not exist \"%COUNT_FILE%\" (\r\n  echo 1>\"%COUNT_FILE%\"\r\n  exit /b 1\r\n)\r\necho crate version 0.1.0 is already uploaded 1>&2\r\nexit /b 1\r\n",
1983            )
1984            .expect("write sequenced fake cargo");
1985        }
1986
1987        #[cfg(not(windows))]
1988        {
1989            use std::os::unix::fs::PermissionsExt;
1990
1991            let path = bin_dir.join("cargo");
1992            fs::write(
1993                &path,
1994                "#!/usr/bin/env sh\nif [ -n \"$SHIPPER_CARGO_ARGS_LOG\" ]; then\n  echo \"$*\" >>\"$SHIPPER_CARGO_ARGS_LOG\"\nfi\ncount_file=\"${SHIPPER_CARGO_COUNT_FILE:-${TMPDIR:-/tmp}/shipper-cargo-count.txt}\"\nif [ ! -f \"$count_file\" ]; then\n  echo 1 >\"$count_file\"\n  exit 1\nfi\necho 'crate version 0.1.0 is already uploaded' >&2\nexit 1\n",
1995            )
1996            .expect("write sequenced fake cargo");
1997            let mut perms = fs::metadata(&path).expect("meta").permissions();
1998            perms.set_mode(0o755);
1999            fs::set_permissions(&path, perms).expect("chmod");
2000        }
2001    }
2002
2003    fn write_fake_git(bin_dir: &Path) {
2004        #[cfg(windows)]
2005        {
2006            fs::write(
2007                bin_dir.join("git.cmd"),
2008                "@echo off\r\nif \"%SHIPPER_GIT_FAIL%\"==\"1\" (\r\n  echo fatal: git failed 1>&2\r\n  exit /b 1\r\n)\r\nif \"%SHIPPER_GIT_CLEAN%\"==\"0\" (\r\n  echo M src/lib.rs\r\n  exit /b 0\r\n)\r\nexit /b 0\r\n",
2009            )
2010            .expect("write fake git");
2011        }
2012
2013        #[cfg(not(windows))]
2014        {
2015            use std::os::unix::fs::PermissionsExt;
2016
2017            let path = bin_dir.join("git");
2018            fs::write(
2019                &path,
2020                "#!/usr/bin/env sh\nif [ \"${SHIPPER_GIT_FAIL:-0}\" = \"1\" ]; then\n  echo 'fatal: git failed' >&2\n  exit 1\nfi\nif [ \"${SHIPPER_GIT_CLEAN:-1}\" = \"0\" ]; then\n  echo 'M src/lib.rs'\nfi\nexit 0\n",
2021            )
2022            .expect("write fake git");
2023            let mut perms = fs::metadata(&path).expect("meta").permissions();
2024            perms.set_mode(0o755);
2025            fs::set_permissions(&path, perms).expect("chmod");
2026        }
2027    }
2028
2029    fn write_fake_tools(bin_dir: &Path) {
2030        fs::create_dir_all(bin_dir).expect("mkdir");
2031        write_fake_cargo(bin_dir);
2032        write_fake_git(bin_dir);
2033    }
2034
2035    struct TestRegistryServer {
2036        base_url: String,
2037        #[allow(clippy::type_complexity)]
2038        seen: Arc<Mutex<Vec<(String, Option<String>)>>>,
2039        handle: thread::JoinHandle<()>,
2040    }
2041
2042    impl TestRegistryServer {
2043        fn join(self) {
2044            self.handle.join().expect("join server");
2045        }
2046    }
2047
2048    fn spawn_registry_server(
2049        mut routes: std::collections::BTreeMap<String, Vec<(u16, String)>>,
2050        expected_requests: usize,
2051    ) -> TestRegistryServer {
2052        let server = Server::http("127.0.0.1:0").expect("server");
2053        let base_url = format!("http://{}", server.server_addr());
2054        let seen = Arc::new(Mutex::new(Vec::<(String, Option<String>)>::new()));
2055        let seen_thread = Arc::clone(&seen);
2056
2057        let handle = thread::spawn(move || {
2058            for _ in 0..expected_requests {
2059                let req = match server.recv_timeout(Duration::from_secs(30)) {
2060                    Ok(Some(r)) => r,
2061                    _ => break,
2062                };
2063                let path = req.url().to_string();
2064                let auth = req
2065                    .headers()
2066                    .iter()
2067                    .find(|h| h.field.equiv("Authorization"))
2068                    .map(|h| h.value.as_str().to_string());
2069                seen_thread.lock().expect("lock").push((path.clone(), auth));
2070
2071                let response = if let Some(list) = routes.get_mut(&path) {
2072                    if list.is_empty() {
2073                        (404, "{}".to_string())
2074                    } else if list.len() == 1 {
2075                        list[0].clone()
2076                    } else {
2077                        list.remove(0)
2078                    }
2079                } else {
2080                    (404, "{}".to_string())
2081                };
2082
2083                let resp = Response::from_string(response.1)
2084                    .with_status_code(StatusCode(response.0))
2085                    .with_header(
2086                        Header::from_bytes("Content-Type", "application/json").expect("header"),
2087                    );
2088                req.respond(resp).expect("respond");
2089            }
2090        });
2091
2092        TestRegistryServer {
2093            base_url,
2094            seen,
2095            handle,
2096        }
2097    }
2098
2099    fn planned_workspace(workspace_root: &Path, api_base: String) -> PlannedWorkspace {
2100        PlannedWorkspace {
2101            workspace_root: workspace_root.to_path_buf(),
2102            plan: ReleasePlan {
2103                plan_version: "1".to_string(),
2104                plan_id: "plan-demo".to_string(),
2105                created_at: Utc::now(),
2106                registry: Registry {
2107                    name: "crates-io".to_string(),
2108                    api_base,
2109                    index_base: None,
2110                },
2111                packages: vec![PlannedPackage {
2112                    name: "demo".to_string(),
2113                    version: "0.1.0".to_string(),
2114                    manifest_path: workspace_root.join("demo").join("Cargo.toml"),
2115                    regime: None,
2116                }],
2117                dependencies: std::collections::BTreeMap::new(),
2118            },
2119            skipped: vec![],
2120        }
2121    }
2122
2123    fn default_opts(state_dir: PathBuf) -> RuntimeOptions {
2124        RuntimeOptions {
2125            allow_dirty: true,
2126            skip_ownership_check: true,
2127            strict_ownership: false,
2128            no_verify: false,
2129            max_attempts: 2,
2130            base_delay: Duration::from_millis(1),
2131            max_delay: Duration::from_millis(2),
2132            verify_timeout: Duration::from_millis(20),
2133            verify_poll_interval: Duration::from_millis(1),
2134            state_dir,
2135            force_resume: false,
2136            policy: crate::types::PublishPolicy::default(),
2137            verify_mode: crate::types::VerifyMode::default(),
2138            readiness: crate::types::ReadinessConfig {
2139                enabled: true,
2140                method: crate::types::ReadinessMethod::Api,
2141                initial_delay: Duration::from_millis(0),
2142                max_delay: Duration::from_millis(20),
2143                max_total_wait: Duration::from_millis(200),
2144                poll_interval: Duration::from_millis(1),
2145                jitter_factor: 0.0,
2146                index_path: None,
2147                prefer_index: false,
2148            },
2149            output_lines: 100,
2150            force: false,
2151            lock_timeout: Duration::from_secs(3600),
2152            parallel: crate::types::ParallelConfig::default(),
2153            webhook: crate::webhook::WebhookConfig::default(),
2154            retry_strategy: crate::retry::RetryStrategyType::Exponential,
2155            retry_jitter: 0.0,
2156            retry_per_error: crate::retry::PerErrorConfig::default(),
2157            encryption: crate::encryption::EncryptionConfig::default(),
2158            registries: vec![],
2159            resume_from: None,
2160            rehearsal_registry: None,
2161            rehearsal_skip: false,
2162            rehearsal_smoke_install: None,
2163        }
2164    }
2165
2166    #[test]
2167    fn classify_cargo_failure_covers_retryable_permanent_and_ambiguous() {
2168        let retryable = classify_cargo_failure("HTTP 429 too many requests", "");
2169        assert_eq!(retryable.0, ErrorClass::Retryable);
2170
2171        let permanent = classify_cargo_failure("permission denied", "");
2172        assert_eq!(permanent.0, ErrorClass::Permanent);
2173
2174        let ambiguous = classify_cargo_failure("strange output", "");
2175        assert_eq!(ambiguous.0, ErrorClass::Ambiguous);
2176    }
2177
2178    #[test]
2179    fn collecting_reporter_error_method_records_message() {
2180        let mut reporter = CollectingReporter::default();
2181        reporter.error("boom");
2182        assert_eq!(reporter.errors, vec!["boom".to_string()]);
2183    }
2184
2185    #[test]
2186    fn helper_functions_return_expected_values() {
2187        let root = PathBuf::from("root");
2188        let rel = resolve_state_dir(&root, &PathBuf::from(".shipper"));
2189        assert_eq!(rel, root.join(".shipper"));
2190
2191        #[cfg(windows)]
2192        {
2193            let abs = PathBuf::from(r"C:\x\state");
2194            assert_eq!(resolve_state_dir(&root, &abs), abs);
2195        }
2196        #[cfg(not(windows))]
2197        {
2198            let abs = PathBuf::from("/x/state");
2199            assert_eq!(resolve_state_dir(&root, &abs), abs);
2200        }
2201
2202        assert_eq!(pkg_key("a", "1.2.3"), "a@1.2.3");
2203        assert_eq!(short_state(&PackageState::Pending), "pending");
2204        assert_eq!(short_state(&PackageState::Uploaded), "uploaded");
2205        assert_eq!(short_state(&PackageState::Published), "published");
2206        assert_eq!(
2207            short_state(&PackageState::Skipped {
2208                reason: "r".to_string()
2209            }),
2210            "skipped"
2211        );
2212        assert_eq!(
2213            short_state(&PackageState::Failed {
2214                class: ErrorClass::Permanent,
2215                message: "m".to_string()
2216            }),
2217            "failed"
2218        );
2219        assert_eq!(
2220            short_state(&PackageState::Ambiguous {
2221                message: "m".to_string()
2222            }),
2223            "ambiguous"
2224        );
2225    }
2226
2227    #[test]
2228    fn backoff_delay_is_bounded_with_jitter() {
2229        let base = Duration::from_millis(100);
2230        let max = Duration::from_millis(500);
2231        let d1 = backoff_delay(
2232            base,
2233            max,
2234            1,
2235            crate::retry::RetryStrategyType::Exponential,
2236            0.5,
2237        );
2238        let d20 = backoff_delay(
2239            base,
2240            max,
2241            20,
2242            crate::retry::RetryStrategyType::Exponential,
2243            0.5,
2244        );
2245
2246        assert!(d1 >= Duration::from_millis(50));
2247        assert!(d1 <= Duration::from_millis(150));
2248
2249        assert!(d20 >= Duration::from_millis(250));
2250        assert!(d20 <= Duration::from_millis(750));
2251    }
2252
2253    #[test]
2254    fn verify_published_returns_true_when_registry_visibility_appears() {
2255        let server = spawn_registry_server(
2256            std::collections::BTreeMap::from([(
2257                "/api/v1/crates/demo/0.1.0".to_string(),
2258                vec![(404, "{}".to_string()), (200, "{}".to_string())],
2259            )]),
2260            2,
2261        );
2262
2263        let reg = RegistryClient::new(Registry {
2264            name: "crates-io".to_string(),
2265            api_base: server.base_url.clone(),
2266            index_base: None,
2267        })
2268        .expect("client");
2269
2270        let config = crate::types::ReadinessConfig {
2271            enabled: true,
2272            method: crate::types::ReadinessMethod::Api,
2273            initial_delay: Duration::from_millis(0),
2274            max_delay: Duration::from_millis(50),
2275            // Keep this generous to avoid timing flakes under highly parallel test execution.
2276            max_total_wait: Duration::from_secs(2),
2277            poll_interval: Duration::from_millis(1),
2278            jitter_factor: 0.0,
2279            index_path: None,
2280            prefer_index: false,
2281        };
2282
2283        let mut reporter = CollectingReporter::default();
2284        let td = tempdir().expect("tempdir");
2285        let events_path = td.path().join("events.jsonl");
2286        let mut event_log = events::EventLog::new();
2287        let (ok, evidence) = verify_published(
2288            &reg,
2289            "demo",
2290            "0.1.0",
2291            &config,
2292            &mut reporter,
2293            &mut event_log,
2294            &events_path,
2295            "demo@0.1.0",
2296        )
2297        .expect("verify");
2298        assert!(ok);
2299        assert!(!reporter.infos.is_empty());
2300        assert!(!evidence.is_empty());
2301        server.join();
2302    }
2303
2304    #[test]
2305    fn verify_published_returns_false_on_timeout() {
2306        let reg = RegistryClient::new(Registry {
2307            name: "crates-io".to_string(),
2308            api_base: "http://127.0.0.1:9".to_string(),
2309            index_base: None,
2310        })
2311        .expect("client");
2312
2313        let config = crate::types::ReadinessConfig {
2314            enabled: true,
2315            method: crate::types::ReadinessMethod::Api,
2316            initial_delay: Duration::from_millis(0),
2317            max_delay: Duration::from_millis(10),
2318            max_total_wait: Duration::from_millis(0),
2319            poll_interval: Duration::from_millis(1),
2320            jitter_factor: 0.0,
2321            index_path: None,
2322            prefer_index: false,
2323        };
2324
2325        let mut reporter = CollectingReporter::default();
2326        let td = tempdir().expect("tempdir");
2327        let events_path = td.path().join("events.jsonl");
2328        let mut event_log = events::EventLog::new();
2329        let (ok, _evidence) = verify_published(
2330            &reg,
2331            "demo",
2332            "0.1.0",
2333            &config,
2334            &mut reporter,
2335            &mut event_log,
2336            &events_path,
2337            "demo@0.1.0",
2338        )
2339        .expect("verify");
2340        assert!(!ok);
2341    }
2342
2343    #[test]
2344    fn registry_server_helper_returns_404_for_unknown_or_empty_routes() {
2345        let server_unknown = spawn_registry_server(std::collections::BTreeMap::new(), 1);
2346        let reg_unknown = RegistryClient::new(Registry {
2347            name: "crates-io".to_string(),
2348            api_base: server_unknown.base_url.clone(),
2349            index_base: None,
2350        })
2351        .expect("client");
2352        let exists_unknown = reg_unknown
2353            .version_exists("demo", "0.1.0")
2354            .expect("version exists");
2355        assert!(!exists_unknown);
2356        server_unknown.join();
2357
2358        let server_empty = spawn_registry_server(
2359            std::collections::BTreeMap::from([("/api/v1/crates/demo/0.1.0".to_string(), vec![])]),
2360            1,
2361        );
2362        let reg_empty = RegistryClient::new(Registry {
2363            name: "crates-io".to_string(),
2364            api_base: server_empty.base_url.clone(),
2365            index_base: None,
2366        })
2367        .expect("client");
2368        let exists_empty = reg_empty
2369            .version_exists("demo", "0.1.0")
2370            .expect("version exists");
2371        assert!(!exists_empty);
2372        server_empty.join();
2373    }
2374
2375    #[test]
2376    #[serial]
2377    fn run_preflight_errors_in_strict_mode_without_token() {
2378        let td = tempdir().expect("tempdir");
2379        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
2380        let mut opts = default_opts(PathBuf::from(".shipper"));
2381        opts.strict_ownership = true;
2382        opts.skip_ownership_check = false;
2383        temp_env::with_vars(
2384            [
2385                (
2386                    "CARGO_HOME",
2387                    Some(td.path().to_str().expect("utf8").to_string()),
2388                ),
2389                ("CARGO_REGISTRY_TOKEN", None::<String>),
2390                ("CARGO_REGISTRIES_CRATES_IO_TOKEN", None::<String>),
2391            ],
2392            || {
2393                let mut reporter = CollectingReporter::default();
2394                let err = run_preflight(&ws, &opts, &mut reporter).expect_err("must fail");
2395                assert!(
2396                    format!("{err:#}").contains("strict ownership requested but no token found")
2397                );
2398            },
2399        );
2400    }
2401
2402    #[test]
2403    #[serial]
2404    fn run_preflight_warns_on_owners_failure_when_not_strict() {
2405        let td = tempdir().expect("tempdir");
2406        let bin = td.path().join("bin");
2407        write_fake_tools(&bin);
2408        let mut env_vars = fake_program_env_vars(&bin);
2409        env_vars.extend([
2410            ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
2411            (
2412                "CARGO_HOME",
2413                Some(td.path().to_str().expect("utf8").to_string()),
2414            ),
2415            ("CARGO_REGISTRY_TOKEN", Some("token-abc".to_string())),
2416        ]);
2417        temp_env::with_vars(env_vars, || {
2418            let server = spawn_registry_server(
2419                std::collections::BTreeMap::from([
2420                    (
2421                        "/api/v1/crates/demo/0.1.0".to_string(),
2422                        vec![(404, "{}".to_string())],
2423                    ),
2424                    (
2425                        "/api/v1/crates/demo".to_string(),
2426                        vec![(404, "{}".to_string())],
2427                    ),
2428                    (
2429                        "/api/v1/crates/demo/owners".to_string(),
2430                        vec![(403, "{}".to_string())],
2431                    ),
2432                ]),
2433                3,
2434            );
2435
2436            let ws = planned_workspace(td.path(), server.base_url.clone());
2437            let mut opts = default_opts(PathBuf::from(".shipper"));
2438            opts.skip_ownership_check = false;
2439            opts.strict_ownership = false;
2440
2441            let mut reporter = CollectingReporter::default();
2442            let rep = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
2443            assert!(rep.token_detected);
2444            assert_eq!(rep.packages.len(), 1);
2445            assert!(!rep.packages[0].already_published);
2446            assert!(!rep.packages[0].ownership_verified);
2447            assert!(rep.packages[0].dry_run_passed);
2448            assert_eq!(rep.finishability, Finishability::NotProven);
2449            assert!(
2450                reporter
2451                    .warns
2452                    .iter()
2453                    .any(|w| w.contains("owners preflight failed"))
2454            );
2455
2456            let seen = server.seen.lock().expect("lock");
2457            assert_eq!(seen.len(), 3);
2458            drop(seen);
2459            server.join();
2460        });
2461    }
2462
2463    #[test]
2464    #[serial]
2465    fn run_preflight_owners_success_path() {
2466        let td = tempdir().expect("tempdir");
2467        let bin = td.path().join("bin");
2468        write_fake_tools(&bin);
2469        let mut env_vars = fake_program_env_vars(&bin);
2470        env_vars.extend([
2471            ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
2472            (
2473                "CARGO_HOME",
2474                Some(td.path().to_str().expect("utf8").to_string()),
2475            ),
2476            ("CARGO_REGISTRY_TOKEN", Some("token-abc".to_string())),
2477        ]);
2478        temp_env::with_vars(env_vars, || {
2479            let server = spawn_registry_server(
2480                std::collections::BTreeMap::from([
2481                    (
2482                        "/api/v1/crates/demo/0.1.0".to_string(),
2483                        vec![(404, "{}".to_string())],
2484                    ),
2485                    (
2486                        "/api/v1/crates/demo".to_string(),
2487                        vec![(200, "{}".to_string())],
2488                    ),
2489                    (
2490                        "/api/v1/crates/demo/owners".to_string(),
2491                        vec![(
2492                            200,
2493                            r#"{"users":[{"id":1,"login":"alice","name":"Alice"}]}"#.to_string(),
2494                        )],
2495                    ),
2496                ]),
2497                3,
2498            );
2499
2500            let ws = planned_workspace(td.path(), server.base_url.clone());
2501            let mut opts = default_opts(PathBuf::from(".shipper"));
2502            opts.skip_ownership_check = false;
2503            opts.strict_ownership = false;
2504
2505            let mut reporter = CollectingReporter::default();
2506            let rep = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
2507            assert_eq!(rep.packages.len(), 1);
2508            assert!(reporter.warns.is_empty());
2509            server.join();
2510        });
2511    }
2512
2513    #[test]
2514    #[serial]
2515    fn run_preflight_returns_error_when_strict_ownership_check_fails() {
2516        let td = tempdir().expect("tempdir");
2517        let bin = td.path().join("bin");
2518        write_fake_tools(&bin);
2519        let mut env_vars = fake_program_env_vars(&bin);
2520        env_vars.extend([
2521            ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
2522            (
2523                "CARGO_HOME",
2524                Some(td.path().to_str().expect("utf8").to_string()),
2525            ),
2526            ("CARGO_REGISTRY_TOKEN", Some("token-abc".to_string())),
2527        ]);
2528        temp_env::with_vars(env_vars, || {
2529            // Crate must exist (200) so ownership check is actually attempted;
2530            // 404 would mean new crate -> ownership check skipped.
2531            let server = spawn_registry_server(
2532                std::collections::BTreeMap::from([
2533                    (
2534                        "/api/v1/crates/demo/0.1.0".to_string(),
2535                        vec![(404, "{}".to_string())],
2536                    ),
2537                    (
2538                        "/api/v1/crates/demo".to_string(),
2539                        vec![(200, "{}".to_string())],
2540                    ),
2541                    (
2542                        "/api/v1/crates/demo/owners".to_string(),
2543                        vec![(403, "{}".to_string())],
2544                    ),
2545                ]),
2546                3,
2547            );
2548
2549            let ws = planned_workspace(td.path(), server.base_url.clone());
2550            let mut opts = default_opts(PathBuf::from(".shipper"));
2551            opts.skip_ownership_check = false;
2552            opts.strict_ownership = true;
2553
2554            let mut reporter = CollectingReporter::default();
2555            let err = run_preflight(&ws, &opts, &mut reporter).expect_err("must fail");
2556            assert!(format!("{err:#}").contains("forbidden when querying owners"));
2557            server.join();
2558        });
2559    }
2560
2561    #[test]
2562    #[serial]
2563    fn run_preflight_strict_skips_ownership_for_new_crate() {
2564        let td = tempdir().expect("tempdir");
2565        let bin = td.path().join("bin");
2566        write_fake_tools(&bin);
2567        let mut env_vars = fake_program_env_vars(&bin);
2568        env_vars.extend([
2569            ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
2570            (
2571                "CARGO_HOME",
2572                Some(td.path().to_str().expect("utf8").to_string()),
2573            ),
2574            ("CARGO_REGISTRY_TOKEN", Some("token-abc".to_string())),
2575        ]);
2576        temp_env::with_vars(env_vars, || {
2577            // Crate returns 404 (new crate) -- ownership check should be skipped.
2578            // No /owners endpoint needed.
2579            let server = spawn_registry_server(
2580                std::collections::BTreeMap::from([
2581                    (
2582                        "/api/v1/crates/demo/0.1.0".to_string(),
2583                        vec![(404, "{}".to_string())],
2584                    ),
2585                    (
2586                        "/api/v1/crates/demo".to_string(),
2587                        vec![(404, "{}".to_string())],
2588                    ),
2589                ]),
2590                2,
2591            );
2592
2593            let mut ws = planned_workspace(td.path(), server.base_url.clone());
2594            let mut opts = default_opts(PathBuf::from(".shipper"));
2595            opts.skip_ownership_check = false;
2596            opts.strict_ownership = true;
2597
2598            let mut reporter = CollectingReporter::default();
2599            let rep = run_preflight_in_place(&mut ws, &opts, &mut reporter).expect("preflight");
2600            assert_eq!(rep.packages.len(), 1);
2601            assert!(!rep.packages[0].ownership_verified);
2602            assert!(rep.packages[0].is_new_crate);
2603            assert!(
2604                reporter
2605                    .infos
2606                    .iter()
2607                    .any(|i| i.contains("new crate, skipping ownership check"))
2608            );
2609            // #106 PR 1: preflight must stamp PublishRegime::FirstPublish
2610            // on the plan so the downstream publish retry loop can
2611            // consume it without re-querying the registry.
2612            assert_eq!(
2613                ws.plan.packages[0].regime,
2614                Some(PublishRegime::FirstPublish),
2615                "preflight should stamp FirstPublish regime on new crates"
2616            );
2617            server.join();
2618        });
2619    }
2620
2621    #[test]
2622    #[serial]
2623    fn run_preflight_legacy_entry_point_preserves_immutable_api() {
2624        let td = tempdir().expect("tempdir");
2625        let bin = td.path().join("bin");
2626        write_fake_tools(&bin);
2627        let mut env_vars = fake_program_env_vars(&bin);
2628        env_vars.extend([("SHIPPER_CARGO_EXIT", Some("0".to_string()))]);
2629        temp_env::with_vars(env_vars, || {
2630            let server = spawn_registry_server(
2631                std::collections::BTreeMap::from([
2632                    (
2633                        "/api/v1/crates/demo/0.1.0".to_string(),
2634                        vec![(404, "{}".to_string())],
2635                    ),
2636                    (
2637                        "/api/v1/crates/demo".to_string(),
2638                        vec![(404, "{}".to_string())],
2639                    ),
2640                ]),
2641                2,
2642            );
2643
2644            let ws = planned_workspace(td.path(), server.base_url.clone());
2645            let opts = default_opts(PathBuf::from(".shipper"));
2646            let mut reporter = CollectingReporter::default();
2647
2648            let rep = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
2649
2650            assert_eq!(rep.packages.len(), 1);
2651            assert!(rep.packages[0].is_new_crate);
2652            assert_eq!(
2653                ws.plan.packages[0].regime, None,
2654                "legacy immutable API should not mutate the caller's plan"
2655            );
2656            server.join();
2657        });
2658    }
2659
2660    /// #106 PR 1: preflight stamps `PublishRegime::Update` on crates
2661    /// that already have at least one published version. Ensures the
2662    /// downstream retry loop can distinguish update vs. first-publish
2663    /// backoff windows without re-querying the registry.
2664    #[test]
2665    #[serial]
2666    fn run_preflight_stamps_update_regime_on_existing_crate() {
2667        let td = tempdir().expect("tempdir");
2668        let bin = td.path().join("bin");
2669        write_fake_tools(&bin);
2670        let mut env_vars = fake_program_env_vars(&bin);
2671        env_vars.extend([
2672            ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
2673            (
2674                "CARGO_HOME",
2675                Some(td.path().to_str().expect("utf8").to_string()),
2676            ),
2677        ]);
2678        temp_env::with_vars(env_vars, || {
2679            // Crate root returns 200 (crate exists), specific version 404
2680            // (this version is not yet published). This is the canonical
2681            // "publishing a new version of an existing crate" preflight.
2682            let server = spawn_registry_server(
2683                std::collections::BTreeMap::from([
2684                    (
2685                        "/api/v1/crates/demo/0.1.0".to_string(),
2686                        vec![(404, "{}".to_string())],
2687                    ),
2688                    (
2689                        "/api/v1/crates/demo".to_string(),
2690                        vec![(200, r#"{"crate":{"name":"demo"}}"#.to_string())],
2691                    ),
2692                ]),
2693                2,
2694            );
2695
2696            let mut ws = planned_workspace(td.path(), server.base_url.clone());
2697            let mut opts = default_opts(PathBuf::from(".shipper"));
2698            opts.skip_ownership_check = true;
2699            opts.strict_ownership = false;
2700
2701            let mut reporter = CollectingReporter::default();
2702            let rep = run_preflight_in_place(&mut ws, &opts, &mut reporter).expect("preflight");
2703            assert_eq!(rep.packages.len(), 1);
2704            assert!(!rep.packages[0].is_new_crate);
2705            assert_eq!(
2706                ws.plan.packages[0].regime,
2707                Some(PublishRegime::Update),
2708                "preflight should stamp Update regime on existing crates"
2709            );
2710            server.join();
2711        });
2712    }
2713
2714    #[test]
2715    #[serial]
2716    fn run_preflight_writes_preflight_events() {
2717        let td = tempdir().expect("tempdir");
2718        let bin = td.path().join("bin");
2719        write_fake_tools(&bin);
2720        let mut env_vars = fake_program_env_vars(&bin);
2721        env_vars.extend([("SHIPPER_CARGO_EXIT", Some("0".to_string()))]);
2722        temp_env::with_vars(env_vars, || {
2723            let server = spawn_registry_server(
2724                std::collections::BTreeMap::from([
2725                    (
2726                        "/api/v1/crates/demo/0.1.0".to_string(),
2727                        vec![(404, "{}".to_string())],
2728                    ),
2729                    (
2730                        "/api/v1/crates/demo".to_string(),
2731                        vec![(404, "{}".to_string())],
2732                    ),
2733                ]),
2734                2,
2735            );
2736
2737            let ws = planned_workspace(td.path(), server.base_url.clone());
2738            let mut opts = default_opts(PathBuf::from(".shipper"));
2739            opts.allow_dirty = true;
2740            opts.skip_ownership_check = true;
2741
2742            let mut reporter = CollectingReporter::default();
2743            let _ = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
2744
2745            let events_path = td.path().join(".shipper").join("events.jsonl");
2746            let log =
2747                crate::state::events::EventLog::read_from_file(&events_path).expect("read events");
2748            let events = log.all_events();
2749
2750            assert!(
2751                events
2752                    .iter()
2753                    .any(|e| matches!(e.event_type, EventType::PreflightStarted))
2754            );
2755            assert!(
2756                events
2757                    .iter()
2758                    .any(|e| matches!(e.event_type, EventType::PreflightWorkspaceVerify { .. }))
2759            );
2760            assert!(
2761                events.iter().any(|e| {
2762                    matches!(e.event_type, EventType::PreflightNewCrateDetected { .. })
2763                })
2764            );
2765            assert!(
2766                events
2767                    .iter()
2768                    .any(|e| matches!(e.event_type, EventType::PreflightOwnershipCheck { .. }))
2769            );
2770            assert!(
2771                events
2772                    .iter()
2773                    .any(|e| matches!(e.event_type, EventType::PreflightComplete { .. }))
2774            );
2775            server.join();
2776        });
2777    }
2778
2779    // #100 — fresh-audit mode must not read or append the authoritative
2780    // `events.jsonl`. We seed a pre-existing `events.jsonl` with a bogus
2781    // event that would fail deserialization, run preflight in
2782    // `fresh_audit` mode, and assert: (a) the authoritative log is
2783    // byte-identical to the seed afterward, and (b) the preflight trace
2784    // lands in the session-isolated sidecar instead.
2785    #[test]
2786    #[serial]
2787    fn run_preflight_fresh_audit_ignores_prior_state() {
2788        let td = tempdir().expect("tempdir");
2789        let bin = td.path().join("bin");
2790        write_fake_tools(&bin);
2791        let mut env_vars = fake_program_env_vars(&bin);
2792        env_vars.extend([("SHIPPER_CARGO_EXIT", Some("0".to_string()))]);
2793        temp_env::with_vars(env_vars, || {
2794            let server = spawn_registry_server(
2795                std::collections::BTreeMap::from([
2796                    (
2797                        "/api/v1/crates/demo/0.1.0".to_string(),
2798                        vec![(404, "{}".to_string())],
2799                    ),
2800                    (
2801                        "/api/v1/crates/demo".to_string(),
2802                        vec![(404, "{}".to_string())],
2803                    ),
2804                ]),
2805                2,
2806            );
2807
2808            let ws = planned_workspace(td.path(), server.base_url.clone());
2809            let mut opts = default_opts(PathBuf::from(".shipper"));
2810            opts.allow_dirty = true;
2811            opts.skip_ownership_check = true;
2812
2813            // Seed a sentinel `events.jsonl`. If fresh-audit touches it at
2814            // all (read or append), we'll see drift after the run.
2815            let authoritative_events = td.path().join(".shipper").join("events.jsonl");
2816            std::fs::create_dir_all(authoritative_events.parent().unwrap()).expect("mkdir");
2817            let sentinel = "{\"sentinel\":\"do not touch\"}\n";
2818            std::fs::write(&authoritative_events, sentinel).expect("seed events");
2819
2820            // Seed a state.json too — fresh-audit must not produce or
2821            // overwrite a publish state file as a side-effect.
2822            let state_json = td.path().join(".shipper").join("state.json");
2823            std::fs::write(&state_json, "{\"sentinel\":\"state\"}").expect("seed state");
2824
2825            let mut reporter = CollectingReporter::default();
2826            let rep = super::run_preflight_with_options(
2827                &ws,
2828                &opts,
2829                &mut reporter,
2830                super::PreflightRunOptions { fresh_audit: true },
2831            )
2832            .expect("preflight");
2833            assert_eq!(rep.packages.len(), 1);
2834
2835            // Authoritative events.jsonl is byte-identical to the seed.
2836            let after = std::fs::read_to_string(&authoritative_events).expect("read events");
2837            assert_eq!(
2838                after, sentinel,
2839                "fresh_audit must NOT append or rewrite events.jsonl"
2840            );
2841
2842            // state.json untouched.
2843            let state_after = std::fs::read_to_string(&state_json).expect("read state");
2844            assert_eq!(
2845                state_after, "{\"sentinel\":\"state\"}",
2846                "fresh_audit must NOT write publish state"
2847            );
2848
2849            // Sidecar carries the full preflight trace.
2850            let sidecar =
2851                crate::state::events::preflight_only_events_paths(&td.path().join(".shipper"))
2852                    .expect("discover sidecars")
2853                    .into_iter()
2854                    .next()
2855                    .expect("fresh audit sidecar");
2856            assert!(sidecar.exists(), "sidecar must be written");
2857            let side_log =
2858                crate::state::events::EventLog::read_from_file(&sidecar).expect("read sidecar");
2859            let events = side_log.all_events();
2860            assert!(
2861                events
2862                    .iter()
2863                    .any(|e| matches!(e.event_type, EventType::PreflightStarted)),
2864                "sidecar must contain PreflightStarted"
2865            );
2866            assert!(
2867                events
2868                    .iter()
2869                    .any(|e| matches!(e.event_type, EventType::PreflightComplete { .. })),
2870                "sidecar must contain PreflightComplete"
2871            );
2872
2873            server.join();
2874        });
2875    }
2876
2877    // #100 — fresh-audit reflects *current* workspace state, not a
2878    // cached prior result. We run preflight twice in fresh_audit mode:
2879    // first against a registry that treats the crate as new (NotProven),
2880    // then against the same workspace but with a server that returns
2881    // 200 + owners — ownership verified, so Proven. If fresh_audit were
2882    // cached/reused, the second run would incorrectly still report
2883    // NotProven; it must reflect the changed registry reality.
2884    //
2885    // This also exercises the "prior events.jsonl NOT appended to"
2886    // invariant under repeated runs: the authoritative log stays empty
2887    // across both invocations.
2888    #[test]
2889    #[serial]
2890    fn run_preflight_with_dirty_git_fresh_audit() {
2891        let td = tempdir().expect("tempdir");
2892        let bin = td.path().join("bin");
2893        write_fake_tools(&bin);
2894        let mut env_vars = fake_program_env_vars(&bin);
2895        env_vars.extend([
2896            ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
2897            (
2898                "CARGO_HOME",
2899                Some(td.path().to_str().expect("utf8").to_string()),
2900            ),
2901            ("CARGO_REGISTRY_TOKEN", Some("token-abc".to_string())),
2902        ]);
2903        temp_env::with_vars(env_vars, || {
2904            // Run 1: new crate (404 on crate lookup) → NotProven (no
2905            // ownership possible for new crates when strict=false and
2906            // token present).
2907            let server1 = spawn_registry_server(
2908                std::collections::BTreeMap::from([
2909                    (
2910                        "/api/v1/crates/demo/0.1.0".to_string(),
2911                        vec![(404, "{}".to_string())],
2912                    ),
2913                    (
2914                        "/api/v1/crates/demo".to_string(),
2915                        vec![(404, "{}".to_string())],
2916                    ),
2917                ]),
2918                2,
2919            );
2920
2921            let ws1 = planned_workspace(td.path(), server1.base_url.clone());
2922            let mut opts = default_opts(PathBuf::from(".shipper"));
2923            opts.allow_dirty = true;
2924            opts.skip_ownership_check = false;
2925            opts.strict_ownership = false;
2926
2927            let mut reporter = CollectingReporter::default();
2928            let rep1 = super::run_preflight_with_options(
2929                &ws1,
2930                &opts,
2931                &mut reporter,
2932                super::PreflightRunOptions { fresh_audit: true },
2933            )
2934            .expect("preflight 1");
2935            assert_eq!(rep1.finishability, Finishability::NotProven);
2936            assert!(rep1.packages[0].is_new_crate);
2937            server1.join();
2938
2939            // Authoritative events.jsonl must not exist after a fresh
2940            // audit (no state persistence).
2941            let authoritative_events = td.path().join(".shipper").join("events.jsonl");
2942            assert!(
2943                !authoritative_events.exists(),
2944                "fresh_audit must not create events.jsonl; found {}",
2945                authoritative_events.display()
2946            );
2947
2948            // Run 2: established crate with confirmed owners → Proven.
2949            // Same workspace, different registry reality. Fresh audit
2950            // must pick up the new state, not cache the previous run.
2951            let server2 = spawn_registry_server(
2952                std::collections::BTreeMap::from([
2953                    (
2954                        "/api/v1/crates/demo/0.1.0".to_string(),
2955                        vec![(404, "{}".to_string())],
2956                    ),
2957                    (
2958                        "/api/v1/crates/demo".to_string(),
2959                        vec![(200, "{}".to_string())],
2960                    ),
2961                    (
2962                        "/api/v1/crates/demo/owners".to_string(),
2963                        vec![(
2964                            200,
2965                            r#"{"users":[{"id":1,"login":"alice","name":"Alice"}]}"#.to_string(),
2966                        )],
2967                    ),
2968                ]),
2969                3,
2970            );
2971
2972            let ws2 = planned_workspace(td.path(), server2.base_url.clone());
2973            let rep2 = super::run_preflight_with_options(
2974                &ws2,
2975                &opts,
2976                &mut reporter,
2977                super::PreflightRunOptions { fresh_audit: true },
2978            )
2979            .expect("preflight 2");
2980            assert_eq!(
2981                rep2.finishability,
2982                Finishability::Proven,
2983                "fresh_audit must reflect current registry state, not cached"
2984            );
2985            assert!(!rep2.packages[0].is_new_crate);
2986            server2.join();
2987
2988            // Still no authoritative events.jsonl after two fresh runs.
2989            assert!(
2990                !authoritative_events.exists(),
2991                "authoritative events.jsonl must remain absent across fresh audits"
2992            );
2993
2994            // Each fresh audit gets its own sidecar so repeated or concurrent
2995            // runs do not overwrite one another.
2996            let sidecars =
2997                crate::state::events::preflight_only_events_paths(&td.path().join(".shipper"))
2998                    .expect("discover sidecars");
2999            assert_eq!(
3000                sidecars.len(),
3001                2,
3002                "each fresh audit should create a new sidecar"
3003            );
3004
3005            for sidecar in sidecars {
3006                let side_log =
3007                    crate::state::events::EventLog::read_from_file(&sidecar).expect("read sidecar");
3008                let started_count = side_log
3009                    .all_events()
3010                    .iter()
3011                    .filter(|e| matches!(e.event_type, EventType::PreflightStarted))
3012                    .count();
3013                assert_eq!(
3014                    started_count, 1,
3015                    "each sidecar should contain exactly one fresh-audit session"
3016                );
3017            }
3018        });
3019    }
3020
3021    #[test]
3022    #[serial]
3023    fn run_preflight_detects_trusted_publishing_auth_type() {
3024        let td = tempdir().expect("tempdir");
3025        let bin = td.path().join("bin");
3026        write_fake_tools(&bin);
3027        let mut env_vars = fake_program_env_vars(&bin);
3028        env_vars.extend([
3029            ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
3030            (
3031                "CARGO_HOME",
3032                Some(td.path().to_str().expect("utf8").to_string()),
3033            ),
3034            ("CARGO_REGISTRY_TOKEN", None::<String>),
3035            ("CARGO_REGISTRIES_CRATES_IO_TOKEN", None::<String>),
3036            (
3037                "ACTIONS_ID_TOKEN_REQUEST_URL",
3038                Some("https://example.invalid/oidc".to_string()),
3039            ),
3040            (
3041                "ACTIONS_ID_TOKEN_REQUEST_TOKEN",
3042                Some("oidc-token".to_string()),
3043            ),
3044        ]);
3045        temp_env::with_vars(env_vars, || {
3046            let server = spawn_registry_server(
3047                std::collections::BTreeMap::from([
3048                    (
3049                        "/api/v1/crates/demo/0.1.0".to_string(),
3050                        vec![(404, "{}".to_string())],
3051                    ),
3052                    (
3053                        "/api/v1/crates/demo".to_string(),
3054                        vec![(404, "{}".to_string())],
3055                    ),
3056                ]),
3057                2,
3058            );
3059
3060            let ws = planned_workspace(td.path(), server.base_url.clone());
3061            let mut opts = default_opts(PathBuf::from(".shipper"));
3062            opts.allow_dirty = true;
3063            opts.skip_ownership_check = true;
3064
3065            let mut reporter = CollectingReporter::default();
3066            let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
3067
3068            assert!(!report.token_detected);
3069            assert_eq!(
3070                report.packages[0].auth_type,
3071                Some(crate::types::AuthType::TrustedPublishing)
3072            );
3073            server.join();
3074        });
3075    }
3076
3077    #[test]
3078    #[serial]
3079    fn run_preflight_warns_when_token_auth_overrides_oidc() {
3080        let td = tempdir().expect("tempdir");
3081        let bin = td.path().join("bin");
3082        write_fake_tools(&bin);
3083        let mut env_vars = fake_program_env_vars(&bin);
3084        env_vars.extend([
3085            ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
3086            (
3087                "CARGO_HOME",
3088                Some(td.path().to_str().expect("utf8").to_string()),
3089            ),
3090            ("CARGO_REGISTRY_TOKEN", Some("token-abc".to_string())),
3091            ("CARGO_REGISTRIES_CRATES_IO_TOKEN", None::<String>),
3092            (
3093                "ACTIONS_ID_TOKEN_REQUEST_URL",
3094                Some("https://example.invalid/oidc".to_string()),
3095            ),
3096            (
3097                "ACTIONS_ID_TOKEN_REQUEST_TOKEN",
3098                Some("oidc-token".to_string()),
3099            ),
3100        ]);
3101        temp_env::with_vars(env_vars, || {
3102            let server = spawn_registry_server(
3103                std::collections::BTreeMap::from([
3104                    (
3105                        "/api/v1/crates/demo/0.1.0".to_string(),
3106                        vec![(404, "{}".to_string())],
3107                    ),
3108                    (
3109                        "/api/v1/crates/demo".to_string(),
3110                        vec![(404, "{}".to_string())],
3111                    ),
3112                ]),
3113                2,
3114            );
3115
3116            let ws = planned_workspace(td.path(), server.base_url.clone());
3117            let mut opts = default_opts(PathBuf::from(".shipper"));
3118            opts.allow_dirty = true;
3119            opts.skip_ownership_check = true;
3120
3121            let mut reporter = CollectingReporter::default();
3122            let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
3123
3124            assert!(report.token_detected);
3125            assert_eq!(report.packages[0].auth_type, Some(AuthType::Token));
3126            let warnings = reporter.warns.join("\n");
3127            assert!(
3128                warnings.contains("Trusted Publishing OIDC environment is present"),
3129                "warnings: {warnings}"
3130            );
3131            assert!(
3132                !warnings.contains("token-abc"),
3133                "warnings must not expose token values: {warnings}"
3134            );
3135            server.join();
3136        });
3137    }
3138
3139    #[test]
3140    #[serial]
3141    fn run_preflight_does_not_warn_for_plain_token_auth() {
3142        let td = tempdir().expect("tempdir");
3143        let bin = td.path().join("bin");
3144        write_fake_tools(&bin);
3145        let mut env_vars = fake_program_env_vars(&bin);
3146        env_vars.extend([
3147            ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
3148            (
3149                "CARGO_HOME",
3150                Some(td.path().to_str().expect("utf8").to_string()),
3151            ),
3152            ("CARGO_REGISTRY_TOKEN", Some("token-abc".to_string())),
3153            ("CARGO_REGISTRIES_CRATES_IO_TOKEN", None::<String>),
3154            ("ACTIONS_ID_TOKEN_REQUEST_URL", None::<String>),
3155            ("ACTIONS_ID_TOKEN_REQUEST_TOKEN", None::<String>),
3156        ]);
3157        temp_env::with_vars(env_vars, || {
3158            let server = spawn_registry_server(
3159                std::collections::BTreeMap::from([
3160                    (
3161                        "/api/v1/crates/demo/0.1.0".to_string(),
3162                        vec![(404, "{}".to_string())],
3163                    ),
3164                    (
3165                        "/api/v1/crates/demo".to_string(),
3166                        vec![(404, "{}".to_string())],
3167                    ),
3168                ]),
3169                2,
3170            );
3171
3172            let ws = planned_workspace(td.path(), server.base_url.clone());
3173            let mut opts = default_opts(PathBuf::from(".shipper"));
3174            opts.allow_dirty = true;
3175            opts.skip_ownership_check = true;
3176
3177            let mut reporter = CollectingReporter::default();
3178            let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
3179
3180            assert!(report.token_detected);
3181            assert_eq!(report.packages[0].auth_type, Some(AuthType::Token));
3182            assert!(
3183                reporter
3184                    .warns
3185                    .iter()
3186                    .all(|warning| !warning.contains("Trusted Publishing OIDC environment")),
3187                "warnings: {:?}",
3188                reporter.warns
3189            );
3190            server.join();
3191        });
3192    }
3193
3194    #[test]
3195    #[serial]
3196    fn run_preflight_checks_git_when_allow_dirty_is_false() {
3197        let td = tempdir().expect("tempdir");
3198        let bin = td.path().join("bin");
3199        write_fake_tools(&bin);
3200        let mut env_vars = fake_program_env_vars(&bin);
3201        env_vars.extend([("SHIPPER_GIT_CLEAN", Some("1".to_string()))]);
3202        temp_env::with_vars(env_vars, || {
3203            let server = spawn_registry_server(
3204                std::collections::BTreeMap::from([
3205                    (
3206                        "/api/v1/crates/demo/0.1.0".to_string(),
3207                        vec![(404, "{}".to_string())],
3208                    ),
3209                    (
3210                        "/api/v1/crates/demo".to_string(),
3211                        vec![(404, "{}".to_string())],
3212                    ),
3213                ]),
3214                2,
3215            );
3216
3217            let ws = planned_workspace(td.path(), server.base_url.clone());
3218            let mut opts = default_opts(PathBuf::from(".shipper"));
3219            opts.allow_dirty = false;
3220            opts.skip_ownership_check = true;
3221
3222            let mut reporter = CollectingReporter::default();
3223            let rep = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
3224            assert_eq!(rep.packages.len(), 1);
3225            server.join();
3226        });
3227    }
3228
3229    #[test]
3230    #[serial]
3231    fn run_publish_skips_when_version_already_exists() {
3232        let td = tempdir().expect("tempdir");
3233        let bin = td.path().join("bin");
3234        write_fake_tools(&bin);
3235        let env_vars = fake_program_env_vars(&bin);
3236        temp_env::with_vars(env_vars, || {
3237            let server = spawn_registry_server(
3238                std::collections::BTreeMap::from([(
3239                    "/api/v1/crates/demo/0.1.0".to_string(),
3240                    vec![(200, "{}".to_string())],
3241                )]),
3242                1,
3243            );
3244            let ws = planned_workspace(td.path(), server.base_url.clone());
3245            let opts = default_opts(PathBuf::from(".shipper"));
3246
3247            let mut reporter = CollectingReporter::default();
3248            let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
3249            assert_eq!(receipt.packages.len(), 1);
3250            assert!(matches!(
3251                receipt.packages[0].state,
3252                PackageState::Skipped { .. }
3253            ));
3254
3255            let state_dir = td.path().join(".shipper");
3256            assert!(state::state_path(&state_dir).exists());
3257            assert!(state::receipt_path(&state_dir).exists());
3258            server.join();
3259        });
3260    }
3261
3262    /// Regression for #125: resume encountering a `Published` state in
3263    /// state.json must emit a `PackageSkipped` event, not silently move
3264    /// on. events.jsonl is the authoritative audit log; a silent skip
3265    /// makes "did resume look at this package at all?" unanswerable from
3266    /// the log alone.
3267    #[test]
3268    #[serial]
3269    fn resume_emits_package_skipped_event_for_already_published_state() {
3270        let td = tempdir().expect("tempdir");
3271        let bin = td.path().join("bin");
3272        write_fake_tools(&bin);
3273        let env_vars = fake_program_env_vars(&bin);
3274        temp_env::with_vars(env_vars, || {
3275            let server = spawn_registry_server(
3276                std::collections::BTreeMap::from([(
3277                    "/api/v1/crates/demo/0.1.0".to_string(),
3278                    vec![(404, "{}".to_string())],
3279                )]),
3280                1,
3281            );
3282            let ws = planned_workspace(td.path(), server.base_url.clone());
3283            let state_dir = td.path().join(".shipper");
3284
3285            // Seed: existing state says demo@0.1.0 is Published. Resume
3286            // should recognize it and skip — but crucially, emit the event.
3287            let mut packages = BTreeMap::new();
3288            packages.insert(
3289                "demo@0.1.0".to_string(),
3290                PackageProgress {
3291                    name: "demo".to_string(),
3292                    version: "0.1.0".to_string(),
3293                    attempts: 1,
3294                    state: PackageState::Published,
3295                    last_updated_at: Utc::now(),
3296                },
3297            );
3298            let seeded = ExecutionState {
3299                state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
3300                plan_id: ws.plan.plan_id.clone(),
3301                registry: ws.plan.registry.clone(),
3302                created_at: Utc::now(),
3303                updated_at: Utc::now(),
3304                attempt_history: Vec::new(),
3305                packages,
3306            };
3307            state::save_state(&state_dir, &seeded).expect("seed state");
3308
3309            let opts = default_opts(PathBuf::from(".shipper"));
3310            let mut reporter = CollectingReporter::default();
3311            let _receipt = run_publish(&ws, &opts, &mut reporter).expect("publish resumes");
3312
3313            // Read events.jsonl and assert a PackageSkipped event exists.
3314            let events_path = events::events_path(&state_dir);
3315            let raw = std::fs::read_to_string(&events_path).expect("events.jsonl");
3316            let skipped_count = raw
3317                .lines()
3318                .filter(|l| !l.trim().is_empty())
3319                .filter(|l| l.contains(r#""type":"package_skipped""#))
3320                .count();
3321            assert!(
3322                skipped_count >= 1,
3323                "expected at least one PackageSkipped event for the already-Published package; \
3324                 events.jsonl was:\n{raw}"
3325            );
3326        });
3327    }
3328
3329    /// Regression for #126: when resume encounters a `Failed` state and
3330    /// the registry confirms the version is visible, `state.json` must
3331    /// transition that package from Failed to Skipped. A stale `failed`
3332    /// flag misleads downstream tools (e.g. plan-yank) into thinking
3333    /// remediation is needed when it isn't.
3334    #[test]
3335    #[serial]
3336    fn resume_from_failed_ambiguous_updates_state_to_skipped_when_registry_visible() {
3337        let td = tempdir().expect("tempdir");
3338        let bin = td.path().join("bin");
3339        write_fake_tools(&bin);
3340        let env_vars = fake_program_env_vars(&bin);
3341        temp_env::with_vars(env_vars, || {
3342            // Registry returns 200 for the version check — the crate IS
3343            // visible, even though run 1 left us with Failed/Ambiguous.
3344            let server = spawn_registry_server(
3345                std::collections::BTreeMap::from([(
3346                    "/api/v1/crates/demo/0.1.0".to_string(),
3347                    vec![(200, "{}".to_string())],
3348                )]),
3349                1,
3350            );
3351            let ws = planned_workspace(td.path(), server.base_url.clone());
3352            let state_dir = td.path().join(".shipper");
3353
3354            let mut packages = BTreeMap::new();
3355            packages.insert(
3356                "demo@0.1.0".to_string(),
3357                PackageProgress {
3358                    name: "demo".to_string(),
3359                    version: "0.1.0".to_string(),
3360                    attempts: 1,
3361                    state: PackageState::Failed {
3362                        class: ErrorClass::Ambiguous,
3363                        message: "prior run: publish outcome ambiguous".to_string(),
3364                    },
3365                    last_updated_at: Utc::now(),
3366                },
3367            );
3368            let seeded = ExecutionState {
3369                state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
3370                plan_id: ws.plan.plan_id.clone(),
3371                registry: ws.plan.registry.clone(),
3372                created_at: Utc::now(),
3373                updated_at: Utc::now(),
3374                attempt_history: Vec::new(),
3375                packages,
3376            };
3377            state::save_state(&state_dir, &seeded).expect("seed state");
3378
3379            let opts = default_opts(PathBuf::from(".shipper"));
3380            let mut reporter = CollectingReporter::default();
3381            let _receipt = run_publish(&ws, &opts, &mut reporter).expect("publish resumes");
3382
3383            // State.json on disk must now say Skipped, not Failed.
3384            let reloaded = state::load_state(&state_dir)
3385                .expect("load")
3386                .expect("state exists");
3387            let pkg_state = &reloaded
3388                .packages
3389                .get("demo@0.1.0")
3390                .expect("package in state")
3391                .state;
3392            assert!(
3393                matches!(pkg_state, PackageState::Skipped { .. }),
3394                "expected state.json to show Skipped after resume reconciled against registry; got {pkg_state:?}"
3395            );
3396            server.join();
3397        });
3398    }
3399
3400    #[test]
3401    #[serial]
3402    fn sequential_ambiguous_publish_reconciles_to_published_without_retry() {
3403        let td = tempdir().expect("tempdir");
3404        let bin = td.path().join("bin");
3405        write_fake_tools(&bin);
3406        let cargo_log = td.path().join("cargo-calls.log");
3407        let mut env_vars = fake_program_env_vars(&bin);
3408        env_vars.extend([
3409            ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
3410            ("SHIPPER_CARGO_STDERR", Some(String::new())),
3411            ("SHIPPER_CARGO_STDOUT", Some(String::new())),
3412            (
3413                "SHIPPER_CARGO_ARGS_LOG",
3414                Some(cargo_log.to_string_lossy().to_string()),
3415            ),
3416        ]);
3417        temp_env::with_vars(env_vars, || {
3418            let server = spawn_registry_server(
3419                std::collections::BTreeMap::from([(
3420                    "/api/v1/crates/demo/0.1.0".to_string(),
3421                    vec![(404, "{}".to_string()), (200, "{}".to_string())],
3422                )]),
3423                2,
3424            );
3425            let ws = planned_workspace(td.path(), server.base_url.clone());
3426            let state_dir = td.path().join(".shipper");
3427            let mut opts = default_opts(state_dir.clone());
3428            opts.max_attempts = 2;
3429            opts.readiness.enabled = false;
3430
3431            let mut reporter = CollectingReporter::default();
3432            let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
3433
3434            assert!(matches!(receipt.packages[0].state, PackageState::Published));
3435            assert!(
3436                reporter
3437                    .infos
3438                    .iter()
3439                    .any(|msg| msg.contains("reconciliation outcome: Published")
3440                        && msg.contains("without retry")),
3441                "infos: {:?}",
3442                reporter.infos
3443            );
3444
3445            let cargo_invocations = std::fs::read_to_string(&cargo_log)
3446                .map(|s| s.lines().filter(|l| !l.trim().is_empty()).count())
3447                .unwrap_or(0);
3448            assert_eq!(cargo_invocations, 1, "must not retry after Published");
3449
3450            let events_path = events::events_path(&state_dir);
3451            let events = events::EventLog::read_from_file(&events_path).expect("events");
3452            assert!(
3453                events
3454                    .all_events()
3455                    .iter()
3456                    .any(|e| { matches!(e.event_type, EventType::PublishReconciling { .. }) })
3457            );
3458            assert!(events.all_events().iter().any(|e| {
3459                matches!(
3460                    &e.event_type,
3461                    EventType::PublishReconciled {
3462                        outcome: ReconciliationOutcome::Published { .. }
3463                    }
3464                )
3465            }));
3466
3467            server.join();
3468        });
3469    }
3470
3471    #[test]
3472    #[serial]
3473    fn sequential_ambiguous_publish_reconciles_to_not_published_then_retries_once() {
3474        let td = tempdir().expect("tempdir");
3475        let bin = td.path().join("bin");
3476        write_fake_tools(&bin);
3477        write_fake_cargo_ambiguous_then_permanent(&bin);
3478        let cargo_log = td.path().join("cargo-calls.log");
3479        let cargo_count = td.path().join("cargo-count.txt");
3480        let mut env_vars = fake_program_env_vars(&bin);
3481        env_vars.extend([
3482            (
3483                "SHIPPER_CARGO_ARGS_LOG",
3484                Some(cargo_log.to_string_lossy().to_string()),
3485            ),
3486            (
3487                "SHIPPER_CARGO_COUNT_FILE",
3488                Some(cargo_count.to_string_lossy().to_string()),
3489            ),
3490        ]);
3491        temp_env::with_vars(env_vars, || {
3492            let server = spawn_registry_server(
3493                std::collections::BTreeMap::from([(
3494                    "/api/v1/crates/demo/0.1.0".to_string(),
3495                    vec![
3496                        (404, "{}".to_string()),
3497                        (404, "{}".to_string()),
3498                        (404, "{}".to_string()),
3499                    ],
3500                )]),
3501                3,
3502            );
3503            let ws = planned_workspace(td.path(), server.base_url.clone());
3504            let state_dir = td.path().join(".shipper");
3505            let mut opts = default_opts(state_dir.clone());
3506            opts.max_attempts = 2;
3507            opts.readiness.enabled = false;
3508
3509            let mut reporter = CollectingReporter::default();
3510            let err = run_publish(&ws, &opts, &mut reporter).expect_err("publish should fail");
3511            let msg = format!("{err:#}");
3512            assert!(msg.contains("permanent failure"), "err: {msg}");
3513            assert!(
3514                reporter
3515                    .infos
3516                    .iter()
3517                    .any(|msg| msg.contains("reconciliation outcome: NotPublished")
3518                        && msg.contains("retry under publish policy")),
3519                "infos: {:?}",
3520                reporter.infos
3521            );
3522
3523            let cargo_invocations = std::fs::read_to_string(&cargo_log)
3524                .map(|s| s.lines().filter(|l| !l.trim().is_empty()).count())
3525                .unwrap_or(0);
3526            assert_eq!(
3527                cargo_invocations, 2,
3528                "NotPublished reconciliation should permit one retry"
3529            );
3530
3531            let events_path = events::events_path(&state_dir);
3532            let events = events::EventLog::read_from_file(&events_path).expect("events");
3533            assert!(events.all_events().iter().any(|e| {
3534                matches!(
3535                    &e.event_type,
3536                    EventType::PublishReconciled {
3537                        outcome: ReconciliationOutcome::NotPublished { .. }
3538                    }
3539                )
3540            }));
3541
3542            server.join();
3543        });
3544    }
3545
3546    #[test]
3547    #[serial]
3548    fn sequential_ambiguous_publish_still_unknown_stops_without_retry() {
3549        let td = tempdir().expect("tempdir");
3550        let bin = td.path().join("bin");
3551        write_fake_tools(&bin);
3552        let cargo_log = td.path().join("cargo-calls.log");
3553        let mut env_vars = fake_program_env_vars(&bin);
3554        env_vars.extend([
3555            ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
3556            ("SHIPPER_CARGO_STDERR", Some(String::new())),
3557            ("SHIPPER_CARGO_STDOUT", Some(String::new())),
3558            (
3559                "SHIPPER_CARGO_ARGS_LOG",
3560                Some(cargo_log.to_string_lossy().to_string()),
3561            ),
3562        ]);
3563        temp_env::with_vars(env_vars, || {
3564            let server = spawn_registry_server(
3565                std::collections::BTreeMap::from([(
3566                    "/api/v1/crates/demo/0.1.0".to_string(),
3567                    vec![(404, "{}".to_string()), (500, "{}".to_string())],
3568                )]),
3569                2,
3570            );
3571            let ws = planned_workspace(td.path(), server.base_url.clone());
3572            let state_dir = td.path().join(".shipper");
3573            let mut opts = default_opts(state_dir.clone());
3574            opts.max_attempts = 2;
3575            opts.readiness.enabled = false;
3576
3577            let mut reporter = CollectingReporter::default();
3578            let err = run_publish(&ws, &opts, &mut reporter).expect_err("publish should stop");
3579            let msg = format!("{err:#}");
3580            assert!(
3581                msg.contains("reconciliation inconclusive"),
3582                "expected inconclusive error, got: {msg}"
3583            );
3584            assert!(
3585                reporter
3586                    .errors
3587                    .iter()
3588                    .any(|msg| msg.contains("reconciliation outcome: StillUnknown")
3589                        && msg.contains("stop before blind retry")),
3590                "errors: {:?}",
3591                reporter.errors
3592            );
3593
3594            let cargo_invocations = std::fs::read_to_string(&cargo_log)
3595                .map(|s| s.lines().filter(|l| !l.trim().is_empty()).count())
3596                .unwrap_or(0);
3597            assert_eq!(cargo_invocations, 1, "must not retry after StillUnknown");
3598
3599            let state = state::load_state(&state_dir)
3600                .expect("load state")
3601                .expect("state exists");
3602            let progress = state.packages.get("demo@0.1.0").expect("package");
3603            assert!(
3604                matches!(progress.state, PackageState::Ambiguous { .. }),
3605                "expected Ambiguous state, got {:?}",
3606                progress.state
3607            );
3608
3609            let events_path = events::events_path(&state_dir);
3610            let events = events::EventLog::read_from_file(&events_path).expect("events");
3611            assert!(events.all_events().iter().any(|e| {
3612                matches!(
3613                    &e.event_type,
3614                    EventType::PublishReconciled {
3615                        outcome: ReconciliationOutcome::StillUnknown { .. }
3616                    }
3617                )
3618            }));
3619
3620            server.join();
3621        });
3622    }
3623
3624    #[test]
3625    #[serial]
3626    fn run_publish_checks_git_when_allow_dirty_is_false() {
3627        let td = tempdir().expect("tempdir");
3628        let bin = td.path().join("bin");
3629        write_fake_tools(&bin);
3630        let mut env_vars = fake_program_env_vars(&bin);
3631        env_vars.extend([("SHIPPER_GIT_CLEAN", Some("1".to_string()))]);
3632        temp_env::with_vars(env_vars, || {
3633            let server = spawn_registry_server(
3634                std::collections::BTreeMap::from([(
3635                    "/api/v1/crates/demo/0.1.0".to_string(),
3636                    vec![(200, "{}".to_string())],
3637                )]),
3638                1,
3639            );
3640            let ws = planned_workspace(td.path(), server.base_url.clone());
3641            let mut opts = default_opts(PathBuf::from(".shipper"));
3642            opts.allow_dirty = false;
3643
3644            let mut reporter = CollectingReporter::default();
3645            let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
3646            assert_eq!(receipt.packages.len(), 1);
3647            server.join();
3648        });
3649    }
3650
3651    #[test]
3652    #[serial]
3653    fn run_publish_adds_missing_package_entries_to_existing_state() {
3654        let td = tempdir().expect("tempdir");
3655        let bin = td.path().join("bin");
3656        write_fake_tools(&bin);
3657        let env_vars = fake_program_env_vars(&bin);
3658        temp_env::with_vars(env_vars, || {
3659            let server = spawn_registry_server(
3660                std::collections::BTreeMap::from([(
3661                    "/api/v1/crates/demo/0.1.0".to_string(),
3662                    vec![(200, "{}".to_string())],
3663                )]),
3664                1,
3665            );
3666
3667            let ws = planned_workspace(td.path(), server.base_url.clone());
3668            let state_dir = td.path().join(".shipper");
3669            let existing = ExecutionState {
3670                state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
3671                plan_id: ws.plan.plan_id.clone(),
3672                registry: ws.plan.registry.clone(),
3673                created_at: Utc::now(),
3674                updated_at: Utc::now(),
3675                attempt_history: Vec::new(),
3676                packages: BTreeMap::new(),
3677            };
3678            state::save_state(&state_dir, &existing).expect("save");
3679
3680            let opts = default_opts(PathBuf::from(".shipper"));
3681            let mut reporter = CollectingReporter::default();
3682            let _ = run_publish(&ws, &opts, &mut reporter).expect("publish");
3683
3684            let st = state::load_state(&state_dir)
3685                .expect("load")
3686                .expect("exists");
3687            assert!(st.packages.contains_key("demo@0.1.0"));
3688            server.join();
3689        });
3690    }
3691
3692    #[test]
3693    #[serial]
3694    fn run_publish_marks_published_after_successful_verify() {
3695        let td = tempdir().expect("tempdir");
3696        let bin = td.path().join("bin");
3697        write_fake_tools(&bin);
3698        let mut env_vars = fake_program_env_vars(&bin);
3699        env_vars.extend([("SHIPPER_CARGO_EXIT", Some("0".to_string()))]);
3700        temp_env::with_vars(env_vars, || {
3701            let server = spawn_registry_server(
3702                std::collections::BTreeMap::from([(
3703                    "/api/v1/crates/demo/0.1.0".to_string(),
3704                    vec![(404, "{}".to_string()), (200, "{}".to_string())],
3705                )]),
3706                2,
3707            );
3708            let ws = planned_workspace(td.path(), server.base_url.clone());
3709            let mut opts = default_opts(PathBuf::from(".shipper"));
3710            opts.verify_timeout = Duration::from_millis(200);
3711            opts.verify_poll_interval = Duration::from_millis(1);
3712
3713            let mut reporter = CollectingReporter::default();
3714            let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
3715            assert!(matches!(receipt.packages[0].state, PackageState::Published));
3716            server.join();
3717        });
3718    }
3719
3720    #[test]
3721    #[serial]
3722    fn run_publish_treats_500_as_not_visible_during_readiness() {
3723        // With the readiness-driven verify, 500 errors are treated as "not visible"
3724        // (graceful degradation). The publish succeeds via cargo but readiness times out,
3725        // leading to an ambiguous failure on the final registry check.
3726        let td = tempdir().expect("tempdir");
3727        let bin = td.path().join("bin");
3728        write_fake_tools(&bin);
3729        let mut env_vars = fake_program_env_vars(&bin);
3730        env_vars.extend([("SHIPPER_CARGO_EXIT", Some("0".to_string()))]);
3731        temp_env::with_vars(env_vars, || {
3732            let server = spawn_registry_server(
3733                std::collections::BTreeMap::from([(
3734                    "/api/v1/crates/demo/0.1.0".to_string(),
3735                    vec![
3736                        (404, "{}".to_string()),
3737                        (500, "{}".to_string()),
3738                        (404, "{}".to_string()),
3739                    ],
3740                )]),
3741                3,
3742            );
3743            let ws = planned_workspace(td.path(), server.base_url.clone());
3744            let mut opts = default_opts(PathBuf::from(".shipper"));
3745            opts.max_attempts = 1;
3746            opts.readiness.max_total_wait = Duration::from_millis(0);
3747
3748            let mut reporter = CollectingReporter::default();
3749            let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
3750            assert!(format!("{err:#}").contains("failed"));
3751            server.join();
3752        });
3753    }
3754
3755    #[test]
3756    #[serial]
3757    fn run_publish_treats_failed_cargo_as_published_if_registry_shows_version() {
3758        let td = tempdir().expect("tempdir");
3759        let bin = td.path().join("bin");
3760        write_fake_tools(&bin);
3761        with_test_env(
3762            &bin,
3763            vec![
3764                ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
3765                (
3766                    "SHIPPER_CARGO_STDERR",
3767                    Some("timeout while uploading".to_string()),
3768                ),
3769            ],
3770            || {
3771                let server = spawn_registry_server(
3772                    std::collections::BTreeMap::from([(
3773                        "/api/v1/crates/demo/0.1.0".to_string(),
3774                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
3775                    )]),
3776                    2,
3777                );
3778                let ws = planned_workspace(td.path(), server.base_url.clone());
3779                let mut opts = default_opts(PathBuf::from(".shipper"));
3780                opts.base_delay = Duration::from_millis(0);
3781                opts.max_delay = Duration::from_millis(0);
3782
3783                let mut reporter = CollectingReporter::default();
3784                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
3785                assert_eq!(receipt.packages.len(), 1);
3786                assert!(matches!(receipt.packages[0].state, PackageState::Published));
3787                assert_eq!(receipt.packages[0].attempts, 1);
3788                server.join();
3789            },
3790        );
3791    }
3792
3793    #[test]
3794    #[serial]
3795    fn run_publish_retries_on_retryable_failures() {
3796        let td = tempdir().expect("tempdir");
3797        let bin = td.path().join("bin");
3798        write_fake_tools(&bin);
3799        with_test_env(
3800            &bin,
3801            vec![
3802                ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
3803                (
3804                    "SHIPPER_CARGO_STDERR",
3805                    Some("timeout talking to server".to_string()),
3806                ),
3807            ],
3808            || {
3809                let server = spawn_registry_server(
3810                    std::collections::BTreeMap::from([(
3811                        "/api/v1/crates/demo/0.1.0".to_string(),
3812                        vec![
3813                            (404, "{}".to_string()),
3814                            (404, "{}".to_string()),
3815                            (200, "{}".to_string()),
3816                        ],
3817                    )]),
3818                    3,
3819                );
3820                let ws = planned_workspace(td.path(), server.base_url.clone());
3821                let mut opts = default_opts(PathBuf::from(".shipper"));
3822                opts.max_attempts = 2;
3823                opts.base_delay = Duration::from_millis(0);
3824                opts.max_delay = Duration::from_millis(0);
3825
3826                let mut reporter = CollectingReporter::default();
3827                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
3828                assert!(matches!(receipt.packages[0].state, PackageState::Published));
3829                assert_eq!(receipt.packages[0].attempts, 2);
3830                assert!(reporter.warns.iter().any(|w| w.contains("next attempt in")));
3831                server.join();
3832            },
3833        );
3834    }
3835
3836    #[test]
3837    #[serial]
3838    fn run_publish_errors_when_cargo_command_cannot_start() {
3839        let td = tempdir().expect("tempdir");
3840        let server = spawn_registry_server(
3841            std::collections::BTreeMap::from([(
3842                "/api/v1/crates/demo/0.1.0".to_string(),
3843                vec![(404, "{}".to_string())],
3844            )]),
3845            1,
3846        );
3847
3848        let ws = planned_workspace(td.path(), server.base_url.clone());
3849        let missing = td.path().join("no-cargo-here");
3850        temp_env::with_vars(
3851            vec![(
3852                "SHIPPER_CARGO_BIN",
3853                Some(missing.to_str().expect("utf8").to_string()),
3854            )],
3855            || {
3856                let opts = default_opts(PathBuf::from(".shipper"));
3857                let mut reporter = CollectingReporter::default();
3858                let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
3859                assert!(format!("{err:#}").contains("failed to execute cargo publish"));
3860            },
3861        );
3862        server.join();
3863    }
3864
3865    #[test]
3866    #[serial]
3867    fn run_publish_returns_error_on_permanent_failure() {
3868        let td = tempdir().expect("tempdir");
3869        let bin = td.path().join("bin");
3870        write_fake_tools(&bin);
3871        with_test_env(
3872            &bin,
3873            vec![
3874                ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
3875                (
3876                    "SHIPPER_CARGO_STDERR",
3877                    Some("permission denied".to_string()),
3878                ),
3879            ],
3880            || {
3881                let server = spawn_registry_server(
3882                    std::collections::BTreeMap::from([(
3883                        "/api/v1/crates/demo/0.1.0".to_string(),
3884                        vec![(404, "{}".to_string()), (404, "{}".to_string())],
3885                    )]),
3886                    2,
3887                );
3888                let ws = planned_workspace(td.path(), server.base_url.clone());
3889                let mut opts = default_opts(PathBuf::from(".shipper"));
3890                opts.base_delay = Duration::from_millis(0);
3891                opts.max_delay = Duration::from_millis(0);
3892
3893                let mut reporter = CollectingReporter::default();
3894                let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
3895                assert!(format!("{err:#}").contains("permanent failure"));
3896
3897                let st = state::load_state(&td.path().join(".shipper"))
3898                    .expect("load")
3899                    .expect("exists");
3900                let pkg = st.packages.get("demo@0.1.0").expect("pkg");
3901                assert!(matches!(
3902                    pkg.state,
3903                    PackageState::Failed {
3904                        class: ErrorClass::Permanent,
3905                        ..
3906                    }
3907                ));
3908                server.join();
3909            },
3910        );
3911    }
3912
3913    #[test]
3914    #[serial]
3915    fn run_publish_marks_ambiguous_failure_after_success_without_registry_visibility() {
3916        let td = tempdir().expect("tempdir");
3917        let bin = td.path().join("bin");
3918        write_fake_tools(&bin);
3919        with_test_env(
3920            &bin,
3921            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
3922            || {
3923                // 3 requests: initial version_exists, readiness check, final chance check
3924                let server = spawn_registry_server(
3925                    std::collections::BTreeMap::from([(
3926                        "/api/v1/crates/demo/0.1.0".to_string(),
3927                        vec![(404, "{}".to_string()), (404, "{}".to_string())],
3928                    )]),
3929                    3,
3930                );
3931                let ws = planned_workspace(td.path(), server.base_url.clone());
3932                let mut opts = default_opts(PathBuf::from(".shipper"));
3933                opts.max_attempts = 1;
3934                opts.readiness.max_total_wait = Duration::from_millis(0);
3935
3936                let mut reporter = CollectingReporter::default();
3937                let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
3938                assert!(format!("{err:#}").contains("failed"));
3939
3940                let st = state::load_state(&td.path().join(".shipper"))
3941                    .expect("load")
3942                    .expect("exists");
3943                let pkg = st.packages.get("demo@0.1.0").expect("pkg");
3944                assert!(matches!(
3945                    pkg.state,
3946                    PackageState::Failed {
3947                        class: ErrorClass::Ambiguous,
3948                        ..
3949                    }
3950                ));
3951                server.join();
3952            },
3953        );
3954    }
3955
3956    #[test]
3957    #[serial]
3958    fn run_publish_recovers_on_final_registry_check_after_ambiguous_verify() {
3959        let td = tempdir().expect("tempdir");
3960        let bin = td.path().join("bin");
3961        write_fake_tools(&bin);
3962        with_test_env(
3963            &bin,
3964            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
3965            || {
3966                let server = spawn_registry_server(
3967                    std::collections::BTreeMap::from([(
3968                        "/api/v1/crates/demo/0.1.0".to_string(),
3969                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
3970                    )]),
3971                    2,
3972                );
3973                let ws = planned_workspace(td.path(), server.base_url.clone());
3974                let mut opts = default_opts(PathBuf::from(".shipper"));
3975                opts.max_attempts = 1;
3976                opts.readiness.max_total_wait = Duration::from_millis(0);
3977
3978                let mut reporter = CollectingReporter::default();
3979                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
3980                assert!(matches!(receipt.packages[0].state, PackageState::Published));
3981                server.join();
3982            },
3983        );
3984    }
3985
3986    #[test]
3987    fn run_publish_errors_on_plan_mismatch_without_force_resume() {
3988        let td = tempdir().expect("tempdir");
3989        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
3990        let state_dir = td.path().join(".shipper");
3991
3992        let mut packages = std::collections::BTreeMap::new();
3993        packages.insert(
3994            "demo@0.1.0".to_string(),
3995            PackageProgress {
3996                name: "demo".to_string(),
3997                version: "0.1.0".to_string(),
3998                attempts: 0,
3999                state: PackageState::Pending,
4000                last_updated_at: Utc::now(),
4001            },
4002        );
4003        let st = ExecutionState {
4004            state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
4005            plan_id: "different-plan".to_string(),
4006            registry: ws.plan.registry.clone(),
4007            created_at: Utc::now(),
4008            updated_at: Utc::now(),
4009            attempt_history: Vec::new(),
4010            packages,
4011        };
4012        state::save_state(&state_dir, &st).expect("save");
4013
4014        let opts = default_opts(PathBuf::from(".shipper"));
4015        let mut reporter = CollectingReporter::default();
4016        let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
4017        assert!(format!("{err:#}").contains("does not match current plan_id"));
4018    }
4019
4020    #[test]
4021    fn run_publish_allows_forced_resume_with_plan_mismatch() {
4022        let td = tempdir().expect("tempdir");
4023        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
4024        let state_dir = td.path().join(".shipper");
4025
4026        let mut packages = std::collections::BTreeMap::new();
4027        packages.insert(
4028            "demo@0.1.0".to_string(),
4029            PackageProgress {
4030                name: "demo".to_string(),
4031                version: "0.1.0".to_string(),
4032                attempts: 1,
4033                state: PackageState::Published,
4034                last_updated_at: Utc::now(),
4035            },
4036        );
4037        let st = ExecutionState {
4038            state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
4039            plan_id: "different-plan".to_string(),
4040            registry: ws.plan.registry.clone(),
4041            created_at: Utc::now(),
4042            updated_at: Utc::now(),
4043            attempt_history: Vec::new(),
4044            packages,
4045        };
4046        state::save_state(&state_dir, &st).expect("save");
4047
4048        let mut opts = default_opts(PathBuf::from(".shipper"));
4049        opts.force_resume = true;
4050
4051        let mut reporter = CollectingReporter::default();
4052        let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
4053        assert!(receipt.packages.is_empty());
4054        assert!(
4055            reporter
4056                .warns
4057                .iter()
4058                .any(|w| w.contains("forcing resume with mismatched plan_id"))
4059        );
4060    }
4061
4062    #[test]
4063    fn run_resume_errors_when_state_is_missing() {
4064        let td = tempdir().expect("tempdir");
4065        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
4066        let opts = default_opts(PathBuf::from(".shipper"));
4067
4068        let mut reporter = CollectingReporter::default();
4069        let err = run_resume(&ws, &opts, &mut reporter).expect_err("must fail");
4070        assert!(format!("{err:#}").contains("no existing state found"));
4071    }
4072
4073    #[test]
4074    fn run_resume_runs_publish_when_state_exists() {
4075        let td = tempdir().expect("tempdir");
4076        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
4077        let state_dir = td.path().join(".shipper");
4078
4079        let mut packages = std::collections::BTreeMap::new();
4080        packages.insert(
4081            "demo@0.1.0".to_string(),
4082            PackageProgress {
4083                name: "demo".to_string(),
4084                version: "0.1.0".to_string(),
4085                attempts: 1,
4086                state: PackageState::Published,
4087                last_updated_at: Utc::now(),
4088            },
4089        );
4090        let st = ExecutionState {
4091            state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
4092            plan_id: ws.plan.plan_id.clone(),
4093            registry: ws.plan.registry.clone(),
4094            created_at: Utc::now(),
4095            updated_at: Utc::now(),
4096            attempt_history: Vec::new(),
4097            packages,
4098        };
4099        state::save_state(&state_dir, &st).expect("save");
4100
4101        let opts = default_opts(PathBuf::from(".shipper"));
4102        let mut reporter = CollectingReporter::default();
4103        let receipt = run_resume(&ws, &opts, &mut reporter).expect("resume");
4104        assert!(receipt.packages.is_empty());
4105    }
4106
4107    // Preflight-specific tests
4108
4109    fn preflight_pkg(name: &str, is_new_crate: bool) -> PreflightPackage {
4110        PreflightPackage {
4111            name: name.to_string(),
4112            version: "0.1.0".to_string(),
4113            already_published: false,
4114            is_new_crate,
4115            auth_type: Some(AuthType::Token),
4116            ownership_verified: true,
4117            dry_run_passed: true,
4118            dry_run_output: None,
4119        }
4120    }
4121
4122    #[test]
4123    fn estimate_preflight_duration_accounts_for_crates_io_first_publish_burst() {
4124        let packages: Vec<_> = (0..6)
4125            .map(|i| preflight_pkg(&format!("crate-{i}"), true))
4126            .collect();
4127
4128        let estimate = preflight::duration::estimate_preflight_duration("crates-io", &packages)
4129            .expect("crates.io profile should estimate");
4130
4131        assert_eq!(estimate.registry_profile, "crates-io");
4132        assert_eq!(estimate.first_publish_count, 6);
4133        assert_eq!(estimate.update_count, 0);
4134        assert_eq!(estimate.minimum_registry_pacing, Duration::from_secs(600));
4135        assert!(
4136            estimate
4137                .notes
4138                .iter()
4139                .any(|note| note.contains("documented registry pacing"))
4140        );
4141    }
4142
4143    #[test]
4144    fn estimate_preflight_duration_is_none_for_unknown_registry() {
4145        let packages = vec![preflight_pkg("demo", true)];
4146
4147        assert!(preflight::duration::estimate_preflight_duration("private", &packages).is_none());
4148    }
4149
4150    #[test]
4151    fn preflight_report_serializes_correctly() {
4152        let report = PreflightReport {
4153            plan_id: "test-plan".to_string(),
4154            token_detected: true,
4155            finishability: Finishability::Proven,
4156            packages: vec![PreflightPackage {
4157                name: "demo".to_string(),
4158                version: "0.1.0".to_string(),
4159                already_published: false,
4160                is_new_crate: false,
4161                auth_type: Some(AuthType::Token),
4162                ownership_verified: true,
4163                dry_run_passed: true,
4164                dry_run_output: None,
4165            }],
4166            timestamp: Utc::now(),
4167            estimated_publish_duration: None,
4168            dry_run_output: None,
4169        };
4170
4171        let json = serde_json::to_string(&report).expect("serialize");
4172        let parsed: PreflightReport = serde_json::from_str(&json).expect("deserialize");
4173        assert_eq!(parsed.plan_id, report.plan_id);
4174        assert_eq!(parsed.token_detected, report.token_detected);
4175        assert_eq!(parsed.finishability, report.finishability);
4176        assert_eq!(parsed.packages.len(), 1);
4177    }
4178
4179    #[test]
4180    fn finishability_proven_when_all_checks_pass() {
4181        let report = PreflightReport {
4182            plan_id: "test-plan".to_string(),
4183            token_detected: true,
4184            finishability: Finishability::Proven,
4185            packages: vec![PreflightPackage {
4186                name: "demo".to_string(),
4187                version: "0.1.0".to_string(),
4188                already_published: false,
4189                is_new_crate: false,
4190                auth_type: Some(AuthType::Token),
4191                ownership_verified: true,
4192                dry_run_passed: true,
4193                dry_run_output: None,
4194            }],
4195            timestamp: Utc::now(),
4196            estimated_publish_duration: None,
4197            dry_run_output: None,
4198        };
4199
4200        assert_eq!(report.finishability, Finishability::Proven);
4201    }
4202
4203    #[test]
4204    fn finishability_not_proven_when_ownership_unverified() {
4205        let report = PreflightReport {
4206            plan_id: "test-plan".to_string(),
4207            token_detected: true,
4208            finishability: Finishability::NotProven,
4209            packages: vec![PreflightPackage {
4210                name: "demo".to_string(),
4211                version: "0.1.0".to_string(),
4212                already_published: false,
4213                is_new_crate: true,
4214                auth_type: Some(AuthType::Token),
4215                ownership_verified: false,
4216                dry_run_passed: true,
4217                dry_run_output: None,
4218            }],
4219            timestamp: Utc::now(),
4220            estimated_publish_duration: None,
4221            dry_run_output: None,
4222        };
4223
4224        assert_eq!(report.finishability, Finishability::NotProven);
4225    }
4226
4227    #[test]
4228    fn finishability_failed_when_dry_run_fails() {
4229        let report = PreflightReport {
4230            plan_id: "test-plan".to_string(),
4231            token_detected: true,
4232            finishability: Finishability::Failed,
4233            packages: vec![PreflightPackage {
4234                name: "demo".to_string(),
4235                version: "0.1.0".to_string(),
4236                already_published: false,
4237                is_new_crate: false,
4238                auth_type: Some(AuthType::Token),
4239                ownership_verified: true,
4240                dry_run_passed: false,
4241                dry_run_output: None,
4242            }],
4243            timestamp: Utc::now(),
4244            estimated_publish_duration: None,
4245            dry_run_output: None,
4246        };
4247
4248        assert_eq!(report.finishability, Finishability::Failed);
4249    }
4250
4251    #[test]
4252    fn preflight_package_serializes_correctly() {
4253        let pkg = PreflightPackage {
4254            name: "demo".to_string(),
4255            version: "0.1.0".to_string(),
4256            already_published: false,
4257            is_new_crate: true,
4258            auth_type: Some(AuthType::Token),
4259            ownership_verified: true,
4260            dry_run_passed: true,
4261            dry_run_output: None,
4262        };
4263
4264        let json = serde_json::to_string(&pkg).expect("serialize");
4265        let parsed: PreflightPackage = serde_json::from_str(&json).expect("deserialize");
4266        assert_eq!(parsed.name, pkg.name);
4267        assert_eq!(parsed.version, pkg.version);
4268        assert_eq!(parsed.already_published, pkg.already_published);
4269        assert_eq!(parsed.is_new_crate, pkg.is_new_crate);
4270        assert_eq!(parsed.auth_type, pkg.auth_type);
4271        assert_eq!(parsed.ownership_verified, pkg.ownership_verified);
4272        assert_eq!(parsed.dry_run_passed, pkg.dry_run_passed);
4273    }
4274
4275    #[test]
4276    fn auth_type_serializes_correctly() {
4277        let token_auth = AuthType::Token;
4278        let tp_auth = AuthType::TrustedPublishing;
4279        let unknown_auth = AuthType::Unknown;
4280
4281        let json_token = serde_json::to_string(&token_auth).expect("serialize");
4282        let parsed_token: AuthType = serde_json::from_str(&json_token).expect("deserialize");
4283        assert_eq!(parsed_token, AuthType::Token);
4284
4285        let json_tp = serde_json::to_string(&tp_auth).expect("serialize");
4286        let parsed_tp: AuthType = serde_json::from_str(&json_tp).expect("deserialize");
4287        assert_eq!(parsed_tp, AuthType::TrustedPublishing);
4288
4289        let json_unknown = serde_json::to_string(&unknown_auth).expect("serialize");
4290        let parsed_unknown: AuthType = serde_json::from_str(&json_unknown).expect("deserialize");
4291        assert_eq!(parsed_unknown, AuthType::Unknown);
4292    }
4293
4294    // Integration tests for preflight scenarios
4295
4296    #[test]
4297    #[serial]
4298    fn preflight_with_all_packages_already_published() {
4299        let td = tempdir().expect("tempdir");
4300        let bin = td.path().join("bin");
4301        write_fake_tools(&bin);
4302        with_test_env(
4303            &bin,
4304            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
4305            || {
4306                // Mock registry: version already exists (200)
4307                let server = spawn_registry_server(
4308                    std::collections::BTreeMap::from([
4309                        (
4310                            "/api/v1/crates/demo/0.1.0".to_string(),
4311                            vec![(200, "{}".to_string())],
4312                        ),
4313                        (
4314                            "/api/v1/crates/demo".to_string(),
4315                            vec![(200, "{}".to_string())],
4316                        ),
4317                    ]),
4318                    2,
4319                );
4320                let ws = planned_workspace(td.path(), server.base_url.clone());
4321                let mut opts = default_opts(PathBuf::from(".shipper"));
4322                opts.allow_dirty = true;
4323                opts.skip_ownership_check = true;
4324
4325                let mut reporter = CollectingReporter::default();
4326                let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
4327
4328                assert_eq!(report.packages.len(), 1);
4329                assert!(report.packages[0].already_published);
4330                assert!(!report.packages[0].is_new_crate);
4331                assert!(report.packages[0].dry_run_passed);
4332                server.join();
4333            },
4334        );
4335    }
4336
4337    #[test]
4338    #[serial]
4339    fn preflight_with_new_crates() {
4340        let td = tempdir().expect("tempdir");
4341        let bin = td.path().join("bin");
4342        write_fake_tools(&bin);
4343        with_test_env(
4344            &bin,
4345            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
4346            || {
4347                // Mock registry: crate doesn't exist (404 for both crate and version)
4348                let server = spawn_registry_server(
4349                    std::collections::BTreeMap::from([
4350                        (
4351                            "/api/v1/crates/demo".to_string(),
4352                            vec![(404, "{}".to_string())],
4353                        ),
4354                        (
4355                            "/api/v1/crates/demo/0.1.0".to_string(),
4356                            vec![(404, "{}".to_string())],
4357                        ),
4358                    ]),
4359                    2,
4360                );
4361                let ws = planned_workspace(td.path(), server.base_url.clone());
4362                let mut opts = default_opts(PathBuf::from(".shipper"));
4363                opts.allow_dirty = true;
4364                opts.skip_ownership_check = true;
4365
4366                let mut reporter = CollectingReporter::default();
4367                let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
4368
4369                assert_eq!(report.packages.len(), 1);
4370                assert!(!report.packages[0].already_published);
4371                assert!(report.packages[0].is_new_crate);
4372                assert!(report.packages[0].dry_run_passed);
4373                server.join();
4374            },
4375        );
4376    }
4377
4378    #[test]
4379    #[serial]
4380    fn preflight_with_ownership_verification_failure() {
4381        let td = tempdir().expect("tempdir");
4382        let bin = td.path().join("bin");
4383        write_fake_tools(&bin);
4384        with_test_env(
4385            &bin,
4386            vec![
4387                ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
4388                ("CARGO_REGISTRY_TOKEN", Some("fake-token".to_string())),
4389            ],
4390            || {
4391                // Mock registry: version doesn't exist, crate exists, ownership check fails with 403
4392                let server = spawn_registry_server(
4393                    std::collections::BTreeMap::from([
4394                        (
4395                            "/api/v1/crates/demo".to_string(),
4396                            vec![(200, "{}".to_string())],
4397                        ),
4398                        (
4399                            "/api/v1/crates/demo/0.1.0".to_string(),
4400                            vec![(404, "{}".to_string())],
4401                        ),
4402                        (
4403                            "/api/v1/crates/demo/owners".to_string(),
4404                            vec![(403, "{}".to_string())],
4405                        ),
4406                    ]),
4407                    3,
4408                );
4409                let ws = planned_workspace(td.path(), server.base_url.clone());
4410                let mut opts = default_opts(PathBuf::from(".shipper"));
4411                opts.allow_dirty = true;
4412                opts.skip_ownership_check = false;
4413
4414                let mut reporter = CollectingReporter::default();
4415                let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
4416
4417                assert_eq!(report.packages.len(), 1);
4418                assert!(!report.packages[0].ownership_verified);
4419                // Should be NotProven because ownership is unverified
4420                assert_eq!(report.finishability, Finishability::NotProven);
4421                server.join();
4422            },
4423        );
4424    }
4425
4426    #[test]
4427    #[serial]
4428    fn preflight_with_dry_run_failure() {
4429        let td = tempdir().expect("tempdir");
4430        let bin = td.path().join("bin");
4431        write_fake_tools(&bin);
4432        with_test_env(
4433            &bin,
4434            vec![
4435                ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
4436                ("SHIPPER_CARGO_STDERR", Some("dry-run failed".to_string())),
4437            ],
4438            || {
4439                let server = spawn_registry_server(
4440                    std::collections::BTreeMap::from([
4441                        (
4442                            "/api/v1/crates/demo/0.1.0".to_string(),
4443                            vec![(404, "{}".to_string())],
4444                        ),
4445                        (
4446                            "/api/v1/crates/demo".to_string(),
4447                            vec![(404, "{}".to_string())],
4448                        ),
4449                    ]),
4450                    2,
4451                );
4452                let ws = planned_workspace(td.path(), server.base_url.clone());
4453                let mut opts = default_opts(PathBuf::from(".shipper"));
4454                opts.allow_dirty = true;
4455                opts.skip_ownership_check = true;
4456
4457                let mut reporter = CollectingReporter::default();
4458                let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
4459
4460                assert_eq!(report.packages.len(), 1);
4461                assert!(!report.packages[0].dry_run_passed);
4462                // Should be Failed because dry-run failed
4463                assert_eq!(report.finishability, Finishability::Failed);
4464                server.join();
4465            },
4466        );
4467    }
4468
4469    #[test]
4470    #[serial]
4471    fn preflight_strict_ownership_requires_token() {
4472        let td = tempdir().expect("tempdir");
4473        let bin = td.path().join("bin");
4474        write_fake_tools(&bin);
4475        with_test_env(
4476            &bin,
4477            vec![
4478                ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
4479                (
4480                    "CARGO_HOME",
4481                    Some(td.path().to_str().expect("utf8").to_string()),
4482                ),
4483                ("CARGO_REGISTRY_TOKEN", None),
4484                ("CARGO_REGISTRIES_CRATES_IO_TOKEN", None),
4485            ],
4486            || {
4487                let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
4488                let mut opts = default_opts(PathBuf::from(".shipper"));
4489                opts.allow_dirty = true;
4490                opts.strict_ownership = true;
4491                // No token set
4492
4493                let mut reporter = CollectingReporter::default();
4494                let err = run_preflight(&ws, &opts, &mut reporter).expect_err("must fail");
4495                assert!(
4496                    format!("{err:#}").contains("strict ownership requested but no token found")
4497                );
4498            },
4499        );
4500    }
4501
4502    #[test]
4503    #[serial]
4504    fn preflight_finishability_proven_with_all_checks_pass() {
4505        let td = tempdir().expect("tempdir");
4506        let bin = td.path().join("bin");
4507        write_fake_tools(&bin);
4508        with_test_env(
4509            &bin,
4510            vec![
4511                ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
4512                ("CARGO_REGISTRY_TOKEN", Some("fake-token".to_string())),
4513            ],
4514            || {
4515                // Mock registry: version doesn't exist, crate exists, ownership succeeds
4516                let server = spawn_registry_server(
4517                    std::collections::BTreeMap::from([
4518                        (
4519                            "/api/v1/crates/demo".to_string(),
4520                            vec![(200, "{}".to_string())],
4521                        ),
4522                        (
4523                            "/api/v1/crates/demo/0.1.0".to_string(),
4524                            vec![(404, "{}".to_string())],
4525                        ),
4526                        (
4527                            "/api/v1/crates/demo/owners".to_string(),
4528                            vec![(200, r#"{"users":[]}"#.to_string())],
4529                        ),
4530                    ]),
4531                    3,
4532                );
4533                let ws = planned_workspace(td.path(), server.base_url.clone());
4534                let mut opts = default_opts(PathBuf::from(".shipper"));
4535                opts.allow_dirty = true;
4536                opts.skip_ownership_check = false;
4537
4538                let mut reporter = CollectingReporter::default();
4539                let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
4540
4541                assert_eq!(report.packages.len(), 1);
4542                assert!(report.packages[0].ownership_verified);
4543                assert!(report.packages[0].dry_run_passed);
4544                assert_eq!(report.finishability, Finishability::Proven);
4545                server.join();
4546            },
4547        );
4548    }
4549
4550    #[test]
4551    #[serial]
4552    fn test_fast_policy_skips_dry_run() {
4553        let td = tempdir().expect("tempdir");
4554        let bin = td.path().join("bin");
4555        write_fake_tools(&bin);
4556        // Deliberately set cargo to fail — if dry-run runs, it would fail
4557        with_test_env(
4558            &bin,
4559            vec![("SHIPPER_CARGO_EXIT", Some("1".to_string()))],
4560            || {
4561                // Only need version_exists + check_new_crate
4562                let server = spawn_registry_server(
4563                    std::collections::BTreeMap::from([
4564                        (
4565                            "/api/v1/crates/demo/0.1.0".to_string(),
4566                            vec![(404, "{}".to_string())],
4567                        ),
4568                        (
4569                            "/api/v1/crates/demo".to_string(),
4570                            vec![(404, "{}".to_string())],
4571                        ),
4572                    ]),
4573                    2,
4574                );
4575                let ws = planned_workspace(td.path(), server.base_url.clone());
4576                let mut opts = default_opts(PathBuf::from(".shipper"));
4577                opts.policy = crate::types::PublishPolicy::Fast;
4578
4579                let mut reporter = CollectingReporter::default();
4580                let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
4581
4582                // dry_run_passed should be true (skipped), not false (cargo would have failed)
4583                assert!(report.packages[0].dry_run_passed);
4584                // ownership_verified should be false (skipped by Fast policy)
4585                assert!(!report.packages[0].ownership_verified);
4586                // Finishability is NotProven because ownership unverified
4587                assert_eq!(report.finishability, Finishability::NotProven);
4588                assert!(
4589                    reporter
4590                        .infos
4591                        .iter()
4592                        .any(|i| i.contains("skipping dry-run"))
4593                );
4594                server.join();
4595            },
4596        );
4597    }
4598
4599    #[test]
4600    #[serial]
4601    fn test_balanced_policy_skips_ownership() {
4602        let td = tempdir().expect("tempdir");
4603        let bin = td.path().join("bin");
4604        write_fake_tools(&bin);
4605        with_test_env(
4606            &bin,
4607            vec![
4608                ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
4609                ("CARGO_REGISTRY_TOKEN", Some("fake-token".to_string())),
4610            ],
4611            || {
4612                // Only need version_exists + check_new_crate (no ownership endpoint)
4613                let server = spawn_registry_server(
4614                    std::collections::BTreeMap::from([
4615                        (
4616                            "/api/v1/crates/demo/0.1.0".to_string(),
4617                            vec![(404, "{}".to_string())],
4618                        ),
4619                        (
4620                            "/api/v1/crates/demo".to_string(),
4621                            vec![(404, "{}".to_string())],
4622                        ),
4623                    ]),
4624                    2,
4625                );
4626                let ws = planned_workspace(td.path(), server.base_url.clone());
4627                let mut opts = default_opts(PathBuf::from(".shipper"));
4628                opts.policy = crate::types::PublishPolicy::Balanced;
4629                opts.skip_ownership_check = false; // would check in Safe, but Balanced overrides
4630
4631                let mut reporter = CollectingReporter::default();
4632                let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
4633
4634                // ownership_verified false (Balanced skips ownership)
4635                assert!(!report.packages[0].ownership_verified);
4636                // dry_run_passed true (Balanced still runs dry-run)
4637                assert!(report.packages[0].dry_run_passed);
4638                server.join();
4639            },
4640        );
4641    }
4642
4643    #[test]
4644    #[serial]
4645    fn test_safe_policy_runs_all_checks() {
4646        let td = tempdir().expect("tempdir");
4647        let bin = td.path().join("bin");
4648        write_fake_tools(&bin);
4649        with_test_env(
4650            &bin,
4651            vec![
4652                ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
4653                ("CARGO_REGISTRY_TOKEN", Some("fake-token".to_string())),
4654            ],
4655            || {
4656                // Need version_exists + check_new_crate + ownership
4657                let server = spawn_registry_server(
4658                    std::collections::BTreeMap::from([
4659                        (
4660                            "/api/v1/crates/demo/0.1.0".to_string(),
4661                            vec![(404, "{}".to_string())],
4662                        ),
4663                        (
4664                            "/api/v1/crates/demo".to_string(),
4665                            vec![(200, "{}".to_string())],
4666                        ),
4667                        (
4668                            "/api/v1/crates/demo/owners".to_string(),
4669                            vec![(200, r#"{"users":[]}"#.to_string())],
4670                        ),
4671                    ]),
4672                    3,
4673                );
4674                let ws = planned_workspace(td.path(), server.base_url.clone());
4675                let mut opts = default_opts(PathBuf::from(".shipper"));
4676                opts.policy = crate::types::PublishPolicy::Safe;
4677                opts.skip_ownership_check = false;
4678
4679                let mut reporter = CollectingReporter::default();
4680                let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
4681
4682                // All checks ran
4683                assert!(report.packages[0].dry_run_passed);
4684                assert!(report.packages[0].ownership_verified);
4685                assert_eq!(report.finishability, Finishability::Proven);
4686                server.join();
4687            },
4688        );
4689    }
4690
4691    #[test]
4692    #[serial]
4693    fn test_verify_mode_none_skips_dry_run() {
4694        let td = tempdir().expect("tempdir");
4695        let bin = td.path().join("bin");
4696        write_fake_tools(&bin);
4697        // Set cargo to fail — if dry-run ran, it would fail
4698        with_test_env(
4699            &bin,
4700            vec![("SHIPPER_CARGO_EXIT", Some("1".to_string()))],
4701            || {
4702                let server = spawn_registry_server(
4703                    std::collections::BTreeMap::from([
4704                        (
4705                            "/api/v1/crates/demo/0.1.0".to_string(),
4706                            vec![(404, "{}".to_string())],
4707                        ),
4708                        (
4709                            "/api/v1/crates/demo".to_string(),
4710                            vec![(404, "{}".to_string())],
4711                        ),
4712                    ]),
4713                    2,
4714                );
4715                let ws = planned_workspace(td.path(), server.base_url.clone());
4716                let mut opts = default_opts(PathBuf::from(".shipper"));
4717                opts.verify_mode = crate::types::VerifyMode::None;
4718
4719                let mut reporter = CollectingReporter::default();
4720                let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
4721
4722                // dry_run_passed is true because verify_mode=None skips it
4723                assert!(report.packages[0].dry_run_passed);
4724                assert!(
4725                    reporter
4726                        .infos
4727                        .iter()
4728                        .any(|i| i.contains("skipping dry-run"))
4729                );
4730                server.join();
4731            },
4732        );
4733    }
4734
4735    #[test]
4736    #[serial]
4737    fn test_verify_mode_package_runs_per_package() {
4738        let td = tempdir().expect("tempdir");
4739        let bin = td.path().join("bin");
4740        write_fake_tools(&bin);
4741        with_test_env(
4742            &bin,
4743            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
4744            || {
4745                let server = spawn_registry_server(
4746                    std::collections::BTreeMap::from([
4747                        (
4748                            "/api/v1/crates/demo/0.1.0".to_string(),
4749                            vec![(404, "{}".to_string())],
4750                        ),
4751                        (
4752                            "/api/v1/crates/demo".to_string(),
4753                            vec![(404, "{}".to_string())],
4754                        ),
4755                    ]),
4756                    2,
4757                );
4758                let ws = planned_workspace(td.path(), server.base_url.clone());
4759                let mut opts = default_opts(PathBuf::from(".shipper"));
4760                opts.verify_mode = crate::types::VerifyMode::Package;
4761
4762                let mut reporter = CollectingReporter::default();
4763                let report = run_preflight(&ws, &opts, &mut reporter).expect("preflight");
4764
4765                assert!(report.packages[0].dry_run_passed);
4766                assert!(
4767                    reporter
4768                        .infos
4769                        .iter()
4770                        .any(|i| i.contains("per-package dry-run"))
4771                );
4772                server.join();
4773            },
4774        );
4775    }
4776
4777    #[test]
4778    #[serial]
4779    fn resume_from_uploaded_skips_cargo_publish_and_reaches_published() {
4780        let td = tempdir().expect("tempdir");
4781        let bin = td.path().join("bin");
4782        write_fake_tools(&bin);
4783        let args_log = td.path().join("cargo_args.txt");
4784        let mut env_vars = fake_program_env_vars(&bin);
4785        env_vars.extend([
4786            ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
4787            (
4788                "SHIPPER_CARGO_ARGS_LOG",
4789                Some(args_log.to_str().expect("utf8").to_string()),
4790            ),
4791        ]);
4792        temp_env::with_vars(env_vars, || {
4793            // First request (early check) returns 404, second (readiness) returns 200
4794            let server = spawn_registry_server(
4795                std::collections::BTreeMap::from([(
4796                    "/api/v1/crates/demo/0.1.0".to_string(),
4797                    vec![(404, "{}".to_string()), (200, "{}".to_string())],
4798                )]),
4799                2,
4800            );
4801
4802            let ws = planned_workspace(td.path(), server.base_url.clone());
4803            let state_dir = td.path().join(".shipper");
4804
4805            // Pre-create state with Uploaded + attempts=1
4806            let mut packages = std::collections::BTreeMap::new();
4807            packages.insert(
4808                "demo@0.1.0".to_string(),
4809                PackageProgress {
4810                    name: "demo".to_string(),
4811                    version: "0.1.0".to_string(),
4812                    attempts: 1,
4813                    state: PackageState::Uploaded,
4814                    last_updated_at: Utc::now(),
4815                },
4816            );
4817            let st = ExecutionState {
4818                state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
4819                plan_id: ws.plan.plan_id.clone(),
4820                registry: ws.plan.registry.clone(),
4821                created_at: Utc::now(),
4822                updated_at: Utc::now(),
4823                attempt_history: Vec::new(),
4824                packages,
4825            };
4826            state::save_state(&state_dir, &st).expect("save");
4827
4828            let opts = default_opts(PathBuf::from(".shipper"));
4829            let mut reporter = CollectingReporter::default();
4830            let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
4831
4832            // Package should reach Published via the readiness verification path
4833            assert_eq!(receipt.packages.len(), 1);
4834            assert!(
4835                matches!(receipt.packages[0].state, PackageState::Published),
4836                "expected Published, got {:?}",
4837                receipt.packages[0].state
4838            );
4839
4840            // Cargo publish should NOT have been invoked
4841            // (args_log should not exist or be empty — no cargo publish calls)
4842            let cargo_invoked = args_log.exists()
4843                && fs::read_to_string(&args_log)
4844                    .unwrap_or_default()
4845                    .contains("publish");
4846            assert!(
4847                !cargo_invoked,
4848                "cargo publish should not have been invoked on resume from Uploaded"
4849            );
4850
4851            // Verify reporter got the resume message
4852            assert!(
4853                reporter
4854                    .infos
4855                    .iter()
4856                    .any(|i| i.contains("resuming from uploaded")
4857                        || i.contains("already published")
4858                        || i.contains("already complete"))
4859            );
4860
4861            // Verify the readiness path was exercised
4862            assert!(
4863                reporter.infos.iter().any(|i| i.contains("verifying")
4864                    || i.contains("visible")
4865                    || i.contains("readiness")),
4866                "expected readiness verification to be exercised, reporter infos: {:?}",
4867                reporter.infos
4868            );
4869
4870            server.join();
4871        });
4872    }
4873
4874    #[test]
4875    #[serial]
4876    fn test_resume_from_skips_initial_packages() {
4877        let td = tempdir().expect("tempdir");
4878        let bin = td.path().join("bin");
4879        write_fake_tools(&bin);
4880        let args_log = td.path().join("cargo_args.txt");
4881        let mut env_vars = fake_program_env_vars(&bin);
4882        env_vars.extend([
4883            ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
4884            (
4885                "SHIPPER_CARGO_ARGS_LOG",
4886                Some(args_log.to_str().expect("utf8").to_string()),
4887            ),
4888        ]);
4889        temp_env::with_vars(env_vars, || {
4890            // Mock registry for two packages
4891            // We expect 2 requests total (pkg2 exists? then pkg2 readiness)
4892            // pkg1 is skipped because of resume_from
4893            let server = spawn_registry_server(
4894                std::collections::BTreeMap::from([
4895                    (
4896                        "/api/v1/crates/pkg1/0.1.0".to_string(),
4897                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
4898                    ),
4899                    (
4900                        "/api/v1/crates/pkg2/0.1.0".to_string(),
4901                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
4902                    ),
4903                ]),
4904                2,
4905            );
4906
4907            let mut ws = planned_workspace(td.path(), server.base_url.clone());
4908            // Update plan to have two packages
4909            ws.plan.packages = vec![
4910                PlannedPackage {
4911                    name: "pkg1".to_string(),
4912                    version: "0.1.0".to_string(),
4913                    manifest_path: td.path().join("pkg1/Cargo.toml"),
4914                    regime: None,
4915                },
4916                PlannedPackage {
4917                    name: "pkg2".to_string(),
4918                    version: "0.1.0".to_string(),
4919                    manifest_path: td.path().join("pkg2/Cargo.toml"),
4920                    regime: None,
4921                },
4922            ];
4923
4924            let mut opts = default_opts(PathBuf::from(".shipper"));
4925            // Resume from second package
4926            opts.resume_from = Some("pkg2".to_string());
4927
4928            let mut reporter = CollectingReporter::default();
4929            let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
4930
4931            // Should only have 1 package in receipt (pkg2)
4932            assert_eq!(receipt.packages.len(), 1);
4933            assert_eq!(receipt.packages[0].name, "pkg2");
4934
4935            // Cargo log should only contain publish for pkg2
4936            let log = std::fs::read_to_string(&args_log).expect("read log");
4937            assert!(!log.contains("pkg1"));
4938            assert!(log.contains("pkg2"));
4939
4940            server.join();
4941        });
4942    }
4943
4944    // ── Additional coverage tests ──────────────────────────────────────
4945
4946    #[test]
4947    fn init_state_creates_pending_entries_for_all_packages() {
4948        let td = tempdir().expect("tempdir");
4949        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
4950        let state_dir = td.path().join(".shipper");
4951
4952        let st = init_state(&ws, &state_dir).expect("init");
4953        assert_eq!(st.plan_id, "plan-demo");
4954        assert_eq!(st.packages.len(), 1);
4955        let progress = st.packages.get("demo@0.1.0").expect("pkg");
4956        assert_eq!(progress.name, "demo");
4957        assert_eq!(progress.version, "0.1.0");
4958        assert_eq!(progress.attempts, 0);
4959        assert!(matches!(progress.state, PackageState::Pending));
4960    }
4961
4962    #[test]
4963    fn init_state_persists_state_to_disk() {
4964        let td = tempdir().expect("tempdir");
4965        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
4966        let state_dir = td.path().join(".shipper");
4967
4968        let _ = init_state(&ws, &state_dir).expect("init");
4969        let loaded = state::load_state(&state_dir)
4970            .expect("load")
4971            .expect("exists");
4972        assert_eq!(loaded.plan_id, "plan-demo");
4973        assert!(loaded.packages.contains_key("demo@0.1.0"));
4974    }
4975
4976    #[test]
4977    fn init_state_with_multi_package_plan() {
4978        let td = tempdir().expect("tempdir");
4979        let mut ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
4980        ws.plan.packages = vec![
4981            PlannedPackage {
4982                name: "alpha".to_string(),
4983                version: "1.0.0".to_string(),
4984                manifest_path: td.path().join("alpha/Cargo.toml"),
4985                regime: None,
4986            },
4987            PlannedPackage {
4988                name: "beta".to_string(),
4989                version: "2.0.0".to_string(),
4990                manifest_path: td.path().join("beta/Cargo.toml"),
4991                regime: None,
4992            },
4993            PlannedPackage {
4994                name: "gamma".to_string(),
4995                version: "0.3.0".to_string(),
4996                manifest_path: td.path().join("gamma/Cargo.toml"),
4997                regime: None,
4998            },
4999        ];
5000        let state_dir = td.path().join(".shipper");
5001
5002        let st = init_state(&ws, &state_dir).expect("init");
5003        assert_eq!(st.packages.len(), 3);
5004        assert!(st.packages.contains_key("alpha@1.0.0"));
5005        assert!(st.packages.contains_key("beta@2.0.0"));
5006        assert!(st.packages.contains_key("gamma@0.3.0"));
5007        for progress in st.packages.values() {
5008            assert_eq!(progress.attempts, 0);
5009            assert!(matches!(progress.state, PackageState::Pending));
5010        }
5011    }
5012
5013    #[test]
5014    fn run_publish_errors_on_invalid_resume_from_target() {
5015        let td = tempdir().expect("tempdir");
5016        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
5017        let mut opts = default_opts(PathBuf::from(".shipper"));
5018        opts.resume_from = Some("nonexistent-package".to_string());
5019
5020        let mut reporter = CollectingReporter::default();
5021        let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
5022        assert!(format!("{err:#}").contains("resume-from package"));
5023        assert!(format!("{err:#}").contains("not found in publish plan"));
5024    }
5025
5026    #[test]
5027    #[serial]
5028    fn run_publish_writes_execution_events() {
5029        let td = tempdir().expect("tempdir");
5030        let bin = td.path().join("bin");
5031        write_fake_tools(&bin);
5032        let env_vars = fake_program_env_vars(&bin);
5033        temp_env::with_vars(env_vars, || {
5034            let server = spawn_registry_server(
5035                std::collections::BTreeMap::from([(
5036                    "/api/v1/crates/demo/0.1.0".to_string(),
5037                    vec![(200, "{}".to_string())],
5038                )]),
5039                1,
5040            );
5041            let ws = planned_workspace(td.path(), server.base_url.clone());
5042            let opts = default_opts(PathBuf::from(".shipper"));
5043
5044            let mut reporter = CollectingReporter::default();
5045            let _ = run_publish(&ws, &opts, &mut reporter).expect("publish");
5046
5047            let events_path = td.path().join(".shipper").join("events.jsonl");
5048            let log =
5049                crate::state::events::EventLog::read_from_file(&events_path).expect("read events");
5050            let events = log.all_events();
5051
5052            assert!(
5053                events
5054                    .iter()
5055                    .any(|e| matches!(e.event_type, EventType::ExecutionStarted))
5056            );
5057            assert!(
5058                events
5059                    .iter()
5060                    .any(|e| matches!(e.event_type, EventType::PlanCreated { .. }))
5061            );
5062            assert!(
5063                events
5064                    .iter()
5065                    .any(|e| matches!(e.event_type, EventType::PackageSkipped { .. }))
5066            );
5067            assert!(
5068                events
5069                    .iter()
5070                    .any(|e| matches!(e.event_type, EventType::ExecutionFinished { .. }))
5071            );
5072            server.join();
5073        });
5074    }
5075
5076    #[test]
5077    #[serial]
5078    fn run_publish_receipt_contains_evidence_after_success() {
5079        let td = tempdir().expect("tempdir");
5080        let bin = td.path().join("bin");
5081        write_fake_tools(&bin);
5082        let mut env_vars = fake_program_env_vars(&bin);
5083        env_vars.extend([
5084            ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
5085            (
5086                "CARGO_REGISTRY_TOKEN",
5087                Some("release-secret-token".to_string()),
5088            ),
5089            (
5090                "ACTIONS_ID_TOKEN_REQUEST_URL",
5091                Some("https://example.invalid/oidc".to_string()),
5092            ),
5093            (
5094                "ACTIONS_ID_TOKEN_REQUEST_TOKEN",
5095                Some("oidc-request-token".to_string()),
5096            ),
5097        ]);
5098        temp_env::with_vars(env_vars, || {
5099            let server = spawn_registry_server(
5100                std::collections::BTreeMap::from([(
5101                    "/api/v1/crates/demo/0.1.0".to_string(),
5102                    vec![(404, "{}".to_string()), (200, "{}".to_string())],
5103                )]),
5104                2,
5105            );
5106            let ws = planned_workspace(td.path(), server.base_url.clone());
5107            let opts = default_opts(PathBuf::from(".shipper"));
5108
5109            let mut reporter = CollectingReporter::default();
5110            let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
5111
5112            assert_eq!(receipt.receipt_version, "shipper.receipt.v2");
5113            assert_eq!(receipt.plan_id, "plan-demo");
5114            assert_eq!(receipt.registry.name, "crates-io");
5115            assert_eq!(receipt.packages.len(), 1);
5116            assert!(matches!(receipt.packages[0].state, PackageState::Published));
5117            assert_eq!(receipt.packages[0].attempts, 1);
5118            assert!(!receipt.packages[0].evidence.attempts.is_empty());
5119            assert_eq!(receipt.packages[0].evidence.attempts[0].attempt_number, 1);
5120            assert_eq!(receipt.packages[0].evidence.attempts[0].exit_code, 0);
5121            let auth_evidence = receipt
5122                .auth_evidence
5123                .as_ref()
5124                .expect("receipt records auth evidence");
5125            assert_eq!(auth_evidence.schema_version, "shipper.auth_evidence.v1");
5126            assert_eq!(auth_evidence.registry, "crates-io");
5127            assert_eq!(
5128                auth_evidence.auth_mode,
5129                AuthEvidenceMode::CargoTokenWithOidcContext
5130            );
5131            assert!(auth_evidence.token_detected);
5132            assert!(auth_evidence.oidc_request_url_present);
5133            assert!(auth_evidence.oidc_request_token_present);
5134            let receipt_json = serde_json::to_string(&receipt).expect("serialize receipt");
5135            assert!(!receipt_json.contains("release-secret-token"));
5136            assert!(!receipt_json.contains("oidc-request-token"));
5137            let state = state::load_state(&td.path().join(".shipper"))
5138                .expect("load state")
5139                .expect("state exists");
5140            assert_eq!(state.attempt_history.len(), 1);
5141            assert_eq!(state.attempt_history[0].package, "demo");
5142            assert_eq!(state.attempt_history[0].version, "0.1.0");
5143            assert_eq!(state.attempt_history[0].attempt, 1);
5144            assert_eq!(state.attempt_history[0].max_attempts, opts.max_attempts);
5145            assert!(state.attempt_history[0].error_class.is_none());
5146            assert!(state.attempt_history[0].next_attempt_at.is_none());
5147            let events = events::EventLog::read_from_file(&td.path().join(".shipper/events.jsonl"))
5148                .expect("read events");
5149            let event_auth_evidence = events
5150                .all_events()
5151                .iter()
5152                .find_map(|event| match &event.event_type {
5153                    EventType::AuthEvidenceRecorded { evidence } => Some(evidence),
5154                    _ => None,
5155                })
5156                .expect("auth evidence event");
5157            assert_eq!(event_auth_evidence, auth_evidence);
5158            assert!(
5159                events
5160                    .all_events()
5161                    .iter()
5162                    .any(|event| matches!(event.event_type, EventType::ReadinessStarted { .. }))
5163            );
5164            assert!(
5165                events
5166                    .all_events()
5167                    .iter()
5168                    .any(|event| matches!(event.event_type, EventType::ReadinessPoll { .. }))
5169            );
5170            assert!(
5171                events
5172                    .all_events()
5173                    .iter()
5174                    .any(|event| matches!(event.event_type, EventType::ReadinessComplete { .. }))
5175            );
5176            server.join();
5177        });
5178    }
5179
5180    #[test]
5181    #[serial]
5182    fn run_publish_receipt_persisted_to_disk() {
5183        let td = tempdir().expect("tempdir");
5184        let bin = td.path().join("bin");
5185        write_fake_tools(&bin);
5186        let env_vars = fake_program_env_vars(&bin);
5187        temp_env::with_vars(env_vars, || {
5188            let server = spawn_registry_server(
5189                std::collections::BTreeMap::from([(
5190                    "/api/v1/crates/demo/0.1.0".to_string(),
5191                    vec![(200, "{}".to_string())],
5192                )]),
5193                1,
5194            );
5195            let ws = planned_workspace(td.path(), server.base_url.clone());
5196            let opts = default_opts(PathBuf::from(".shipper"));
5197
5198            let mut reporter = CollectingReporter::default();
5199            let _ = run_publish(&ws, &opts, &mut reporter).expect("publish");
5200
5201            let state_dir = td.path().join(".shipper");
5202            let receipt_path = state::receipt_path(&state_dir);
5203            assert!(receipt_path.exists());
5204
5205            let receipt_json = fs::read_to_string(&receipt_path).expect("read receipt");
5206            let parsed: Receipt = serde_json::from_str(&receipt_json).expect("parse receipt");
5207            assert_eq!(parsed.plan_id, "plan-demo");
5208            assert_eq!(parsed.receipt_version, "shipper.receipt.v2");
5209            server.join();
5210        });
5211    }
5212
5213    #[test]
5214    #[serial]
5215    fn run_publish_dirty_git_fails_when_not_allowed() {
5216        let td = tempdir().expect("tempdir");
5217        let bin = td.path().join("bin");
5218        write_fake_tools(&bin);
5219        let mut env_vars = fake_program_env_vars(&bin);
5220        env_vars.extend([("SHIPPER_GIT_CLEAN", Some("0".to_string()))]);
5221        temp_env::with_vars(env_vars, || {
5222            let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
5223            let mut opts = default_opts(PathBuf::from(".shipper"));
5224            opts.allow_dirty = false;
5225
5226            let mut reporter = CollectingReporter::default();
5227            let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
5228            let msg = format!("{err:#}");
5229            assert!(
5230                msg.contains("dirty") || msg.contains("uncommitted") || msg.contains("git"),
5231                "unexpected error: {msg}"
5232            );
5233        });
5234    }
5235
5236    #[test]
5237    #[serial]
5238    fn run_publish_state_attempts_counter_increments_on_retry() {
5239        let td = tempdir().expect("tempdir");
5240        let bin = td.path().join("bin");
5241        write_fake_tools(&bin);
5242        with_test_env(
5243            &bin,
5244            vec![
5245                ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
5246                (
5247                    "SHIPPER_CARGO_STDERR",
5248                    Some("rate limit exceeded".to_string()),
5249                ),
5250            ],
5251            || {
5252                // All registry checks return 404 so the package is never found
5253                let server = spawn_registry_server(
5254                    std::collections::BTreeMap::from([
5255                        (
5256                            "/api/v1/crates/demo/0.1.0".to_string(),
5257                            vec![
5258                                (404, "{}".to_string()),
5259                                (404, "{}".to_string()),
5260                                (404, "{}".to_string()),
5261                                (404, "{}".to_string()),
5262                            ],
5263                        ),
5264                        (
5265                            "/api/v1/crates/demo".to_string(),
5266                            vec![(200, "{}".to_string())],
5267                        ),
5268                    ]),
5269                    5,
5270                );
5271                let ws = planned_workspace(td.path(), server.base_url.clone());
5272                let mut opts = default_opts(PathBuf::from(".shipper"));
5273                opts.max_attempts = 2;
5274                opts.base_delay = Duration::from_millis(0);
5275                opts.max_delay = Duration::from_millis(0);
5276
5277                let mut reporter = CollectingReporter::default();
5278                let _ = run_publish(&ws, &opts, &mut reporter);
5279
5280                let st = state::load_state(&td.path().join(".shipper"))
5281                    .expect("load")
5282                    .expect("exists");
5283                let pkg = st.packages.get("demo@0.1.0").expect("pkg");
5284                assert_eq!(pkg.attempts, 2, "expected 2 attempts");
5285                assert_eq!(st.attempt_history.len(), 2);
5286                assert_eq!(st.attempt_history[0].attempt, 1);
5287                assert_eq!(
5288                    st.attempt_history[0].error_class,
5289                    Some(ErrorClass::Ambiguous)
5290                );
5291                assert!(st.attempt_history[0].next_attempt_at.is_some());
5292                assert_eq!(st.attempt_history[1].attempt, 2);
5293                assert_eq!(
5294                    st.attempt_history[1].error_class,
5295                    Some(ErrorClass::Ambiguous)
5296                );
5297                assert!(st.attempt_history[1].next_attempt_at.is_none());
5298                let events =
5299                    events::EventLog::read_from_file(&td.path().join(".shipper/events.jsonl"))
5300                        .expect("read events");
5301                assert!(
5302                    events.all_events().iter().any(|event| matches!(
5303                        event.event_type,
5304                        EventType::RateLimitObserved { .. }
5305                    ))
5306                );
5307                assert!(
5308                    events
5309                        .all_events()
5310                        .iter()
5311                        .any(|event| matches!(event.event_type, EventType::RetryScheduled { .. }))
5312                );
5313                assert!(
5314                    events
5315                        .all_events()
5316                        .iter()
5317                        .any(|event| matches!(event.event_type, EventType::PublishWaiting { .. }))
5318                );
5319                server.join();
5320            },
5321        );
5322    }
5323
5324    #[test]
5325    #[serial]
5326    fn run_publish_permanent_failure_emits_failed_event() {
5327        let td = tempdir().expect("tempdir");
5328        let bin = td.path().join("bin");
5329        write_fake_tools(&bin);
5330        with_test_env(
5331            &bin,
5332            vec![
5333                ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
5334                (
5335                    "SHIPPER_CARGO_STDERR",
5336                    Some("permission denied".to_string()),
5337                ),
5338            ],
5339            || {
5340                let server = spawn_registry_server(
5341                    std::collections::BTreeMap::from([(
5342                        "/api/v1/crates/demo/0.1.0".to_string(),
5343                        vec![(404, "{}".to_string()), (404, "{}".to_string())],
5344                    )]),
5345                    2,
5346                );
5347                let ws = planned_workspace(td.path(), server.base_url.clone());
5348                let mut opts = default_opts(PathBuf::from(".shipper"));
5349                opts.base_delay = Duration::from_millis(0);
5350                opts.max_delay = Duration::from_millis(0);
5351
5352                let mut reporter = CollectingReporter::default();
5353                let _ = run_publish(&ws, &opts, &mut reporter);
5354
5355                let events_path = td.path().join(".shipper").join("events.jsonl");
5356                let log = crate::state::events::EventLog::read_from_file(&events_path)
5357                    .expect("read events");
5358                let events = log.all_events();
5359
5360                assert!(
5361                    events
5362                        .iter()
5363                        .any(|e| matches!(e.event_type, EventType::PackageFailed { .. }))
5364                );
5365                assert!(
5366                    events
5367                        .iter()
5368                        .any(|e| matches!(e.event_type, EventType::PackageAttempted { .. }))
5369                );
5370                server.join();
5371            },
5372        );
5373    }
5374
5375    #[test]
5376    #[serial]
5377    fn run_publish_multi_package_first_published_second_skipped() {
5378        let td = tempdir().expect("tempdir");
5379        let bin = td.path().join("bin");
5380        write_fake_tools(&bin);
5381        let mut env_vars = fake_program_env_vars(&bin);
5382        env_vars.extend([("SHIPPER_CARGO_EXIT", Some("0".to_string()))]);
5383        temp_env::with_vars(env_vars, || {
5384            let server = spawn_registry_server(
5385                std::collections::BTreeMap::from([
5386                    (
5387                        "/api/v1/crates/alpha/1.0.0".to_string(),
5388                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
5389                    ),
5390                    (
5391                        "/api/v1/crates/beta/2.0.0".to_string(),
5392                        vec![(200, "{}".to_string())],
5393                    ),
5394                ]),
5395                3,
5396            );
5397            let mut ws = planned_workspace(td.path(), server.base_url.clone());
5398            ws.plan.packages = vec![
5399                PlannedPackage {
5400                    name: "alpha".to_string(),
5401                    version: "1.0.0".to_string(),
5402                    manifest_path: td.path().join("alpha/Cargo.toml"),
5403                    regime: None,
5404                },
5405                PlannedPackage {
5406                    name: "beta".to_string(),
5407                    version: "2.0.0".to_string(),
5408                    manifest_path: td.path().join("beta/Cargo.toml"),
5409                    regime: None,
5410                },
5411            ];
5412            let opts = default_opts(PathBuf::from(".shipper"));
5413
5414            let mut reporter = CollectingReporter::default();
5415            let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
5416
5417            assert_eq!(receipt.packages.len(), 2);
5418            assert!(matches!(receipt.packages[0].state, PackageState::Published));
5419            assert!(matches!(
5420                receipt.packages[1].state,
5421                PackageState::Skipped { .. }
5422            ));
5423            assert_eq!(receipt.packages[0].name, "alpha");
5424            assert_eq!(receipt.packages[1].name, "beta");
5425            server.join();
5426        });
5427    }
5428
5429    #[test]
5430    fn backoff_delay_linear_strategy() {
5431        let base = Duration::from_millis(100);
5432        let max = Duration::from_millis(500);
5433        let d1 = backoff_delay(base, max, 1, crate::retry::RetryStrategyType::Linear, 0.0);
5434        let d3 = backoff_delay(base, max, 3, crate::retry::RetryStrategyType::Linear, 0.0);
5435        let d20 = backoff_delay(base, max, 20, crate::retry::RetryStrategyType::Linear, 0.0);
5436
5437        assert_eq!(d1, Duration::from_millis(100));
5438        assert!(d3 > d1);
5439        assert!(d20 <= max, "linear delay should be capped at max");
5440    }
5441
5442    #[test]
5443    fn backoff_delay_constant_strategy() {
5444        let base = Duration::from_millis(200);
5445        let max = Duration::from_millis(1000);
5446        let d1 = backoff_delay(base, max, 1, crate::retry::RetryStrategyType::Constant, 0.0);
5447        let d5 = backoff_delay(base, max, 5, crate::retry::RetryStrategyType::Constant, 0.0);
5448        let d10 = backoff_delay(
5449            base,
5450            max,
5451            10,
5452            crate::retry::RetryStrategyType::Constant,
5453            0.0,
5454        );
5455
5456        assert_eq!(d1, base);
5457        assert_eq!(d5, base);
5458        assert_eq!(d10, base);
5459    }
5460
5461    #[test]
5462    fn backoff_delay_immediate_strategy() {
5463        let base = Duration::from_millis(200);
5464        let max = Duration::from_millis(1000);
5465        let d1 = backoff_delay(
5466            base,
5467            max,
5468            1,
5469            crate::retry::RetryStrategyType::Immediate,
5470            0.0,
5471        );
5472        let d5 = backoff_delay(
5473            base,
5474            max,
5475            5,
5476            crate::retry::RetryStrategyType::Immediate,
5477            0.0,
5478        );
5479
5480        assert_eq!(d1, Duration::ZERO);
5481        assert_eq!(d5, Duration::ZERO);
5482    }
5483
5484    #[test]
5485    fn backoff_delay_exponential_zero_jitter_is_deterministic() {
5486        let base = Duration::from_millis(100);
5487        let max = Duration::from_secs(10);
5488        let d1a = backoff_delay(
5489            base,
5490            max,
5491            1,
5492            crate::retry::RetryStrategyType::Exponential,
5493            0.0,
5494        );
5495        let d1b = backoff_delay(
5496            base,
5497            max,
5498            1,
5499            crate::retry::RetryStrategyType::Exponential,
5500            0.0,
5501        );
5502        assert_eq!(d1a, d1b);
5503        assert_eq!(d1a, base);
5504    }
5505
5506    #[test]
5507    fn classify_cargo_failure_rate_limit() {
5508        let (class, _msg) = classify_cargo_failure("HTTP 429 too many requests", "");
5509        assert_eq!(class, ErrorClass::Retryable);
5510    }
5511
5512    #[test]
5513    fn classify_cargo_failure_timeout() {
5514        let (class, _msg) = classify_cargo_failure("timeout talking to server", "");
5515        assert_eq!(class, ErrorClass::Retryable);
5516    }
5517
5518    #[test]
5519    fn classify_cargo_failure_service_unavailable() {
5520        let (class, _msg) = classify_cargo_failure("HTTP 503 service unavailable", "");
5521        assert_eq!(class, ErrorClass::Retryable);
5522    }
5523
5524    #[test]
5525    fn classify_cargo_failure_auth_failure() {
5526        let (class, _msg) = classify_cargo_failure("permission denied", "");
5527        assert_eq!(class, ErrorClass::Permanent);
5528    }
5529
5530    #[test]
5531    fn classify_cargo_failure_unknown_error_is_ambiguous() {
5532        let (class, _msg) = classify_cargo_failure("something totally unexpected", "");
5533        assert_eq!(class, ErrorClass::Ambiguous);
5534    }
5535
5536    #[test]
5537    fn short_state_covers_all_variants() {
5538        assert_eq!(short_state(&PackageState::Pending), "pending");
5539        assert_eq!(short_state(&PackageState::Uploaded), "uploaded");
5540        assert_eq!(short_state(&PackageState::Published), "published");
5541        assert_eq!(
5542            short_state(&PackageState::Skipped {
5543                reason: "already published".to_string()
5544            }),
5545            "skipped"
5546        );
5547        assert_eq!(
5548            short_state(&PackageState::Failed {
5549                class: ErrorClass::Retryable,
5550                message: "timeout".to_string()
5551            }),
5552            "failed"
5553        );
5554        assert_eq!(
5555            short_state(&PackageState::Failed {
5556                class: ErrorClass::Ambiguous,
5557                message: "unknown".to_string()
5558            }),
5559            "failed"
5560        );
5561        assert_eq!(
5562            short_state(&PackageState::Ambiguous {
5563                message: "not sure".to_string()
5564            }),
5565            "ambiguous"
5566        );
5567    }
5568
5569    #[test]
5570    fn pkg_key_formats_correctly() {
5571        assert_eq!(pkg_key("my-crate", "1.2.3"), "my-crate@1.2.3");
5572        assert_eq!(pkg_key("a", "0.0.1"), "a@0.0.1");
5573        assert_eq!(pkg_key("foo_bar-baz", "10.20.30"), "foo_bar-baz@10.20.30");
5574    }
5575
5576    #[test]
5577    #[serial]
5578    fn run_publish_skipped_package_receipt_has_empty_evidence() {
5579        let td = tempdir().expect("tempdir");
5580        let bin = td.path().join("bin");
5581        write_fake_tools(&bin);
5582        let env_vars = fake_program_env_vars(&bin);
5583        temp_env::with_vars(env_vars, || {
5584            let server = spawn_registry_server(
5585                std::collections::BTreeMap::from([(
5586                    "/api/v1/crates/demo/0.1.0".to_string(),
5587                    vec![(200, "{}".to_string())],
5588                )]),
5589                1,
5590            );
5591            let ws = planned_workspace(td.path(), server.base_url.clone());
5592            let opts = default_opts(PathBuf::from(".shipper"));
5593
5594            let mut reporter = CollectingReporter::default();
5595            let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
5596
5597            assert_eq!(receipt.packages.len(), 1);
5598            assert!(matches!(
5599                receipt.packages[0].state,
5600                PackageState::Skipped { .. }
5601            ));
5602            assert!(
5603                receipt.packages[0].evidence.attempts.is_empty(),
5604                "skipped packages should have no attempt evidence"
5605            );
5606            assert!(
5607                receipt.packages[0].evidence.readiness_checks.is_empty(),
5608                "skipped packages should have no readiness evidence"
5609            );
5610            server.join();
5611        });
5612    }
5613
5614    #[test]
5615    #[serial]
5616    fn run_publish_execution_result_is_success_when_all_published() {
5617        let td = tempdir().expect("tempdir");
5618        let bin = td.path().join("bin");
5619        write_fake_tools(&bin);
5620        let mut env_vars = fake_program_env_vars(&bin);
5621        env_vars.extend([("SHIPPER_CARGO_EXIT", Some("0".to_string()))]);
5622        temp_env::with_vars(env_vars, || {
5623            let server = spawn_registry_server(
5624                std::collections::BTreeMap::from([(
5625                    "/api/v1/crates/demo/0.1.0".to_string(),
5626                    vec![(404, "{}".to_string()), (200, "{}".to_string())],
5627                )]),
5628                2,
5629            );
5630            let ws = planned_workspace(td.path(), server.base_url.clone());
5631            let opts = default_opts(PathBuf::from(".shipper"));
5632
5633            let mut reporter = CollectingReporter::default();
5634            let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
5635
5636            // Check that the receipt event log was written
5637            assert!(receipt.event_log_path.exists());
5638            let log = crate::state::events::EventLog::read_from_file(&receipt.event_log_path)
5639                .expect("events");
5640            let finish_events: Vec<_> = log
5641                .all_events()
5642                .iter()
5643                .filter(|e| matches!(e.event_type, EventType::ExecutionFinished { .. }))
5644                .collect();
5645            assert_eq!(finish_events.len(), 1);
5646            if let EventType::ExecutionFinished { result } = &finish_events[0].event_type {
5647                assert!(
5648                    matches!(result, ExecutionResult::Success),
5649                    "expected Success, got {result:?}"
5650                );
5651            }
5652            server.join();
5653        });
5654    }
5655
5656    #[test]
5657    fn run_publish_force_skips_lock_timeout() {
5658        // When force=true, lock_timeout is set to ZERO. This test verifies
5659        // the opts are respected without blocking.
5660        let td = tempdir().expect("tempdir");
5661        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
5662        let state_dir = td.path().join(".shipper");
5663
5664        // Pre-create state with all packages published
5665        let mut packages = std::collections::BTreeMap::new();
5666        packages.insert(
5667            "demo@0.1.0".to_string(),
5668            PackageProgress {
5669                name: "demo".to_string(),
5670                version: "0.1.0".to_string(),
5671                attempts: 1,
5672                state: PackageState::Published,
5673                last_updated_at: Utc::now(),
5674            },
5675        );
5676        let st = ExecutionState {
5677            state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
5678            plan_id: ws.plan.plan_id.clone(),
5679            registry: ws.plan.registry.clone(),
5680            created_at: Utc::now(),
5681            updated_at: Utc::now(),
5682            attempt_history: Vec::new(),
5683            packages,
5684        };
5685        state::save_state(&state_dir, &st).expect("save");
5686
5687        let mut opts = default_opts(PathBuf::from(".shipper"));
5688        opts.force = true;
5689
5690        let mut reporter = CollectingReporter::default();
5691        let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
5692        assert!(receipt.packages.is_empty());
5693    }
5694
5695    #[test]
5696    #[serial]
5697    fn run_publish_resume_from_skips_before_and_warns() {
5698        let td = tempdir().expect("tempdir");
5699        let bin = td.path().join("bin");
5700        write_fake_tools(&bin);
5701        let mut env_vars = fake_program_env_vars(&bin);
5702        env_vars.extend([("SHIPPER_CARGO_EXIT", Some("0".to_string()))]);
5703        temp_env::with_vars(env_vars, || {
5704            let server = spawn_registry_server(
5705                std::collections::BTreeMap::from([
5706                    (
5707                        "/api/v1/crates/alpha/1.0.0".to_string(),
5708                        vec![(404, "{}".to_string())],
5709                    ),
5710                    (
5711                        "/api/v1/crates/beta/2.0.0".to_string(),
5712                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
5713                    ),
5714                ]),
5715                2,
5716            );
5717
5718            let mut ws = planned_workspace(td.path(), server.base_url.clone());
5719            ws.plan.packages = vec![
5720                PlannedPackage {
5721                    name: "alpha".to_string(),
5722                    version: "1.0.0".to_string(),
5723                    manifest_path: td.path().join("alpha/Cargo.toml"),
5724                    regime: None,
5725                },
5726                PlannedPackage {
5727                    name: "beta".to_string(),
5728                    version: "2.0.0".to_string(),
5729                    manifest_path: td.path().join("beta/Cargo.toml"),
5730                    regime: None,
5731                },
5732            ];
5733            let mut opts = default_opts(PathBuf::from(".shipper"));
5734            opts.resume_from = Some("beta".to_string());
5735
5736            let mut reporter = CollectingReporter::default();
5737            let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
5738
5739            // Only beta should be in receipt
5740            assert_eq!(receipt.packages.len(), 1);
5741            assert_eq!(receipt.packages[0].name, "beta");
5742
5743            // Alpha was pending, so it should produce a warning about skipping
5744            assert!(
5745                reporter
5746                    .warns
5747                    .iter()
5748                    .any(|w| w.contains("skipping") && w.contains("before resume point")),
5749                "expected warning about skipping alpha, got: {:?}",
5750                reporter.warns
5751            );
5752            server.join();
5753        });
5754    }
5755
5756    #[test]
5757    #[serial]
5758    fn run_publish_resume_from_already_done_skipped_silently() {
5759        let td = tempdir().expect("tempdir");
5760        let bin = td.path().join("bin");
5761        write_fake_tools(&bin);
5762        let mut env_vars = fake_program_env_vars(&bin);
5763        env_vars.extend([("SHIPPER_CARGO_EXIT", Some("0".to_string()))]);
5764        temp_env::with_vars(env_vars, || {
5765            let server = spawn_registry_server(
5766                std::collections::BTreeMap::from([(
5767                    "/api/v1/crates/beta/2.0.0".to_string(),
5768                    vec![(404, "{}".to_string()), (200, "{}".to_string())],
5769                )]),
5770                2,
5771            );
5772
5773            let mut ws = planned_workspace(td.path(), server.base_url.clone());
5774            ws.plan.packages = vec![
5775                PlannedPackage {
5776                    name: "alpha".to_string(),
5777                    version: "1.0.0".to_string(),
5778                    manifest_path: td.path().join("alpha/Cargo.toml"),
5779                    regime: None,
5780                },
5781                PlannedPackage {
5782                    name: "beta".to_string(),
5783                    version: "2.0.0".to_string(),
5784                    manifest_path: td.path().join("beta/Cargo.toml"),
5785                    regime: None,
5786                },
5787            ];
5788
5789            // Pre-create state with alpha already published
5790            let state_dir = td.path().join(".shipper");
5791            let mut packages = std::collections::BTreeMap::new();
5792            packages.insert(
5793                "alpha@1.0.0".to_string(),
5794                PackageProgress {
5795                    name: "alpha".to_string(),
5796                    version: "1.0.0".to_string(),
5797                    attempts: 1,
5798                    state: PackageState::Published,
5799                    last_updated_at: Utc::now(),
5800                },
5801            );
5802            let st = ExecutionState {
5803                state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
5804                plan_id: ws.plan.plan_id.clone(),
5805                registry: ws.plan.registry.clone(),
5806                created_at: Utc::now(),
5807                updated_at: Utc::now(),
5808                attempt_history: Vec::new(),
5809                packages,
5810            };
5811            state::save_state(&state_dir, &st).expect("save");
5812
5813            let mut opts = default_opts(PathBuf::from(".shipper"));
5814            opts.resume_from = Some("beta".to_string());
5815
5816            let mut reporter = CollectingReporter::default();
5817            let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
5818
5819            // Alpha should be silently skipped (already complete), beta published
5820            assert_eq!(receipt.packages.len(), 1);
5821            assert_eq!(receipt.packages[0].name, "beta");
5822
5823            // Alpha should produce an info about "already complete" not a warning
5824            assert!(
5825                reporter
5826                    .infos
5827                    .iter()
5828                    .any(|i| i.contains("already complete") && i.contains("alpha")),
5829                "expected info about alpha being already complete, got: {:?}",
5830                reporter.infos
5831            );
5832            server.join();
5833        });
5834    }
5835
5836    #[test]
5837    fn update_state_transitions_correctly() {
5838        let td = tempdir().expect("tempdir");
5839        let state_dir = td.path().join(".shipper");
5840        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
5841
5842        let mut st = init_state(&ws, &state_dir).expect("init");
5843        let key = "demo@0.1.0";
5844
5845        // Pending -> Uploaded
5846        update_state(&mut st, &state_dir, key, PackageState::Uploaded).expect("update");
5847        assert!(matches!(
5848            st.packages.get(key).unwrap().state,
5849            PackageState::Uploaded
5850        ));
5851
5852        // Uploaded -> Published
5853        update_state(&mut st, &state_dir, key, PackageState::Published).expect("update");
5854        assert!(matches!(
5855            st.packages.get(key).unwrap().state,
5856            PackageState::Published
5857        ));
5858
5859        // Verify persisted to disk
5860        let loaded = state::load_state(&state_dir)
5861            .expect("load")
5862            .expect("exists");
5863        assert!(matches!(
5864            loaded.packages.get(key).unwrap().state,
5865            PackageState::Published
5866        ));
5867    }
5868
5869    #[test]
5870    fn update_state_to_failed() {
5871        let td = tempdir().expect("tempdir");
5872        let state_dir = td.path().join(".shipper");
5873        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
5874
5875        let mut st = init_state(&ws, &state_dir).expect("init");
5876        let key = "demo@0.1.0";
5877
5878        let failed = PackageState::Failed {
5879            class: ErrorClass::Permanent,
5880            message: "auth failure".to_string(),
5881        };
5882        update_state(&mut st, &state_dir, key, failed).expect("update");
5883
5884        let pkg = st.packages.get(key).unwrap();
5885        match &pkg.state {
5886            PackageState::Failed { class, message } => {
5887                assert_eq!(*class, ErrorClass::Permanent);
5888                assert_eq!(message, "auth failure");
5889            }
5890            other => panic!("expected Failed, got {other:?}"),
5891        }
5892    }
5893
5894    #[test]
5895    fn update_state_to_skipped() {
5896        let td = tempdir().expect("tempdir");
5897        let state_dir = td.path().join(".shipper");
5898        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
5899
5900        let mut st = init_state(&ws, &state_dir).expect("init");
5901        let key = "demo@0.1.0";
5902
5903        let skipped = PackageState::Skipped {
5904            reason: "already published".to_string(),
5905        };
5906        update_state(&mut st, &state_dir, key, skipped).expect("update");
5907
5908        match &st.packages.get(key).unwrap().state {
5909            PackageState::Skipped { reason } => {
5910                assert_eq!(reason, "already published");
5911            }
5912            other => panic!("expected Skipped, got {other:?}"),
5913        }
5914    }
5915
5916    #[test]
5917    fn receipt_serialization_roundtrip() {
5918        let receipt = Receipt {
5919            receipt_version: "shipper.receipt.v2".to_string(),
5920            plan_id: "plan-test-123".to_string(),
5921            registry: Registry {
5922                name: "crates-io".to_string(),
5923                api_base: "https://crates.io".to_string(),
5924                index_base: None,
5925            },
5926            started_at: Utc::now(),
5927            finished_at: Utc::now(),
5928            packages: vec![
5929                PackageReceipt {
5930                    name: "alpha".to_string(),
5931                    version: "1.0.0".to_string(),
5932                    attempts: 1,
5933                    state: PackageState::Published,
5934                    started_at: Utc::now(),
5935                    finished_at: Utc::now(),
5936                    duration_ms: 1234,
5937                    evidence: crate::types::PackageEvidence {
5938                        attempts: vec![AttemptEvidence {
5939                            attempt_number: 1,
5940                            command: "cargo publish -p alpha".to_string(),
5941                            exit_code: 0,
5942                            stdout_tail: "Uploading alpha".to_string(),
5943                            stderr_tail: String::new(),
5944                            timestamp: Utc::now(),
5945                            duration: Duration::from_millis(500),
5946                        }],
5947                        readiness_checks: vec![ReadinessEvidence {
5948                            attempt: 1,
5949                            visible: true,
5950                            timestamp: Utc::now(),
5951                            delay_before: Duration::from_millis(100),
5952                        }],
5953                    },
5954                    compromised_at: None,
5955                    compromised_by: None,
5956                    superseded_by: None,
5957                },
5958                PackageReceipt {
5959                    name: "beta".to_string(),
5960                    version: "2.0.0".to_string(),
5961                    attempts: 0,
5962                    state: PackageState::Skipped {
5963                        reason: "already published".to_string(),
5964                    },
5965                    started_at: Utc::now(),
5966                    finished_at: Utc::now(),
5967                    duration_ms: 10,
5968                    evidence: crate::types::PackageEvidence {
5969                        attempts: vec![],
5970                        readiness_checks: vec![],
5971                    },
5972                    compromised_at: None,
5973                    compromised_by: None,
5974                    superseded_by: None,
5975                },
5976            ],
5977            event_log_path: PathBuf::from(".shipper/events.jsonl"),
5978            git_context: None,
5979            environment: environment::collect_environment_fingerprint(),
5980            auth_evidence: None,
5981        };
5982
5983        let json = serde_json::to_string_pretty(&receipt).expect("serialize");
5984        let parsed: Receipt = serde_json::from_str(&json).expect("deserialize");
5985        assert_eq!(parsed.plan_id, receipt.plan_id);
5986        assert_eq!(parsed.receipt_version, receipt.receipt_version);
5987        assert_eq!(parsed.packages.len(), 2);
5988        assert!(matches!(parsed.packages[0].state, PackageState::Published));
5989        assert!(matches!(
5990            parsed.packages[1].state,
5991            PackageState::Skipped { .. }
5992        ));
5993        assert_eq!(parsed.packages[0].evidence.attempts.len(), 1);
5994        assert_eq!(parsed.packages[0].evidence.readiness_checks.len(), 1);
5995    }
5996
5997    #[test]
5998    fn execution_state_serialization_roundtrip() {
5999        let mut packages = BTreeMap::new();
6000        packages.insert(
6001            "demo@0.1.0".to_string(),
6002            PackageProgress {
6003                name: "demo".to_string(),
6004                version: "0.1.0".to_string(),
6005                attempts: 3,
6006                state: PackageState::Failed {
6007                    class: ErrorClass::Retryable,
6008                    message: "timeout".to_string(),
6009                },
6010                last_updated_at: Utc::now(),
6011            },
6012        );
6013        let st = ExecutionState {
6014            state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
6015            plan_id: "plan-serde-test".to_string(),
6016            registry: Registry {
6017                name: "crates-io".to_string(),
6018                api_base: "https://crates.io".to_string(),
6019                index_base: None,
6020            },
6021            created_at: Utc::now(),
6022            updated_at: Utc::now(),
6023            attempt_history: Vec::new(),
6024            packages,
6025        };
6026
6027        let json = serde_json::to_string(&st).expect("serialize");
6028        let parsed: ExecutionState = serde_json::from_str(&json).expect("deserialize");
6029        assert_eq!(parsed.plan_id, st.plan_id);
6030        assert_eq!(parsed.packages.len(), 1);
6031        let pkg = parsed.packages.get("demo@0.1.0").expect("pkg");
6032        assert_eq!(pkg.attempts, 3);
6033        match &pkg.state {
6034            PackageState::Failed { class, message } => {
6035                assert_eq!(*class, ErrorClass::Retryable);
6036                assert_eq!(message, "timeout");
6037            }
6038            other => panic!("expected Failed, got {other:?}"),
6039        }
6040    }
6041
6042    #[test]
6043    fn collecting_reporter_tracks_all_message_types() {
6044        let mut reporter = CollectingReporter::default();
6045        reporter.info("info-1");
6046        reporter.info("info-2");
6047        reporter.warn("warn-1");
6048        reporter.error("error-1");
6049        reporter.error("error-2");
6050        reporter.error("error-3");
6051
6052        assert_eq!(reporter.infos.len(), 2);
6053        assert_eq!(reporter.warns.len(), 1);
6054        assert_eq!(reporter.errors.len(), 3);
6055        assert_eq!(reporter.infos[0], "info-1");
6056        assert_eq!(reporter.warns[0], "warn-1");
6057        assert_eq!(reporter.errors[2], "error-3");
6058    }
6059
6060    #[test]
6061    fn verify_published_disabled_does_single_check() {
6062        // When readiness is disabled, it still does one version_exists check
6063        let server = spawn_registry_server(
6064            std::collections::BTreeMap::from([(
6065                "/api/v1/crates/demo/0.1.0".to_string(),
6066                vec![(200, "{}".to_string())],
6067            )]),
6068            1,
6069        );
6070        let reg = RegistryClient::new(Registry {
6071            name: "crates-io".to_string(),
6072            api_base: server.base_url.clone(),
6073            index_base: None,
6074        })
6075        .expect("client");
6076
6077        let config = crate::types::ReadinessConfig {
6078            enabled: false,
6079            method: crate::types::ReadinessMethod::Api,
6080            initial_delay: Duration::from_millis(0),
6081            max_delay: Duration::from_millis(10),
6082            max_total_wait: Duration::from_millis(0),
6083            poll_interval: Duration::from_millis(1),
6084            jitter_factor: 0.0,
6085            index_path: None,
6086            prefer_index: false,
6087        };
6088
6089        let mut reporter = CollectingReporter::default();
6090        let td = tempdir().expect("tempdir");
6091        let events_path = td.path().join("events.jsonl");
6092        let mut event_log = events::EventLog::new();
6093        let (ok, evidence) = verify_published(
6094            &reg,
6095            "demo",
6096            "0.1.0",
6097            &config,
6098            &mut reporter,
6099            &mut event_log,
6100            &events_path,
6101            "demo@0.1.0",
6102        )
6103        .expect("verify");
6104        assert!(
6105            ok,
6106            "disabled readiness with 200 response should return true"
6107        );
6108        assert_eq!(
6109            evidence.len(),
6110            1,
6111            "disabled readiness does exactly one check"
6112        );
6113        assert!(evidence[0].visible);
6114        server.join();
6115    }
6116
6117    #[test]
6118    #[serial]
6119    fn run_publish_already_published_packages_skipped_in_state() {
6120        let td = tempdir().expect("tempdir");
6121        let bin = td.path().join("bin");
6122        write_fake_tools(&bin);
6123        let env_vars = fake_program_env_vars(&bin);
6124        temp_env::with_vars(env_vars, || {
6125            let server = spawn_registry_server(
6126                std::collections::BTreeMap::from([(
6127                    "/api/v1/crates/demo/0.1.0".to_string(),
6128                    vec![(200, "{}".to_string())],
6129                )]),
6130                1,
6131            );
6132            let ws = planned_workspace(td.path(), server.base_url.clone());
6133            let opts = default_opts(PathBuf::from(".shipper"));
6134
6135            let mut reporter = CollectingReporter::default();
6136            let _ = run_publish(&ws, &opts, &mut reporter).expect("publish");
6137
6138            // Verify state file reflects skipped
6139            let st = state::load_state(&td.path().join(".shipper"))
6140                .expect("load")
6141                .expect("exists");
6142            let pkg = st.packages.get("demo@0.1.0").expect("pkg");
6143            assert!(
6144                matches!(pkg.state, PackageState::Skipped { .. }),
6145                "expected Skipped in state, got {:?}",
6146                pkg.state
6147            );
6148            server.join();
6149        });
6150    }
6151
6152    #[test]
6153    fn policy_effects_default_options() {
6154        let opts = default_opts(PathBuf::from(".shipper"));
6155        let effects = policy_effects(&opts);
6156        // Default policy is Balanced, which runs dry-run but skips ownership by default
6157        // (skip_ownership_check=true in default_opts)
6158        assert!(effects.run_dry_run);
6159    }
6160
6161    // ── Retry logic edge-case tests ────────────────────────────────────
6162
6163    #[test]
6164    #[serial]
6165    fn run_publish_zero_max_attempts_skips_publish_loop() {
6166        let td = tempdir().expect("tempdir");
6167        let bin = td.path().join("bin");
6168        write_fake_tools(&bin);
6169        with_test_env(
6170            &bin,
6171            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
6172            || {
6173                // Registry says version doesn't exist, so engine enters the publish loop
6174                // with max_attempts=0 → loop body never executes → package stays Pending
6175                let server = spawn_registry_server(
6176                    std::collections::BTreeMap::from([(
6177                        "/api/v1/crates/demo/0.1.0".to_string(),
6178                        vec![(404, "{}".to_string())],
6179                    )]),
6180                    1,
6181                );
6182                let ws = planned_workspace(td.path(), server.base_url.clone());
6183                let mut opts = default_opts(PathBuf::from(".shipper"));
6184                opts.max_attempts = 0;
6185
6186                let mut reporter = CollectingReporter::default();
6187                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
6188
6189                // With 0 max_attempts the loop never runs; package stays Pending
6190                assert_eq!(receipt.packages.len(), 1);
6191                assert!(
6192                    matches!(receipt.packages[0].state, PackageState::Pending),
6193                    "expected Pending with 0 max_attempts, got {:?}",
6194                    receipt.packages[0].state
6195                );
6196
6197                let st = state::load_state(&td.path().join(".shipper"))
6198                    .expect("load")
6199                    .expect("exists");
6200                let pkg = st.packages.get("demo@0.1.0").expect("pkg");
6201                assert_eq!(pkg.attempts, 0, "no attempts should have been made");
6202                server.join();
6203            },
6204        );
6205    }
6206
6207    #[test]
6208    #[serial]
6209    fn run_publish_max_retries_exceeded_marks_failed() {
6210        let td = tempdir().expect("tempdir");
6211        let bin = td.path().join("bin");
6212        write_fake_tools(&bin);
6213        with_test_env(
6214            &bin,
6215            vec![
6216                ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
6217                (
6218                    "SHIPPER_CARGO_STDERR",
6219                    Some("HTTP 503 service unavailable".to_string()),
6220                ),
6221            ],
6222            || {
6223                // 3 attempts * (version check + registry fallback) + final check = many 404s
6224                let server = spawn_registry_server(
6225                    std::collections::BTreeMap::from([(
6226                        "/api/v1/crates/demo/0.1.0".to_string(),
6227                        vec![
6228                            (404, "{}".to_string()),
6229                            (404, "{}".to_string()),
6230                            (404, "{}".to_string()),
6231                            (404, "{}".to_string()),
6232                            (404, "{}".to_string()),
6233                        ],
6234                    )]),
6235                    5,
6236                );
6237                let ws = planned_workspace(td.path(), server.base_url.clone());
6238                let mut opts = default_opts(PathBuf::from(".shipper"));
6239                opts.max_attempts = 3;
6240                opts.base_delay = Duration::from_millis(0);
6241                opts.max_delay = Duration::from_millis(0);
6242
6243                let mut reporter = CollectingReporter::default();
6244                let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
6245                assert!(format!("{err:#}").contains("failed"));
6246
6247                let st = state::load_state(&td.path().join(".shipper"))
6248                    .expect("load")
6249                    .expect("exists");
6250                let pkg = st.packages.get("demo@0.1.0").expect("pkg");
6251                assert_eq!(pkg.attempts, 3, "should have exhausted all 3 attempts");
6252                assert!(
6253                    matches!(pkg.state, PackageState::Failed { .. }),
6254                    "expected Failed, got {:?}",
6255                    pkg.state
6256                );
6257                server.join();
6258            },
6259        );
6260    }
6261
6262    #[test]
6263    #[serial]
6264    fn run_publish_single_attempt_succeeds_on_first_try() {
6265        let td = tempdir().expect("tempdir");
6266        let bin = td.path().join("bin");
6267        write_fake_tools(&bin);
6268        with_test_env(
6269            &bin,
6270            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
6271            || {
6272                let server = spawn_registry_server(
6273                    std::collections::BTreeMap::from([(
6274                        "/api/v1/crates/demo/0.1.0".to_string(),
6275                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
6276                    )]),
6277                    2,
6278                );
6279                let ws = planned_workspace(td.path(), server.base_url.clone());
6280                let mut opts = default_opts(PathBuf::from(".shipper"));
6281                opts.max_attempts = 1;
6282
6283                let mut reporter = CollectingReporter::default();
6284                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
6285                assert_eq!(receipt.packages[0].attempts, 1);
6286                assert!(matches!(receipt.packages[0].state, PackageState::Published));
6287                server.join();
6288            },
6289        );
6290    }
6291
6292    // ── State persistence: Uploaded vs Published transitions ──────────
6293
6294    #[test]
6295    fn state_transition_pending_to_uploaded_persists() {
6296        let td = tempdir().expect("tempdir");
6297        let state_dir = td.path().join(".shipper");
6298        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
6299
6300        let mut st = init_state(&ws, &state_dir).expect("init");
6301        let key = "demo@0.1.0";
6302
6303        update_state(&mut st, &state_dir, key, PackageState::Uploaded).expect("update");
6304
6305        // Verify in-memory
6306        assert!(matches!(
6307            st.packages.get(key).unwrap().state,
6308            PackageState::Uploaded
6309        ));
6310
6311        // Verify on disk
6312        let loaded = state::load_state(&state_dir)
6313            .expect("load")
6314            .expect("exists");
6315        assert!(matches!(
6316            loaded.packages.get(key).unwrap().state,
6317            PackageState::Uploaded
6318        ));
6319    }
6320
6321    #[test]
6322    fn state_transition_uploaded_to_published_persists() {
6323        let td = tempdir().expect("tempdir");
6324        let state_dir = td.path().join(".shipper");
6325        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
6326
6327        let mut st = init_state(&ws, &state_dir).expect("init");
6328        let key = "demo@0.1.0";
6329
6330        // Pending -> Uploaded -> Published
6331        update_state(&mut st, &state_dir, key, PackageState::Uploaded).expect("upload");
6332        update_state(&mut st, &state_dir, key, PackageState::Published).expect("publish");
6333
6334        let loaded = state::load_state(&state_dir)
6335            .expect("load")
6336            .expect("exists");
6337        assert!(matches!(
6338            loaded.packages.get(key).unwrap().state,
6339            PackageState::Published
6340        ));
6341    }
6342
6343    #[test]
6344    fn state_transition_pending_to_failed_persists() {
6345        let td = tempdir().expect("tempdir");
6346        let state_dir = td.path().join(".shipper");
6347        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
6348
6349        let mut st = init_state(&ws, &state_dir).expect("init");
6350        let key = "demo@0.1.0";
6351
6352        let failed = PackageState::Failed {
6353            class: ErrorClass::Retryable,
6354            message: "service unavailable".to_string(),
6355        };
6356        update_state(&mut st, &state_dir, key, failed).expect("update");
6357
6358        let loaded = state::load_state(&state_dir)
6359            .expect("load")
6360            .expect("exists");
6361        match &loaded.packages.get(key).unwrap().state {
6362            PackageState::Failed { class, message } => {
6363                assert_eq!(*class, ErrorClass::Retryable);
6364                assert_eq!(message, "service unavailable");
6365            }
6366            other => panic!("expected Failed, got {other:?}"),
6367        }
6368    }
6369
6370    #[test]
6371    fn state_updated_at_advances_on_transition() {
6372        let td = tempdir().expect("tempdir");
6373        let state_dir = td.path().join(".shipper");
6374        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
6375
6376        let mut st = init_state(&ws, &state_dir).expect("init");
6377        let key = "demo@0.1.0";
6378        let initial_updated = st.updated_at;
6379
6380        // Small sleep to ensure time difference
6381        thread::sleep(Duration::from_millis(10));
6382
6383        update_state(&mut st, &state_dir, key, PackageState::Uploaded).expect("update");
6384        assert!(st.updated_at > initial_updated);
6385
6386        let pkg = st.packages.get(key).unwrap();
6387        assert!(pkg.last_updated_at >= initial_updated);
6388    }
6389
6390    // ── Error classification tests ─────────────────────────────────────
6391
6392    #[test]
6393    fn classify_cargo_failure_connection_refused_is_retryable() {
6394        let (class, _) = classify_cargo_failure("connection refused", "");
6395        assert_eq!(class, ErrorClass::Retryable);
6396    }
6397
6398    #[test]
6399    fn classify_cargo_failure_500_is_retryable() {
6400        let (class, _) = classify_cargo_failure("HTTP 500 internal server error", "");
6401        assert_eq!(class, ErrorClass::Retryable);
6402    }
6403
6404    #[test]
6405    fn classify_cargo_failure_version_exists_is_permanent() {
6406        let (class, _) = classify_cargo_failure("crate version `0.1.0` is already uploaded", "");
6407        assert_eq!(class, ErrorClass::Permanent);
6408    }
6409
6410    #[test]
6411    fn classify_cargo_failure_empty_output_is_ambiguous() {
6412        let (class, _) = classify_cargo_failure("", "");
6413        assert_eq!(class, ErrorClass::Ambiguous);
6414    }
6415
6416    #[test]
6417    fn classify_cargo_failure_message_is_nonempty() {
6418        let (_, msg) = classify_cargo_failure("timeout talking to server", "");
6419        assert!(!msg.is_empty(), "error message should be nonempty");
6420    }
6421
6422    #[test]
6423    fn classify_cargo_failure_stderr_vs_stdout() {
6424        // Some errors appear in stdout, some in stderr
6425        let (class_stderr, _) = classify_cargo_failure("HTTP 429 too many requests", "");
6426        let (class_stdout, _) = classify_cargo_failure("", "HTTP 429 too many requests");
6427        assert_eq!(class_stderr, ErrorClass::Retryable);
6428        // stdout-only message should also be classified
6429        assert!(
6430            class_stdout == ErrorClass::Retryable || class_stdout == ErrorClass::Ambiguous,
6431            "expected Retryable or Ambiguous from stdout, got {class_stdout:?}"
6432        );
6433    }
6434
6435    // ── State machine transition tests ────────────────────────────────
6436
6437    /// Helper: build a multi-package workspace for state machine tests.
6438    fn multi_package_workspace(
6439        workspace_root: &Path,
6440        api_base: String,
6441        packages: Vec<(&str, &str)>,
6442    ) -> PlannedWorkspace {
6443        PlannedWorkspace {
6444            workspace_root: workspace_root.to_path_buf(),
6445            plan: ReleasePlan {
6446                plan_version: "1".to_string(),
6447                plan_id: "plan-sm-test".to_string(),
6448                created_at: Utc::now(),
6449                registry: Registry {
6450                    name: "crates-io".to_string(),
6451                    api_base,
6452                    index_base: None,
6453                },
6454                packages: packages
6455                    .iter()
6456                    .map(|(name, ver)| PlannedPackage {
6457                        name: name.to_string(),
6458                        version: ver.to_string(),
6459                        manifest_path: workspace_root.join(*name).join("Cargo.toml"),
6460                        regime: None,
6461                    })
6462                    .collect(),
6463                dependencies: std::collections::BTreeMap::new(),
6464            },
6465            skipped: vec![],
6466        }
6467    }
6468
6469    // 1. Pending → Published (happy path, single crate)
6470    #[test]
6471    #[serial]
6472    fn sm_pending_to_published_happy_path() {
6473        let td = tempdir().expect("tempdir");
6474        let bin = td.path().join("bin");
6475        write_fake_tools(&bin);
6476        with_test_env(
6477            &bin,
6478            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
6479            || {
6480                let server = spawn_registry_server(
6481                    std::collections::BTreeMap::from([(
6482                        "/api/v1/crates/demo/0.1.0".to_string(),
6483                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
6484                    )]),
6485                    2,
6486                );
6487                let ws = planned_workspace(td.path(), server.base_url.clone());
6488                let opts = default_opts(PathBuf::from(".shipper"));
6489
6490                let mut reporter = CollectingReporter::default();
6491                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
6492
6493                assert_eq!(receipt.packages.len(), 1);
6494                assert!(matches!(receipt.packages[0].state, PackageState::Published));
6495                assert_eq!(receipt.packages[0].attempts, 1);
6496
6497                // Verify state on disk matches
6498                let st = state::load_state(&td.path().join(".shipper"))
6499                    .expect("load")
6500                    .expect("exists");
6501                let pkg = st.packages.get("demo@0.1.0").expect("pkg");
6502                assert!(matches!(pkg.state, PackageState::Published));
6503                server.join();
6504            },
6505        );
6506    }
6507
6508    // 2. Pending → Uploaded → Verified (two-phase with readiness check)
6509    #[test]
6510    #[serial]
6511    fn sm_pending_uploaded_verified_two_phase() {
6512        let td = tempdir().expect("tempdir");
6513        let bin = td.path().join("bin");
6514        write_fake_tools(&bin);
6515        with_test_env(
6516            &bin,
6517            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
6518            || {
6519                // First 404 = not yet published, second 200 = visible after readiness
6520                let server = spawn_registry_server(
6521                    std::collections::BTreeMap::from([(
6522                        "/api/v1/crates/demo/0.1.0".to_string(),
6523                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
6524                    )]),
6525                    2,
6526                );
6527                let ws = planned_workspace(td.path(), server.base_url.clone());
6528                let opts = default_opts(PathBuf::from(".shipper"));
6529
6530                let mut reporter = CollectingReporter::default();
6531                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
6532
6533                assert!(matches!(receipt.packages[0].state, PackageState::Published));
6534                // Verify the publish-then-verify flow happened
6535                assert!(reporter.infos.iter().any(|i| i.contains("publishing")));
6536                assert!(
6537                    reporter
6538                        .infos
6539                        .iter()
6540                        .any(|i| i.contains("verifying") || i.contains("visible"))
6541                );
6542                server.join();
6543            },
6544        );
6545    }
6546
6547    // 3. Pending → Failed (permanent error, no retry)
6548    #[test]
6549    #[serial]
6550    fn sm_pending_to_failed_permanent_no_retry() {
6551        let td = tempdir().expect("tempdir");
6552        let bin = td.path().join("bin");
6553        write_fake_tools(&bin);
6554        with_test_env(
6555            &bin,
6556            vec![
6557                ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
6558                (
6559                    "SHIPPER_CARGO_STDERR",
6560                    Some("permission denied".to_string()),
6561                ),
6562            ],
6563            || {
6564                let server = spawn_registry_server(
6565                    std::collections::BTreeMap::from([(
6566                        "/api/v1/crates/demo/0.1.0".to_string(),
6567                        vec![(404, "{}".to_string()), (404, "{}".to_string())],
6568                    )]),
6569                    2,
6570                );
6571                let ws = planned_workspace(td.path(), server.base_url.clone());
6572                let mut opts = default_opts(PathBuf::from(".shipper"));
6573                opts.max_attempts = 5; // should not retry on permanent error
6574                opts.base_delay = Duration::from_millis(0);
6575                opts.max_delay = Duration::from_millis(0);
6576
6577                let mut reporter = CollectingReporter::default();
6578                let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
6579                assert!(format!("{err:#}").contains("permanent failure"));
6580
6581                let st = state::load_state(&td.path().join(".shipper"))
6582                    .expect("load")
6583                    .expect("exists");
6584                let pkg = st.packages.get("demo@0.1.0").expect("pkg");
6585                // Should have only attempted once (permanent = no retry)
6586                assert_eq!(pkg.attempts, 1);
6587                assert!(matches!(
6588                    pkg.state,
6589                    PackageState::Failed {
6590                        class: ErrorClass::Permanent,
6591                        ..
6592                    }
6593                ));
6594                server.join();
6595            },
6596        );
6597    }
6598
6599    // 4. Pending → Uploaded → Failed (upload ok, verify failed)
6600    #[test]
6601    #[serial]
6602    fn sm_uploaded_then_verify_failed() {
6603        let td = tempdir().expect("tempdir");
6604        let bin = td.path().join("bin");
6605        write_fake_tools(&bin);
6606        with_test_env(
6607            &bin,
6608            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
6609            || {
6610                // cargo succeeds, but registry never shows the version
6611                let server = spawn_registry_server(
6612                    std::collections::BTreeMap::from([(
6613                        "/api/v1/crates/demo/0.1.0".to_string(),
6614                        vec![
6615                            (404, "{}".to_string()), // initial version_exists
6616                            (404, "{}".to_string()), // readiness check
6617                            (404, "{}".to_string()), // final chance
6618                        ],
6619                    )]),
6620                    3,
6621                );
6622                let ws = planned_workspace(td.path(), server.base_url.clone());
6623                let mut opts = default_opts(PathBuf::from(".shipper"));
6624                opts.max_attempts = 1;
6625                opts.readiness.max_total_wait = Duration::from_millis(0);
6626
6627                let mut reporter = CollectingReporter::default();
6628                let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
6629                assert!(format!("{err:#}").contains("failed"));
6630
6631                let st = state::load_state(&td.path().join(".shipper"))
6632                    .expect("load")
6633                    .expect("exists");
6634                let pkg = st.packages.get("demo@0.1.0").expect("pkg");
6635                assert!(matches!(pkg.state, PackageState::Failed { .. }));
6636                server.join();
6637            },
6638        );
6639    }
6640
6641    // 5. Multiple packages: first succeeds, second fails → partial progress saved
6642    #[test]
6643    #[serial]
6644    fn sm_multi_package_partial_progress() {
6645        let td = tempdir().expect("tempdir");
6646        let bin = td.path().join("bin");
6647        write_fake_tools(&bin);
6648        with_test_env(
6649            &bin,
6650            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
6651            || {
6652                // alpha: 404 then 200 (publishes ok)
6653                // beta: 404 then always 404 (readiness never passes)
6654                let server = spawn_registry_server(
6655                    std::collections::BTreeMap::from([
6656                        (
6657                            "/api/v1/crates/alpha/1.0.0".to_string(),
6658                            vec![(404, "{}".to_string()), (200, "{}".to_string())],
6659                        ),
6660                        (
6661                            "/api/v1/crates/beta/2.0.0".to_string(),
6662                            vec![
6663                                (404, "{}".to_string()), // initial
6664                                (404, "{}".to_string()), // readiness
6665                                (404, "{}".to_string()), // final
6666                            ],
6667                        ),
6668                    ]),
6669                    5,
6670                );
6671                let ws = multi_package_workspace(
6672                    td.path(),
6673                    server.base_url.clone(),
6674                    vec![("alpha", "1.0.0"), ("beta", "2.0.0")],
6675                );
6676                let mut opts = default_opts(PathBuf::from(".shipper"));
6677                opts.max_attempts = 1;
6678                opts.readiness.max_total_wait = Duration::from_millis(0);
6679
6680                let mut reporter = CollectingReporter::default();
6681                let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
6682                assert!(format!("{err:#}").contains("beta"));
6683
6684                // Verify partial progress saved: alpha is Published
6685                let st = state::load_state(&td.path().join(".shipper"))
6686                    .expect("load")
6687                    .expect("exists");
6688                let alpha = st.packages.get("alpha@1.0.0").expect("alpha");
6689                assert!(
6690                    matches!(alpha.state, PackageState::Published),
6691                    "alpha should be Published, got {:?}",
6692                    alpha.state
6693                );
6694                let beta = st.packages.get("beta@2.0.0").expect("beta");
6695                assert!(
6696                    matches!(beta.state, PackageState::Failed { .. }),
6697                    "beta should be Failed, got {:?}",
6698                    beta.state
6699                );
6700                server.join();
6701            },
6702        );
6703    }
6704
6705    // 6. Resume from partial state — only remaining packages are attempted
6706    #[test]
6707    #[serial]
6708    fn sm_resume_from_partial_state() {
6709        let td = tempdir().expect("tempdir");
6710        let bin = td.path().join("bin");
6711        write_fake_tools(&bin);
6712        let args_log = td.path().join("cargo_args.txt");
6713        with_test_env(
6714            &bin,
6715            vec![
6716                ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
6717                (
6718                    "SHIPPER_CARGO_ARGS_LOG",
6719                    Some(args_log.to_str().expect("utf8").to_string()),
6720                ),
6721            ],
6722            || {
6723                // beta: 404 then 200 (publishes ok)
6724                let server = spawn_registry_server(
6725                    std::collections::BTreeMap::from([(
6726                        "/api/v1/crates/beta/2.0.0".to_string(),
6727                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
6728                    )]),
6729                    2,
6730                );
6731                let ws = multi_package_workspace(
6732                    td.path(),
6733                    server.base_url.clone(),
6734                    vec![("alpha", "1.0.0"), ("beta", "2.0.0")],
6735                );
6736                let state_dir = td.path().join(".shipper");
6737
6738                // Pre-create state: alpha already Published
6739                let mut packages = std::collections::BTreeMap::new();
6740                packages.insert(
6741                    "alpha@1.0.0".to_string(),
6742                    PackageProgress {
6743                        name: "alpha".to_string(),
6744                        version: "1.0.0".to_string(),
6745                        attempts: 1,
6746                        state: PackageState::Published,
6747                        last_updated_at: Utc::now(),
6748                    },
6749                );
6750                packages.insert(
6751                    "beta@2.0.0".to_string(),
6752                    PackageProgress {
6753                        name: "beta".to_string(),
6754                        version: "2.0.0".to_string(),
6755                        attempts: 0,
6756                        state: PackageState::Pending,
6757                        last_updated_at: Utc::now(),
6758                    },
6759                );
6760                let st = ExecutionState {
6761                    state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
6762                    plan_id: ws.plan.plan_id.clone(),
6763                    registry: ws.plan.registry.clone(),
6764                    created_at: Utc::now(),
6765                    updated_at: Utc::now(),
6766                    attempt_history: Vec::new(),
6767                    packages,
6768                };
6769                state::save_state(&state_dir, &st).expect("save");
6770
6771                let opts = default_opts(PathBuf::from(".shipper"));
6772                let mut reporter = CollectingReporter::default();
6773                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
6774
6775                // alpha should be skipped (already complete)
6776                assert!(
6777                    reporter
6778                        .infos
6779                        .iter()
6780                        .any(|i| i.contains("alpha") && i.contains("already complete"))
6781                );
6782
6783                // beta should be published
6784                assert_eq!(receipt.packages.len(), 1);
6785                assert_eq!(receipt.packages[0].name, "beta");
6786                assert!(matches!(receipt.packages[0].state, PackageState::Published));
6787
6788                // cargo publish should only have been called for beta
6789                let log = fs::read_to_string(&args_log).unwrap_or_default();
6790                assert!(
6791                    !log.contains("alpha"),
6792                    "alpha should not have been published"
6793                );
6794                assert!(log.contains("beta"), "beta should have been published");
6795
6796                server.join();
6797            },
6798        );
6799    }
6800
6801    // 7. Plan ID mismatch on resume — verify rejection
6802    #[test]
6803    fn sm_plan_id_mismatch_rejected() {
6804        let td = tempdir().expect("tempdir");
6805        let ws = multi_package_workspace(
6806            td.path(),
6807            "http://127.0.0.1:9".to_string(),
6808            vec![("demo", "0.1.0")],
6809        );
6810        let state_dir = td.path().join(".shipper");
6811
6812        let mut packages = std::collections::BTreeMap::new();
6813        packages.insert(
6814            "demo@0.1.0".to_string(),
6815            PackageProgress {
6816                name: "demo".to_string(),
6817                version: "0.1.0".to_string(),
6818                attempts: 0,
6819                state: PackageState::Pending,
6820                last_updated_at: Utc::now(),
6821            },
6822        );
6823        let st = ExecutionState {
6824            state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
6825            plan_id: "completely-different-plan".to_string(),
6826            registry: ws.plan.registry.clone(),
6827            created_at: Utc::now(),
6828            updated_at: Utc::now(),
6829            attempt_history: Vec::new(),
6830            packages,
6831        };
6832        state::save_state(&state_dir, &st).expect("save");
6833
6834        let opts = default_opts(PathBuf::from(".shipper"));
6835        let mut reporter = CollectingReporter::default();
6836        let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
6837        let msg = format!("{err:#}");
6838        assert!(msg.contains("does not match current plan_id"), "got: {msg}");
6839    }
6840
6841    // 8. Empty package list — verify graceful handling
6842    #[test]
6843    fn sm_empty_package_list_graceful() {
6844        let td = tempdir().expect("tempdir");
6845        let ws = multi_package_workspace(
6846            td.path(),
6847            "http://127.0.0.1:9".to_string(),
6848            vec![], // no packages
6849        );
6850        let opts = default_opts(PathBuf::from(".shipper"));
6851
6852        let mut reporter = CollectingReporter::default();
6853        let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
6854        assert!(receipt.packages.is_empty());
6855    }
6856
6857    // 9. Dry-run mode: cargo publishes but readiness check prevents registering
6858    //    We test that when no_verify is set, the publish still proceeds normally.
6859    //    (The engine doesn't have an explicit dry-run mode separate from no_verify;
6860    //    "dry-run" is a preflight concept. We test that no_verify flag is respected.)
6861    #[test]
6862    #[serial]
6863    fn sm_no_verify_flag_respected() {
6864        let td = tempdir().expect("tempdir");
6865        let bin = td.path().join("bin");
6866        write_fake_tools(&bin);
6867        let args_log = td.path().join("cargo_args.txt");
6868        with_test_env(
6869            &bin,
6870            vec![
6871                ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
6872                (
6873                    "SHIPPER_CARGO_ARGS_LOG",
6874                    Some(args_log.to_str().expect("utf8").to_string()),
6875                ),
6876            ],
6877            || {
6878                let server = spawn_registry_server(
6879                    std::collections::BTreeMap::from([(
6880                        "/api/v1/crates/demo/0.1.0".to_string(),
6881                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
6882                    )]),
6883                    2,
6884                );
6885                let ws = planned_workspace(td.path(), server.base_url.clone());
6886                let mut opts = default_opts(PathBuf::from(".shipper"));
6887                opts.no_verify = true;
6888
6889                let mut reporter = CollectingReporter::default();
6890                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
6891                assert!(matches!(receipt.packages[0].state, PackageState::Published));
6892
6893                // Verify cargo was called with the right flags
6894                let log = fs::read_to_string(&args_log).unwrap_or_default();
6895                assert!(
6896                    log.contains("publish"),
6897                    "cargo publish should have been called"
6898                );
6899                server.join();
6900            },
6901        );
6902    }
6903
6904    // 10. Max retries exceeded — verify proper error with attempt count
6905    #[test]
6906    #[serial]
6907    fn sm_max_retries_exceeded_attempt_count() {
6908        let td = tempdir().expect("tempdir");
6909        let bin = td.path().join("bin");
6910        write_fake_tools(&bin);
6911        with_test_env(
6912            &bin,
6913            vec![
6914                ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
6915                (
6916                    "SHIPPER_CARGO_STDERR",
6917                    Some("timeout talking to server".to_string()),
6918                ),
6919            ],
6920            || {
6921                let server = spawn_registry_server(
6922                    std::collections::BTreeMap::from([(
6923                        "/api/v1/crates/demo/0.1.0".to_string(),
6924                        vec![
6925                            (404, "{}".to_string()),
6926                            (404, "{}".to_string()),
6927                            (404, "{}".to_string()),
6928                            (404, "{}".to_string()),
6929                            (404, "{}".to_string()),
6930                            (404, "{}".to_string()),
6931                            (404, "{}".to_string()),
6932                        ],
6933                    )]),
6934                    7,
6935                );
6936                let ws = planned_workspace(td.path(), server.base_url.clone());
6937                let mut opts = default_opts(PathBuf::from(".shipper"));
6938                opts.max_attempts = 3;
6939                opts.base_delay = Duration::from_millis(0);
6940                opts.max_delay = Duration::from_millis(0);
6941
6942                let mut reporter = CollectingReporter::default();
6943                let err = run_publish(&ws, &opts, &mut reporter).expect_err("must fail");
6944                assert!(format!("{err:#}").contains("failed"));
6945
6946                let st = state::load_state(&td.path().join(".shipper"))
6947                    .expect("load")
6948                    .expect("exists");
6949                let pkg = st.packages.get("demo@0.1.0").expect("pkg");
6950                assert_eq!(pkg.attempts, 3, "should have made exactly 3 attempts");
6951                assert!(matches!(pkg.state, PackageState::Failed { .. }));
6952
6953                // Verify reporter logged all retry attempts
6954                let attempt_msgs: Vec<_> = reporter
6955                    .infos
6956                    .iter()
6957                    .filter(|i| i.contains("attempt"))
6958                    .collect();
6959                assert!(
6960                    attempt_msgs.len() >= 3,
6961                    "expected at least 3 attempt messages, got {}: {:?}",
6962                    attempt_msgs.len(),
6963                    attempt_msgs
6964                );
6965                server.join();
6966            },
6967        );
6968    }
6969
6970    // 11. Timeout during publish — verify state preservation
6971    #[test]
6972    #[serial]
6973    fn sm_timeout_preserves_state() {
6974        let td = tempdir().expect("tempdir");
6975        let bin = td.path().join("bin");
6976        write_fake_tools(&bin);
6977        with_test_env(
6978            &bin,
6979            vec![
6980                ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
6981                (
6982                    "SHIPPER_CARGO_STDERR",
6983                    Some("timeout while uploading".to_string()),
6984                ),
6985            ],
6986            || {
6987                // cargo fails with timeout, but registry shows the version exists
6988                let server = spawn_registry_server(
6989                    std::collections::BTreeMap::from([(
6990                        "/api/v1/crates/demo/0.1.0".to_string(),
6991                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
6992                    )]),
6993                    2,
6994                );
6995                let ws = planned_workspace(td.path(), server.base_url.clone());
6996                let mut opts = default_opts(PathBuf::from(".shipper"));
6997                opts.base_delay = Duration::from_millis(0);
6998                opts.max_delay = Duration::from_millis(0);
6999
7000                let mut reporter = CollectingReporter::default();
7001                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
7002
7003                // Even though cargo reported failure, registry check found the version
7004                assert!(
7005                    matches!(receipt.packages[0].state, PackageState::Published),
7006                    "expected Published after timeout recovery, got {:?}",
7007                    receipt.packages[0].state
7008                );
7009
7010                // State file on disk should also reflect Published
7011                let st = state::load_state(&td.path().join(".shipper"))
7012                    .expect("load")
7013                    .expect("exists");
7014                let pkg = st.packages.get("demo@0.1.0").expect("pkg");
7015                assert!(matches!(pkg.state, PackageState::Published));
7016                server.join();
7017            },
7018        );
7019    }
7020
7021    // 12. Concurrent package independence — parallel packages don't affect each other
7022    //     (sequential mode: verify each package state is independent)
7023    #[test]
7024    #[serial]
7025    fn sm_package_independence_sequential() {
7026        let td = tempdir().expect("tempdir");
7027        let bin = td.path().join("bin");
7028        write_fake_tools(&bin);
7029        with_test_env(
7030            &bin,
7031            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
7032            || {
7033                let server = spawn_registry_server(
7034                    std::collections::BTreeMap::from([
7035                        (
7036                            "/api/v1/crates/alpha/1.0.0".to_string(),
7037                            vec![(200, "{}".to_string())], // already published
7038                        ),
7039                        (
7040                            "/api/v1/crates/beta/2.0.0".to_string(),
7041                            vec![(404, "{}".to_string()), (200, "{}".to_string())],
7042                        ),
7043                        (
7044                            "/api/v1/crates/gamma/3.0.0".to_string(),
7045                            vec![(200, "{}".to_string())], // already published
7046                        ),
7047                    ]),
7048                    4,
7049                );
7050                let ws = multi_package_workspace(
7051                    td.path(),
7052                    server.base_url.clone(),
7053                    vec![("alpha", "1.0.0"), ("beta", "2.0.0"), ("gamma", "3.0.0")],
7054                );
7055                let opts = default_opts(PathBuf::from(".shipper"));
7056
7057                let mut reporter = CollectingReporter::default();
7058                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
7059
7060                assert_eq!(receipt.packages.len(), 3);
7061                // alpha: skipped (already published)
7062                assert!(
7063                    matches!(receipt.packages[0].state, PackageState::Skipped { .. }),
7064                    "alpha should be Skipped, got {:?}",
7065                    receipt.packages[0].state
7066                );
7067                // beta: published
7068                assert!(
7069                    matches!(receipt.packages[1].state, PackageState::Published),
7070                    "beta should be Published, got {:?}",
7071                    receipt.packages[1].state
7072                );
7073                // gamma: skipped (already published)
7074                assert!(
7075                    matches!(receipt.packages[2].state, PackageState::Skipped { .. }),
7076                    "gamma should be Skipped, got {:?}",
7077                    receipt.packages[2].state
7078                );
7079                server.join();
7080            },
7081        );
7082    }
7083
7084    // 13. Resume from Uploaded state skips cargo publish, goes straight to verify
7085    #[test]
7086    #[serial]
7087    fn sm_resume_from_uploaded_skips_cargo() {
7088        let td = tempdir().expect("tempdir");
7089        let bin = td.path().join("bin");
7090        write_fake_tools(&bin);
7091        let args_log = td.path().join("cargo_args.txt");
7092        with_test_env(
7093            &bin,
7094            vec![
7095                ("SHIPPER_CARGO_EXIT", Some("0".to_string())),
7096                (
7097                    "SHIPPER_CARGO_ARGS_LOG",
7098                    Some(args_log.to_str().expect("utf8").to_string()),
7099                ),
7100            ],
7101            || {
7102                let server = spawn_registry_server(
7103                    std::collections::BTreeMap::from([(
7104                        "/api/v1/crates/demo/0.1.0".to_string(),
7105                        vec![(404, "{}".to_string()), (200, "{}".to_string())],
7106                    )]),
7107                    2,
7108                );
7109                let ws = planned_workspace(td.path(), server.base_url.clone());
7110                let state_dir = td.path().join(".shipper");
7111
7112                // Pre-create state with Uploaded
7113                let mut packages = std::collections::BTreeMap::new();
7114                packages.insert(
7115                    "demo@0.1.0".to_string(),
7116                    PackageProgress {
7117                        name: "demo".to_string(),
7118                        version: "0.1.0".to_string(),
7119                        attempts: 1,
7120                        state: PackageState::Uploaded,
7121                        last_updated_at: Utc::now(),
7122                    },
7123                );
7124                let st = ExecutionState {
7125                    state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
7126                    plan_id: ws.plan.plan_id.clone(),
7127                    registry: ws.plan.registry.clone(),
7128                    created_at: Utc::now(),
7129                    updated_at: Utc::now(),
7130                    attempt_history: Vec::new(),
7131                    packages,
7132                };
7133                state::save_state(&state_dir, &st).expect("save");
7134
7135                let opts = default_opts(PathBuf::from(".shipper"));
7136                let mut reporter = CollectingReporter::default();
7137                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
7138
7139                assert!(matches!(receipt.packages[0].state, PackageState::Published));
7140                assert!(
7141                    reporter
7142                        .infos
7143                        .iter()
7144                        .any(|i| i.contains("resuming from uploaded"))
7145                );
7146
7147                // cargo publish should NOT have been called
7148                let cargo_called = args_log.exists()
7149                    && fs::read_to_string(&args_log)
7150                        .unwrap_or_default()
7151                        .contains("publish");
7152                assert!(
7153                    !cargo_called,
7154                    "cargo publish should not run on resume from Uploaded"
7155                );
7156                server.join();
7157            },
7158        );
7159    }
7160
7161    // 14. Failed package produces correct event log entries
7162    #[test]
7163    #[serial]
7164    fn sm_failed_package_event_log() {
7165        let td = tempdir().expect("tempdir");
7166        let bin = td.path().join("bin");
7167        write_fake_tools(&bin);
7168        with_test_env(
7169            &bin,
7170            vec![
7171                ("SHIPPER_CARGO_EXIT", Some("1".to_string())),
7172                (
7173                    "SHIPPER_CARGO_STDERR",
7174                    Some("permission denied".to_string()),
7175                ),
7176            ],
7177            || {
7178                let server = spawn_registry_server(
7179                    std::collections::BTreeMap::from([(
7180                        "/api/v1/crates/demo/0.1.0".to_string(),
7181                        vec![(404, "{}".to_string()), (404, "{}".to_string())],
7182                    )]),
7183                    2,
7184                );
7185                let ws = planned_workspace(td.path(), server.base_url.clone());
7186                let opts = default_opts(PathBuf::from(".shipper"));
7187
7188                let mut reporter = CollectingReporter::default();
7189                let _ = run_publish(&ws, &opts, &mut reporter);
7190
7191                let events_path = td.path().join(".shipper").join("events.jsonl");
7192                let log = crate::state::events::EventLog::read_from_file(&events_path)
7193                    .expect("read events");
7194                let events = log.all_events();
7195
7196                // Must have: ExecutionStarted, PlanCreated, PackageStarted, PackageAttempted, PackageFailed
7197                assert!(
7198                    events
7199                        .iter()
7200                        .any(|e| matches!(e.event_type, EventType::ExecutionStarted))
7201                );
7202                assert!(
7203                    events
7204                        .iter()
7205                        .any(|e| matches!(e.event_type, EventType::PlanCreated { .. }))
7206                );
7207                assert!(
7208                    events
7209                        .iter()
7210                        .any(|e| matches!(e.event_type, EventType::PackageStarted { .. }))
7211                );
7212                assert!(
7213                    events
7214                        .iter()
7215                        .any(|e| matches!(e.event_type, EventType::PackageAttempted { .. }))
7216                );
7217                assert!(
7218                    events
7219                        .iter()
7220                        .any(|e| matches!(e.event_type, EventType::PackageFailed { .. }))
7221                );
7222                server.join();
7223            },
7224        );
7225    }
7226
7227    // 15. State file persists across transitions with correct version
7228    #[test]
7229    fn sm_state_version_preserved_through_transitions() {
7230        let td = tempdir().expect("tempdir");
7231        let state_dir = td.path().join(".shipper");
7232        let ws = multi_package_workspace(
7233            td.path(),
7234            "http://127.0.0.1:9".to_string(),
7235            vec![("alpha", "1.0.0"), ("beta", "2.0.0")],
7236        );
7237
7238        let mut st = init_state(&ws, &state_dir).expect("init");
7239        assert_eq!(
7240            st.state_version,
7241            crate::state::execution_state::CURRENT_STATE_VERSION
7242        );
7243
7244        // Transition alpha: Pending → Uploaded → Published
7245        update_state(&mut st, &state_dir, "alpha@1.0.0", PackageState::Uploaded).expect("update");
7246        update_state(&mut st, &state_dir, "alpha@1.0.0", PackageState::Published).expect("update");
7247
7248        // Transition beta: Pending → Failed
7249        update_state(
7250            &mut st,
7251            &state_dir,
7252            "beta@2.0.0",
7253            PackageState::Failed {
7254                class: ErrorClass::Permanent,
7255                message: "denied".to_string(),
7256            },
7257        )
7258        .expect("update");
7259
7260        let loaded = state::load_state(&state_dir)
7261            .expect("load")
7262            .expect("exists");
7263        assert_eq!(
7264            loaded.state_version,
7265            crate::state::execution_state::CURRENT_STATE_VERSION
7266        );
7267        assert!(matches!(
7268            loaded.packages.get("alpha@1.0.0").unwrap().state,
7269            PackageState::Published
7270        ));
7271        assert!(matches!(
7272            loaded.packages.get("beta@2.0.0").unwrap().state,
7273            PackageState::Failed { .. }
7274        ));
7275    }
7276
7277    // 16. Snapshot: receipt after successful multi-package publish
7278    #[test]
7279    #[serial]
7280    fn sm_snapshot_receipt_multi_package() {
7281        let td = tempdir().expect("tempdir");
7282        let bin = td.path().join("bin");
7283        write_fake_tools(&bin);
7284        with_test_env(
7285            &bin,
7286            vec![("SHIPPER_CARGO_EXIT", Some("0".to_string()))],
7287            || {
7288                let server = spawn_registry_server(
7289                    std::collections::BTreeMap::from([
7290                        (
7291                            "/api/v1/crates/alpha/1.0.0".to_string(),
7292                            vec![(404, "{}".to_string()), (200, "{}".to_string())],
7293                        ),
7294                        (
7295                            "/api/v1/crates/beta/2.0.0".to_string(),
7296                            vec![(200, "{}".to_string())],
7297                        ),
7298                    ]),
7299                    3,
7300                );
7301                let ws = multi_package_workspace(
7302                    td.path(),
7303                    server.base_url.clone(),
7304                    vec![("alpha", "1.0.0"), ("beta", "2.0.0")],
7305                );
7306                let opts = default_opts(PathBuf::from(".shipper"));
7307
7308                let mut reporter = CollectingReporter::default();
7309                let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
7310
7311                // Snapshot the package states (redact timestamps)
7312                let snapshot: Vec<String> = receipt
7313                    .packages
7314                    .iter()
7315                    .map(|p| {
7316                        format!(
7317                            "name={} version={} attempts={} state={}",
7318                            p.name,
7319                            p.version,
7320                            p.attempts,
7321                            short_state(&p.state)
7322                        )
7323                    })
7324                    .collect();
7325                insta::assert_debug_snapshot!("sm_receipt_multi_package", snapshot);
7326                server.join();
7327            },
7328        );
7329    }
7330
7331    // 17. Snapshot: state after partial failure
7332    #[test]
7333    fn sm_snapshot_state_partial_failure() {
7334        let td = tempdir().expect("tempdir");
7335        let state_dir = td.path().join(".shipper");
7336        let ws = multi_package_workspace(
7337            td.path(),
7338            "http://127.0.0.1:9".to_string(),
7339            vec![("alpha", "1.0.0"), ("beta", "2.0.0"), ("gamma", "3.0.0")],
7340        );
7341
7342        let mut st = init_state(&ws, &state_dir).expect("init");
7343
7344        // alpha: Published, beta: Uploaded (in-progress), gamma: still Pending
7345        update_state(&mut st, &state_dir, "alpha@1.0.0", PackageState::Published).expect("update");
7346        update_state(&mut st, &state_dir, "beta@2.0.0", PackageState::Uploaded).expect("update");
7347
7348        let snapshot: Vec<String> = st
7349            .packages
7350            .iter()
7351            .map(|(k, v)| {
7352                format!(
7353                    "key={} attempts={} state={}",
7354                    k,
7355                    v.attempts,
7356                    short_state(&v.state)
7357                )
7358            })
7359            .collect();
7360        insta::assert_debug_snapshot!("sm_state_partial_failure", snapshot);
7361    }
7362
7363    // 18. Force resume with plan mismatch proceeds with warning
7364    #[test]
7365    fn sm_force_resume_with_mismatch() {
7366        let td = tempdir().expect("tempdir");
7367        let ws = multi_package_workspace(
7368            td.path(),
7369            "http://127.0.0.1:9".to_string(),
7370            vec![("demo", "0.1.0")],
7371        );
7372        let state_dir = td.path().join(".shipper");
7373
7374        // Create state with different plan_id but all packages Published
7375        let mut packages = std::collections::BTreeMap::new();
7376        packages.insert(
7377            "demo@0.1.0".to_string(),
7378            PackageProgress {
7379                name: "demo".to_string(),
7380                version: "0.1.0".to_string(),
7381                attempts: 1,
7382                state: PackageState::Published,
7383                last_updated_at: Utc::now(),
7384            },
7385        );
7386        let st = ExecutionState {
7387            state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
7388            plan_id: "old-plan-id".to_string(),
7389            registry: ws.plan.registry.clone(),
7390            created_at: Utc::now(),
7391            updated_at: Utc::now(),
7392            attempt_history: Vec::new(),
7393            packages,
7394        };
7395        state::save_state(&state_dir, &st).expect("save");
7396
7397        let mut opts = default_opts(PathBuf::from(".shipper"));
7398        opts.force_resume = true;
7399
7400        let mut reporter = CollectingReporter::default();
7401        let receipt = run_publish(&ws, &opts, &mut reporter).expect("publish");
7402        assert!(receipt.packages.is_empty()); // already complete
7403        assert!(
7404            reporter
7405                .warns
7406                .iter()
7407                .any(|w| w.contains("forcing resume with mismatched plan_id"))
7408        );
7409    }
7410
7411    // ── Snapshot tests with insta for engine state ─────────────────────
7412
7413    #[test]
7414    fn snapshot_init_state_single_package() {
7415        let td = tempdir().expect("tempdir");
7416        let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
7417        let state_dir = td.path().join(".shipper");
7418
7419        let st = init_state(&ws, &state_dir).expect("init");
7420
7421        // Redact timestamps for deterministic snapshots
7422        let snapshot: Vec<String> = st
7423            .packages
7424            .iter()
7425            .map(|(k, v)| {
7426                format!(
7427                    "key={} name={} version={} attempts={} state={}",
7428                    k,
7429                    v.name,
7430                    v.version,
7431                    v.attempts,
7432                    short_state(&v.state)
7433                )
7434            })
7435            .collect();
7436        insta::assert_debug_snapshot!("init_state_single_package", snapshot);
7437    }
7438
7439    #[test]
7440    fn snapshot_init_state_multi_package() {
7441        let td = tempdir().expect("tempdir");
7442        let mut ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
7443        ws.plan.packages = vec![
7444            PlannedPackage {
7445                name: "alpha".to_string(),
7446                version: "1.0.0".to_string(),
7447                manifest_path: td.path().join("alpha/Cargo.toml"),
7448                regime: None,
7449            },
7450            PlannedPackage {
7451                name: "beta".to_string(),
7452                version: "2.0.0".to_string(),
7453                manifest_path: td.path().join("beta/Cargo.toml"),
7454                regime: None,
7455            },
7456            PlannedPackage {
7457                name: "gamma".to_string(),
7458                version: "0.3.0".to_string(),
7459                manifest_path: td.path().join("gamma/Cargo.toml"),
7460                regime: None,
7461            },
7462        ];
7463        let state_dir = td.path().join(".shipper");
7464
7465        let st = init_state(&ws, &state_dir).expect("init");
7466
7467        let snapshot: Vec<String> = st
7468            .packages
7469            .iter()
7470            .map(|(k, v)| {
7471                format!(
7472                    "key={} name={} version={} attempts={} state={}",
7473                    k,
7474                    v.name,
7475                    v.version,
7476                    v.attempts,
7477                    short_state(&v.state)
7478                )
7479            })
7480            .collect();
7481        insta::assert_debug_snapshot!("init_state_multi_package", snapshot);
7482    }
7483
7484    #[test]
7485    fn snapshot_state_after_transitions() {
7486        let td = tempdir().expect("tempdir");
7487        let mut ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
7488        ws.plan.packages = vec![
7489            PlannedPackage {
7490                name: "alpha".to_string(),
7491                version: "1.0.0".to_string(),
7492                manifest_path: td.path().join("alpha/Cargo.toml"),
7493                regime: None,
7494            },
7495            PlannedPackage {
7496                name: "beta".to_string(),
7497                version: "2.0.0".to_string(),
7498                manifest_path: td.path().join("beta/Cargo.toml"),
7499                regime: None,
7500            },
7501        ];
7502        let state_dir = td.path().join(".shipper");
7503        let mut st = init_state(&ws, &state_dir).expect("init");
7504
7505        // Simulate: alpha published, beta failed
7506        update_state(&mut st, &state_dir, "alpha@1.0.0", PackageState::Published).expect("update");
7507        update_state(
7508            &mut st,
7509            &state_dir,
7510            "beta@2.0.0",
7511            PackageState::Failed {
7512                class: ErrorClass::Permanent,
7513                message: "auth failure".to_string(),
7514            },
7515        )
7516        .expect("update");
7517
7518        let snapshot: Vec<String> = st
7519            .packages
7520            .iter()
7521            .map(|(k, v)| {
7522                format!(
7523                    "key={} attempts={} state={}",
7524                    k,
7525                    v.attempts,
7526                    short_state(&v.state)
7527                )
7528            })
7529            .collect();
7530        insta::assert_debug_snapshot!("state_after_mixed_transitions", snapshot);
7531    }
7532
7533    #[test]
7534    fn snapshot_error_class_classification_matrix() {
7535        let cases = vec![
7536            ("HTTP 429 too many requests", ""),
7537            ("timeout talking to server", ""),
7538            ("HTTP 503 service unavailable", ""),
7539            ("connection refused", ""),
7540            ("HTTP 500 internal server error", ""),
7541            ("permission denied", ""),
7542            ("crate version `0.1.0` is already uploaded", ""),
7543            ("something totally unexpected", ""),
7544            ("", ""),
7545        ];
7546
7547        let snapshot: Vec<String> = cases
7548            .iter()
7549            .map(|(stderr, stdout)| {
7550                let (class, msg) = classify_cargo_failure(stderr, stdout);
7551                format!(
7552                    "stderr={:50} class={:10} msg={}",
7553                    format!("{stderr:?}"),
7554                    format!("{class:?}"),
7555                    msg
7556                )
7557            })
7558            .collect();
7559        insta::assert_debug_snapshot!("error_classification_matrix", snapshot);
7560    }
7561
7562    // ── Proptest: state transition invariants ──────────────────────────
7563
7564    mod engine_proptests {
7565        use super::*;
7566        use proptest::prelude::*;
7567
7568        fn arb_error_class() -> impl Strategy<Value = ErrorClass> {
7569            prop_oneof![
7570                Just(ErrorClass::Retryable),
7571                Just(ErrorClass::Permanent),
7572                Just(ErrorClass::Ambiguous),
7573            ]
7574        }
7575
7576        fn arb_package_state() -> impl Strategy<Value = PackageState> {
7577            prop_oneof![
7578                Just(PackageState::Pending),
7579                Just(PackageState::Uploaded),
7580                Just(PackageState::Published),
7581                ".*".prop_map(|r| PackageState::Skipped { reason: r }),
7582                (arb_error_class(), ".*").prop_map(|(c, m)| PackageState::Failed {
7583                    class: c,
7584                    message: m
7585                }),
7586                ".*".prop_map(|m| PackageState::Ambiguous { message: m }),
7587            ]
7588        }
7589
7590        proptest! {
7591            /// update_state always persists new_state to disk
7592            #[test]
7593            fn update_state_always_persists(new_state in arb_package_state()) {
7594                let td = tempdir().expect("tempdir");
7595                let state_dir = td.path().join(".shipper");
7596                let ws = planned_workspace(td.path(), "http://127.0.0.1:9".to_string());
7597                let mut st = init_state(&ws, &state_dir).expect("init");
7598                let key = "demo@0.1.0";
7599
7600                update_state(&mut st, &state_dir, key, new_state.clone()).expect("update");
7601
7602                // In-memory matches
7603                assert_eq!(st.packages.get(key).unwrap().state, new_state);
7604
7605                // On-disk matches
7606                let loaded = state::load_state(&state_dir)
7607                    .expect("load")
7608                    .expect("exists");
7609                assert_eq!(loaded.packages.get(key).unwrap().state, new_state);
7610            }
7611
7612            /// short_state never panics on any PackageState variant
7613            #[test]
7614            fn short_state_never_panics(state in arb_package_state()) {
7615                let label = short_state(&state);
7616                assert!(!label.is_empty());
7617            }
7618
7619            /// pkg_key is deterministic and reversible
7620            #[test]
7621            fn pkg_key_deterministic(
7622                name in "[a-z][a-z0-9_-]{0,19}",
7623                version in "[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}"
7624            ) {
7625                let key1 = pkg_key(&name, &version);
7626                let key2 = pkg_key(&name, &version);
7627                assert_eq!(key1, key2);
7628                assert!(key1.contains('@'));
7629                assert!(key1.starts_with(&name));
7630                assert!(key1.ends_with(&version));
7631            }
7632
7633            /// backoff_delay is always bounded by [0, max + jitter headroom]
7634            #[test]
7635            fn backoff_delay_bounded(
7636                base_ms in 1u64..5000,
7637                max_ms in 100u64..30000,
7638                attempt in 1u32..50,
7639                jitter in 0.0f64..1.0,
7640            ) {
7641                let base = Duration::from_millis(base_ms.min(max_ms));
7642                let max = Duration::from_millis(max_ms);
7643
7644                let delay = backoff_delay(
7645                    base,
7646                    max,
7647                    attempt,
7648                    crate::retry::RetryStrategyType::Exponential,
7649                    jitter,
7650                );
7651
7652                // With jitter, delay can be at most max * (1 + jitter)
7653                let upper_bound_ms = (max_ms as f64 * (1.0 + jitter)).ceil() as u64 + 1;
7654                assert!(
7655                    delay.as_millis() <= upper_bound_ms as u128,
7656                    "delay {}ms exceeded upper bound {}ms (base={}ms, max={}ms, attempt={}, jitter={})",
7657                    delay.as_millis(), upper_bound_ms, base_ms, max_ms, attempt, jitter
7658                );
7659            }
7660
7661            /// ExecutionState roundtrips through JSON for arbitrary states
7662            #[test]
7663            fn execution_state_roundtrip(
7664                attempts in 0u32..100,
7665                state in arb_package_state()
7666            ) {
7667                let mut packages = BTreeMap::new();
7668                packages.insert(
7669                    "test@1.0.0".to_string(),
7670                    PackageProgress {
7671                        name: "test".to_string(),
7672                        version: "1.0.0".to_string(),
7673                        attempts,
7674                        state,
7675                        last_updated_at: Utc::now(),
7676                    },
7677                );
7678                let st = ExecutionState {
7679                    state_version: crate::state::execution_state::CURRENT_STATE_VERSION.to_string(),
7680                    plan_id: "plan-proptest".to_string(),
7681                    registry: Registry {
7682                        name: "crates-io".to_string(),
7683                        api_base: "https://crates.io".to_string(),
7684                        index_base: None,
7685                    },
7686                    created_at: Utc::now(),
7687                    updated_at: Utc::now(),
7688                    attempt_history: Vec::new(),
7689                    packages,
7690                };
7691
7692                let json = serde_json::to_string(&st).expect("serialize");
7693                let parsed: ExecutionState = serde_json::from_str(&json).expect("deserialize");
7694                assert_eq!(parsed.packages.len(), 1);
7695                let pkg = parsed.packages.get("test@1.0.0").unwrap();
7696                assert_eq!(pkg.attempts, attempts);
7697            }
7698        }
7699    }
7700
7701    // ───────────────────────────────────────────────────────────────────
7702    // run_rehearsal (#97 PR 2) — phase-2 preflight against an alt registry
7703    // ───────────────────────────────────────────────────────────────────
7704
7705    fn read_events_raw(state_dir: &Path) -> Vec<serde_json::Value> {
7706        let path = events::events_path(state_dir);
7707        let raw = std::fs::read_to_string(&path).unwrap_or_default();
7708        raw.lines()
7709            .filter(|l| !l.trim().is_empty())
7710            .map(|l| serde_json::from_str(l).expect("events.jsonl must parse"))
7711            .collect()
7712    }
7713
7714    fn event_discriminator(event: &serde_json::Value) -> Option<String> {
7715        event
7716            .get("event_type")
7717            .and_then(|et| et.get("type"))
7718            .and_then(|t| t.as_str())
7719            .map(str::to_owned)
7720    }
7721
7722    #[test]
7723    #[serial]
7724    fn run_rehearsal_errors_when_no_rehearsal_registry_configured() {
7725        let td = tempdir().expect("tempdir");
7726        let bin = td.path().join("bin");
7727        write_fake_tools(&bin);
7728        let env_vars = fake_program_env_vars(&bin);
7729        temp_env::with_vars(env_vars, || {
7730            let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
7731            let opts = default_opts(PathBuf::from(".shipper"));
7732
7733            let mut reporter = CollectingReporter::default();
7734            let err = run_rehearsal(&ws, &opts, &mut reporter).expect_err("must fail");
7735            let msg = format!("{err:#}");
7736            assert!(msg.contains("no rehearsal registry"), "err was: {msg}");
7737        });
7738    }
7739
7740    #[test]
7741    #[serial]
7742    fn run_rehearsal_errors_when_rehearsal_equals_live_target() {
7743        let td = tempdir().expect("tempdir");
7744        let bin = td.path().join("bin");
7745        write_fake_tools(&bin);
7746        let env_vars = fake_program_env_vars(&bin);
7747        temp_env::with_vars(env_vars, || {
7748            let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
7749            let mut opts = default_opts(PathBuf::from(".shipper"));
7750            // Point rehearsal at the same registry name as the live target.
7751            opts.rehearsal_registry = Some("crates-io".to_string());
7752            opts.registries = vec![Registry {
7753                name: "crates-io".to_string(),
7754                api_base: "http://127.0.0.1:1".to_string(),
7755                index_base: None,
7756            }];
7757
7758            let mut reporter = CollectingReporter::default();
7759            let err = run_rehearsal(&ws, &opts, &mut reporter).expect_err("must fail");
7760            let msg = format!("{err:#}");
7761            assert!(
7762                msg.contains("must differ from the live target"),
7763                "err was: {msg}"
7764            );
7765        });
7766    }
7767
7768    #[test]
7769    #[serial]
7770    fn run_rehearsal_errors_when_registry_name_not_in_config() {
7771        let td = tempdir().expect("tempdir");
7772        let bin = td.path().join("bin");
7773        write_fake_tools(&bin);
7774        let env_vars = fake_program_env_vars(&bin);
7775        temp_env::with_vars(env_vars, || {
7776            let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
7777            let mut opts = default_opts(PathBuf::from(".shipper"));
7778            opts.rehearsal_registry = Some("bogus-registry".to_string());
7779            // opts.registries is empty — bogus-registry won't resolve.
7780
7781            let mut reporter = CollectingReporter::default();
7782            let err = run_rehearsal(&ws, &opts, &mut reporter).expect_err("must fail");
7783            let msg = format!("{err:#}");
7784            assert!(msg.contains("is not configured"), "err was: {msg}");
7785        });
7786    }
7787
7788    #[test]
7789    #[serial]
7790    fn run_rehearsal_skip_flag_returns_without_running() {
7791        let td = tempdir().expect("tempdir");
7792        let bin = td.path().join("bin");
7793        write_fake_tools(&bin);
7794        let env_vars = fake_program_env_vars(&bin);
7795        temp_env::with_vars(env_vars, || {
7796            let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
7797            let mut opts = default_opts(PathBuf::from(".shipper"));
7798            opts.rehearsal_registry = Some("rehearsal".to_string());
7799            opts.rehearsal_skip = true;
7800
7801            let mut reporter = CollectingReporter::default();
7802            let outcome =
7803                run_rehearsal(&ws, &opts, &mut reporter).expect("skip path should not error");
7804            assert!(!outcome.passed, "skip should not claim a pass");
7805            assert_eq!(outcome.packages_published, 0);
7806            assert!(outcome.summary.contains("skipped"));
7807            // Skip path must not write events — nothing to audit.
7808            let events_path = events::events_path(&td.path().join(".shipper"));
7809            assert!(
7810                !events_path.exists(),
7811                "skip path must not create events.jsonl"
7812            );
7813        });
7814    }
7815
7816    #[test]
7817    #[serial]
7818    fn run_rehearsal_happy_path_emits_started_published_complete_events() {
7819        let td = tempdir().expect("tempdir");
7820        let bin = td.path().join("bin");
7821        write_fake_tools(&bin);
7822        let env_vars = fake_program_env_vars(&bin);
7823        temp_env::with_vars(env_vars, || {
7824            // Rehearsal-registry mock: returns 404 for the preflight lookup
7825            // (not here) and 200 for the post-publish visibility check.
7826            let rehearsal_server = spawn_registry_server(
7827                std::collections::BTreeMap::from([(
7828                    "/api/v1/crates/demo/0.1.0".to_string(),
7829                    vec![(200, "{}".to_string())],
7830                )]),
7831                1,
7832            );
7833
7834            let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
7835            let mut opts = default_opts(PathBuf::from(".shipper"));
7836            opts.rehearsal_registry = Some("rehearsal".to_string());
7837            opts.registries = vec![Registry {
7838                name: "rehearsal".to_string(),
7839                api_base: rehearsal_server.base_url.clone(),
7840                index_base: None,
7841            }];
7842
7843            let mut reporter = CollectingReporter::default();
7844            let outcome = run_rehearsal(&ws, &opts, &mut reporter).expect("rehearse");
7845            assert!(outcome.passed, "outcome: {outcome:?}");
7846            assert_eq!(outcome.packages_published, 1);
7847
7848            let events = read_events_raw(&td.path().join(".shipper"));
7849            let types: Vec<String> = events.iter().filter_map(event_discriminator).collect();
7850            assert!(
7851                types.contains(&"rehearsal_started".to_string()),
7852                "types: {types:?}"
7853            );
7854            assert!(
7855                types.contains(&"rehearsal_package_published".to_string()),
7856                "types: {types:?}"
7857            );
7858            assert!(
7859                types.contains(&"rehearsal_complete".to_string()),
7860                "types: {types:?}"
7861            );
7862
7863            // RehearsalComplete must carry passed=true for the happy path.
7864            let complete = events
7865                .iter()
7866                .find(|e| event_discriminator(e).as_deref() == Some("rehearsal_complete"))
7867                .expect("RehearsalComplete event");
7868            assert_eq!(
7869                complete["event_type"]["passed"].as_bool(),
7870                Some(true),
7871                "complete event: {complete}"
7872            );
7873
7874            rehearsal_server.join();
7875        });
7876    }
7877
7878    // ───────────────────────────────────────────────────────────────────
7879    // enforce_rehearsal_gate (#97 PR 3)
7880    // ───────────────────────────────────────────────────────────────────
7881
7882    fn write_rehearsal_receipt(
7883        state_dir: &Path,
7884        plan_id: &str,
7885        passed: bool,
7886    ) -> crate::state::rehearsal::RehearsalReceipt {
7887        let receipt = crate::state::rehearsal::RehearsalReceipt {
7888            schema_version: crate::state::rehearsal::CURRENT_REHEARSAL_VERSION.to_string(),
7889            plan_id: plan_id.to_string(),
7890            registry: "rehearsal".to_string(),
7891            passed,
7892            packages_attempted: 1,
7893            packages_published: if passed { 1 } else { 0 },
7894            summary: if passed {
7895                "rehearsed 1 package successfully".into()
7896            } else {
7897                "rehearsal failed".into()
7898            },
7899            started_at: Utc::now(),
7900            completed_at: Utc::now(),
7901        };
7902        crate::state::rehearsal::save_rehearsal(state_dir, &receipt).expect("write");
7903        receipt
7904    }
7905
7906    #[test]
7907    fn gate_is_dormant_when_rehearsal_registry_is_none() {
7908        let td = tempdir().expect("tempdir");
7909        let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
7910        let opts = default_opts(PathBuf::from(".shipper"));
7911        // opts.rehearsal_registry is None by default.
7912        let mut reporter = CollectingReporter::default();
7913        enforce_rehearsal_gate(&ws, &opts, td.path(), &mut reporter).expect("gate dormant");
7914    }
7915
7916    #[test]
7917    fn gate_proceeds_with_warning_when_skip_is_set() {
7918        let td = tempdir().expect("tempdir");
7919        let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
7920        let mut opts = default_opts(PathBuf::from(".shipper"));
7921        opts.rehearsal_registry = Some("rehearsal".into());
7922        opts.rehearsal_skip = true;
7923        let mut reporter = CollectingReporter::default();
7924        enforce_rehearsal_gate(&ws, &opts, td.path(), &mut reporter).expect("skip bypass");
7925        assert!(
7926            reporter
7927                .warns
7928                .iter()
7929                .any(|w| w.contains("--skip-rehearsal")),
7930            "warns: {:?}",
7931            reporter.warns
7932        );
7933    }
7934
7935    #[test]
7936    fn gate_refuses_when_no_receipt_exists() {
7937        let td = tempdir().expect("tempdir");
7938        let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
7939        let mut opts = default_opts(PathBuf::from(".shipper"));
7940        opts.rehearsal_registry = Some("rehearsal".into());
7941        let mut reporter = CollectingReporter::default();
7942        let err =
7943            enforce_rehearsal_gate(&ws, &opts, td.path(), &mut reporter).expect_err("must fail");
7944        let msg = format!("{err:#}");
7945        assert!(msg.contains("no rehearsal receipt was found"), "err: {msg}");
7946        assert!(
7947            msg.contains("shipper rehearse"),
7948            "err should hint fix: {msg}"
7949        );
7950    }
7951
7952    #[test]
7953    fn gate_refuses_on_plan_id_mismatch() {
7954        let td = tempdir().expect("tempdir");
7955        let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
7956        let mut opts = default_opts(PathBuf::from(".shipper"));
7957        opts.rehearsal_registry = Some("rehearsal".into());
7958
7959        write_rehearsal_receipt(td.path(), "some-other-plan", true);
7960
7961        let mut reporter = CollectingReporter::default();
7962        let err =
7963            enforce_rehearsal_gate(&ws, &opts, td.path(), &mut reporter).expect_err("must fail");
7964        let msg = format!("{err:#}");
7965        assert!(msg.contains("stale"), "err: {msg}");
7966        assert!(
7967            msg.contains(&ws.plan.plan_id),
7968            "err should reference current plan_id: {msg}"
7969        );
7970    }
7971
7972    #[test]
7973    fn gate_refuses_on_failing_receipt() {
7974        let td = tempdir().expect("tempdir");
7975        let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
7976        let mut opts = default_opts(PathBuf::from(".shipper"));
7977        opts.rehearsal_registry = Some("rehearsal".into());
7978
7979        write_rehearsal_receipt(td.path(), &ws.plan.plan_id, false);
7980
7981        let mut reporter = CollectingReporter::default();
7982        let err =
7983            enforce_rehearsal_gate(&ws, &opts, td.path(), &mut reporter).expect_err("must fail");
7984        let msg = format!("{err:#}");
7985        assert!(msg.contains("did NOT pass"), "err: {msg}");
7986    }
7987
7988    #[test]
7989    fn gate_passes_on_fresh_passing_receipt() {
7990        let td = tempdir().expect("tempdir");
7991        let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
7992        let mut opts = default_opts(PathBuf::from(".shipper"));
7993        opts.rehearsal_registry = Some("rehearsal".into());
7994
7995        write_rehearsal_receipt(td.path(), &ws.plan.plan_id, true);
7996
7997        let mut reporter = CollectingReporter::default();
7998        enforce_rehearsal_gate(&ws, &opts, td.path(), &mut reporter).expect("fresh pass");
7999        assert!(
8000            reporter.infos.iter().any(|i| i.contains("passing receipt")),
8001            "infos: {:?}",
8002            reporter.infos
8003        );
8004    }
8005
8006    /// End-to-end: `run_publish` refuses to run when rehearsal is required
8007    /// but no receipt exists. This is the gate's actual contract — the
8008    /// finer-grained tests above exercise the gate helper in isolation;
8009    /// this one confirms run_publish wires it in correctly.
8010    #[test]
8011    #[serial]
8012    fn run_publish_refuses_without_rehearsal_when_required() {
8013        let td = tempdir().expect("tempdir");
8014        let bin = td.path().join("bin");
8015        write_fake_tools(&bin);
8016        let env_vars = fake_program_env_vars(&bin);
8017        temp_env::with_vars(env_vars, || {
8018            let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
8019            let state_dir = td.path().join(".shipper");
8020            let mut opts = default_opts(state_dir);
8021            opts.rehearsal_registry = Some("rehearsal".into());
8022
8023            let mut reporter = CollectingReporter::default();
8024            let err = run_publish(&ws, &opts, &mut reporter).expect_err("gate must bail");
8025            let msg = format!("{err:#}");
8026            assert!(
8027                msg.contains("rehearsal is required") || msg.contains("no rehearsal receipt"),
8028                "expected gate error, got: {msg}"
8029            );
8030        });
8031    }
8032
8033    /// #97 PR 4 — smoke-install happy path. --smoke-install names a
8034    /// crate in the plan; fake cargo returns 0 for the install call;
8035    /// rehearsal emits smoke-check events and passes.
8036    #[test]
8037    #[serial]
8038    fn run_rehearsal_smoke_install_happy_path_emits_succeeded_event() {
8039        let td = tempdir().expect("tempdir");
8040        let bin = td.path().join("bin");
8041        write_fake_tools(&bin);
8042        let env_vars = fake_program_env_vars(&bin);
8043        temp_env::with_vars(env_vars, || {
8044            let rehearsal_server = spawn_registry_server(
8045                std::collections::BTreeMap::from([(
8046                    "/api/v1/crates/demo/0.1.0".to_string(),
8047                    vec![(200, "{}".to_string())],
8048                )]),
8049                1,
8050            );
8051
8052            let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
8053            let mut opts = default_opts(PathBuf::from(".shipper"));
8054            opts.rehearsal_registry = Some("rehearsal".to_string());
8055            opts.registries = vec![Registry {
8056                name: "rehearsal".to_string(),
8057                api_base: rehearsal_server.base_url.clone(),
8058                index_base: None,
8059            }];
8060            opts.rehearsal_smoke_install = Some("demo".to_string());
8061
8062            let mut reporter = CollectingReporter::default();
8063            let outcome = run_rehearsal(&ws, &opts, &mut reporter).expect("rehearse");
8064            assert!(outcome.passed, "outcome: {outcome:?}");
8065
8066            let events = read_events_raw(&td.path().join(".shipper"));
8067            let types: Vec<String> = events.iter().filter_map(event_discriminator).collect();
8068            assert!(
8069                types.contains(&"rehearsal_smoke_check_started".to_string()),
8070                "types: {types:?}"
8071            );
8072            assert!(
8073                types.contains(&"rehearsal_smoke_check_succeeded".to_string()),
8074                "types: {types:?}"
8075            );
8076            rehearsal_server.join();
8077        });
8078    }
8079
8080    /// #97 PR 4 — smoke-install named a crate not in the plan. Warn-only
8081    /// path: rehearsal itself still passes (publish was fine) but the
8082    /// reporter surfaces the misconfiguration so the operator can fix it.
8083    #[test]
8084    #[serial]
8085    fn run_rehearsal_smoke_install_missing_target_warns_without_failing() {
8086        let td = tempdir().expect("tempdir");
8087        let bin = td.path().join("bin");
8088        write_fake_tools(&bin);
8089        let env_vars = fake_program_env_vars(&bin);
8090        temp_env::with_vars(env_vars, || {
8091            let rehearsal_server = spawn_registry_server(
8092                std::collections::BTreeMap::from([(
8093                    "/api/v1/crates/demo/0.1.0".to_string(),
8094                    vec![(200, "{}".to_string())],
8095                )]),
8096                1,
8097            );
8098
8099            let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
8100            let mut opts = default_opts(PathBuf::from(".shipper"));
8101            opts.rehearsal_registry = Some("rehearsal".to_string());
8102            opts.registries = vec![Registry {
8103                name: "rehearsal".to_string(),
8104                api_base: rehearsal_server.base_url.clone(),
8105                index_base: None,
8106            }];
8107            opts.rehearsal_smoke_install = Some("nonexistent".to_string());
8108
8109            let mut reporter = CollectingReporter::default();
8110            let outcome = run_rehearsal(&ws, &opts, &mut reporter).expect("rehearse");
8111            assert!(outcome.passed);
8112            assert!(
8113                reporter
8114                    .warns
8115                    .iter()
8116                    .any(|w| w.contains("not in the rehearsal plan")),
8117                "warns: {:?}",
8118                reporter.warns
8119            );
8120            rehearsal_server.join();
8121        });
8122    }
8123
8124    #[test]
8125    #[serial]
8126    fn run_rehearsal_cargo_failure_emits_package_failed_and_marks_not_passed() {
8127        let td = tempdir().expect("tempdir");
8128        let bin = td.path().join("bin");
8129        write_fake_tools(&bin);
8130        let mut env_vars = fake_program_env_vars(&bin);
8131        env_vars.extend([("SHIPPER_CARGO_EXIT", Some("101".to_string()))]);
8132        temp_env::with_vars(env_vars, || {
8133            // Rehearsal registry is never hit because cargo fails.
8134            let rehearsal_server = spawn_registry_server(std::collections::BTreeMap::new(), 0);
8135
8136            let ws = planned_workspace(td.path(), "http://127.0.0.1:1".into());
8137            let mut opts = default_opts(PathBuf::from(".shipper"));
8138            opts.rehearsal_registry = Some("rehearsal".to_string());
8139            opts.registries = vec![Registry {
8140                name: "rehearsal".to_string(),
8141                api_base: rehearsal_server.base_url.clone(),
8142                index_base: None,
8143            }];
8144
8145            let mut reporter = CollectingReporter::default();
8146            let outcome = run_rehearsal(&ws, &opts, &mut reporter).expect("rehearse");
8147            assert!(!outcome.passed);
8148            assert_eq!(outcome.packages_published, 0);
8149
8150            let events = read_events_raw(&td.path().join(".shipper"));
8151            let types: Vec<String> = events.iter().filter_map(event_discriminator).collect();
8152            assert!(
8153                types.contains(&"rehearsal_package_failed".to_string()),
8154                "types: {types:?}"
8155            );
8156            assert!(
8157                types.contains(&"rehearsal_complete".to_string()),
8158                "types: {types:?}"
8159            );
8160
8161            let complete = events
8162                .iter()
8163                .find(|e| event_discriminator(e).as_deref() == Some("rehearsal_complete"))
8164                .expect("RehearsalComplete");
8165            assert_eq!(complete["event_type"]["passed"].as_bool(), Some(false));
8166            rehearsal_server.join();
8167        });
8168    }
8169}
8170
8171/// Wave-based parallel publishing engine.
8172pub mod parallel;
8173
8174/// Plan-yank: reverse-topological containment plan from a receipt (#98 PR 2).
8175pub mod plan_yank;
8176
8177/// Fix-forward: supersession plan from a compromised receipt (#98 PR 3).
8178pub mod fix_forward;
8179
8180/// Remediation dry-run artifact planning.
8181pub mod remediation;