hermit-run 0.4.0

Hermit: a deterministic Linux execution engine for reproducible runs, record and replay, and concurrency-race search.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree.
 */

use std::io;
use std::io::IsTerminal;
use std::io::Write;
use std::io::stderr;
use std::mem;
use std::ptr::NonNull;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;

use hermit::Backend;
use hermit::liteinst_bootstrap::EffectiveFilter;
use tracing::Subscriber;
use tracing::metadata::LevelFilter;
use tracing_subscriber::EnvFilter;
use tracing_subscriber::Layer;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;

const DEFAULT_TRACE_LEVEL: LevelFilter = LevelFilter::WARN;

/// Environment override for [`DEFAULT_LOG_MAX_BYTES`]; `0` disables the bound.
pub const LOG_MAX_BYTES_ENV: &str = "HERMIT_LOG_MAX_BYTES";

/// Ceiling on one run's log file.
///
/// A run that makes no progress still logs. Measured 2026-08-11 over the 4111
/// retained `--verify` comparison logs on one host: p50 0.4 MiB, p90 43.5 MiB,
/// p99 145.5 MiB -- and then a tail to 928.8 GiB, 4357.4 GiB in total. The
/// largest single file was a KVM guest livelocked on `sched_yield`, logging the
/// same fifteen line shapes for 11.7 hours; its registers at the 50 GiB and
/// 900 GiB offsets were identical.
///
/// 1 GiB is about 7x p99, so it never truncates a log anyone reads, and it
/// removes 98.6% of those bytes while touching 0.56% of the files. The
/// distribution is bimodal enough that any bound from 0.25 to 16 GiB reclaims
/// over 94%, so this is a safety limit rather than a tuning parameter.
///
/// Compression is NOT the control. `/tmp` happens to be zstd-compressed here,
/// which is why a terabyte of repetitive DETLOG has been survivable, but a
/// compression ratio scales with how repetitive the runaway happens to be and
/// silently converts a hard failure into an invisible one.
pub const DEFAULT_LOG_MAX_BYTES: u64 = 1 << 30;

/// Written once, in-band, when the bound is reached.
///
/// A truncated log MUST say so. Output that simply stops reads as a run that
/// ENDED, which would be a worse evidence defect than the disk hazard this
/// bound removes: a reader would draw conclusions from an absence we created.
///
/// The surrounding newlines are load-bearing, not cosmetic. The marker must
/// occupy a LINE OF ITS OWN at the very END of the file, because that is what
/// [`detcore::logdiff::log_was_truncated`] anchors on to tell a truncated log
/// apart from a log that merely quotes the text. This literal is deliberately
/// NOT `detcore::logdiff::TRUNCATION_MARKER` itself: keeping the two texts
/// independent is what leaves
/// `the_written_marker_is_the_text_the_comparator_matches` able to fail.
const TRUNCATION_MARKER: &[u8] =
    b"\n=== HERMIT LOG TRUNCATED: reached the configured size bound (HERMIT_LOG_MAX_BYTES). \
Output beyond this point was DISCARDED. The run itself continued and was NOT affected. ===\n";

/// Resolve the configured ceiling; `0` means unlimited.
///
/// A malformed value is an ERROR rather than a fallback. Silently substituting
/// the 1 GiB default would mean `HERMIT_LOG_MAX_BYTES=unlimited`, or any typo
/// in the value used to disable the bound, quietly re-enabled it -- i.e. the
/// documented way to turn the bound off is one keystroke away from turning it
/// back on without saying so.
pub fn log_max_bytes() -> Result<u64, String> {
    match std::env::var(LOG_MAX_BYTES_ENV) {
        Ok(raw) => raw.trim().parse().map_err(|_| {
            format!(
                "{LOG_MAX_BYTES_ENV}={raw:?} is not a byte count. Set a non-negative integer \
                 number of bytes, or 0 to disable the bound."
            )
        }),
        Err(_) => Ok(DEFAULT_LOG_MAX_BYTES),
    }
}

/// A writer that stops after `limit` bytes and says so in-band.
///
/// Beyond the bound this reports the bytes as consumed rather than returning a
/// short count: `tracing_appender` treats a short write as an I/O error, so a
/// truthful-but-short return would turn a size limit into a logging failure.
pub struct BoundedWriter<W: Write> {
    inner: W,
    remaining: u64,
    bounded: bool,
    announced: bool,
}

#[derive(Debug)]
struct SharedWriteError {
    address: NonNull<AtomicBool>,
}

// SAFETY: address points to one initialized atomic in a MAP_SHARED mapping.
// AtomicBool supplies synchronization, and the mapping remains live while any
// clone of WriteErrorLatch exists in this process.
unsafe impl Send for SharedWriteError {}
unsafe impl Sync for SharedWriteError {}

impl Drop for SharedWriteError {
    fn drop(&mut self) {
        // SAFETY: this process owns this mapping, created with exactly this
        // address and length in WriteErrorLatch::new. A fork receives its own
        // Arc refcount and unmaps only its own process mapping on drop.
        unsafe {
            libc::munmap(self.address.as_ptr().cast(), mem::size_of::<AtomicBool>());
        }
    }
}

/// A process-shared latch for errors a tracing formatter otherwise discards.
///
/// Ordinary runs initialize tracing after the container fork. A heap atomic
/// would therefore be copy-on-write: the writer child could record an error
/// that the parent publishing the manifest never observes. This anonymous
/// MAP_SHARED cell is created before that fork and is not inherited across the
/// guest exec.
#[derive(Clone, Debug)]
pub struct WriteErrorLatch {
    shared: Arc<SharedWriteError>,
}

impl WriteErrorLatch {
    pub fn new() -> io::Result<Self> {
        // SAFETY: anonymous shared mapping, no fixed address and no backing fd.
        let address = unsafe {
            libc::mmap(
                std::ptr::null_mut(),
                mem::size_of::<AtomicBool>(),
                libc::PROT_READ | libc::PROT_WRITE,
                libc::MAP_SHARED | libc::MAP_ANONYMOUS,
                -1,
                0,
            )
        };
        if address == libc::MAP_FAILED {
            return Err(io::Error::last_os_error());
        }
        let address = NonNull::new(address.cast::<AtomicBool>())
            .expect("mmap returned a non-null non-failure address");
        // SAFETY: the fresh mapping is writable and suitably page-aligned.
        unsafe { address.as_ptr().write(AtomicBool::new(false)) };
        Ok(Self {
            shared: Arc::new(SharedWriteError { address }),
        })
    }

    fn cell(&self) -> &AtomicBool {
        // SAFETY: the Arc keeps the initialized mapping live in this process.
        unsafe { self.shared.address.as_ref() }
    }

    fn record_failure(&self) {
        self.cell().store(true, Ordering::Release);
    }

    pub fn failed(&self) -> bool {
        self.cell().load(Ordering::Acquire)
    }
}

/// Records every write or flush error before returning it to tracing.
///
/// tracing-subscriber intentionally treats formatter I/O as diagnostic-only
/// and discards these errors. Evidence cannot: a valid prefix with a lost tail
/// is incomplete even when that prefix still parses.
pub struct LatchedWriter<W> {
    inner: W,
    latch: WriteErrorLatch,
}

impl<W> LatchedWriter<W> {
    pub fn new(inner: W, latch: WriteErrorLatch) -> Self {
        Self { inner, latch }
    }
}

impl<W: Write> Write for LatchedWriter<W> {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        match self.inner.write(buf) {
            Ok(0) if !buf.is_empty() => {
                self.latch.record_failure();
                Ok(0)
            }
            Ok(written) => Ok(written),
            Err(error) => {
                self.latch.record_failure();
                Err(error)
            }
        }
    }

    fn flush(&mut self) -> io::Result<()> {
        self.inner.flush().inspect_err(|_| {
            self.latch.record_failure();
        })
    }
}

impl<W: Write> BoundedWriter<W> {
    /// `limit == 0` disables the bound entirely.
    pub fn new(inner: W, limit: u64) -> Self {
        Self {
            inner,
            remaining: limit,
            bounded: limit != 0,
            announced: false,
        }
    }
}

impl<W: Write> BoundedWriter<W> {
    /// Announce truncation the FIRST time bytes are actually discarded.
    ///
    /// Deferring this to the next `write` leaves a log that was truncated on
    /// its final write silent, which is the exact failure the marker exists to
    /// prevent -- caught end-to-end: a 100-byte bound produced exactly 100
    /// bytes and no marker, because no further write ever arrived.
    fn announce_truncation(&mut self) -> io::Result<()> {
        if !self.announced {
            self.announced = true;
            self.inner.write_all(TRUNCATION_MARKER)?;
            self.inner.flush()?;
        }
        Ok(())
    }
}

impl<W: Write> Write for BoundedWriter<W> {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        if !self.bounded {
            return self.inner.write(buf);
        }
        if self.remaining == 0 {
            self.announce_truncation()?;
            return Ok(buf.len());
        }
        let take = buf.len().min(self.remaining as usize);
        let written = self.inner.write(&buf[..take])?;
        self.remaining -= written as u64;
        if written < take {
            // A genuine short write by the inner writer: report it truthfully
            // and let the caller retry the remainder.
            return Ok(written);
        }
        if take < buf.len() {
            // This write is the one that crossed the bound; its tail is being
            // dropped, so say so now rather than hoping for another write.
            self.announce_truncation()?;
        }
        Ok(buf.len())
    }

    fn flush(&mut self) -> io::Result<()> {
        self.inner.flush()
    }
}

fn env_filter(level: LevelFilter) -> EnvFilter {
    EffectiveFilter::from_default_env(level).into_filter()
}

/// Keeps a nonblocking public writer alive when one is installed. A private
/// evidence layer is synchronous and therefore needs no drain worker or guard.
pub struct TracingGuard {
    _worker: Option<tracing_appender::non_blocking::WorkerGuard>,
}

impl TracingGuard {
    fn synchronous() -> Self {
        Self { _worker: None }
    }
}

/// Returns a non-blocking subscriber for logging to a file.
///
/// NOTE: Writes to `f` are unbuffered, so this may be slow.
fn file_subscriber<W: Write + Send + 'static>(
    level: LevelFilter,
    f: W,
) -> (impl Subscriber, tracing_appender::non_blocking::WorkerGuard) {
    let filter = env_filter(level);
    let (writer, guard) = tracing_appender::non_blocking(f);

    let subscriber = tracing_subscriber::fmt()
        .with_env_filter(filter)
        .with_writer(writer)
        .with_ansi(false)
        .finish();

    (subscriber, guard)
}

/// A file subscriber whose writes CANNOT be lost when the process dies.
///
/// `file_subscriber` uses `tracing_appender::non_blocking`, which queues to a
/// background thread and only drains when its guard drops. A run that dies
/// without unwinding -- which is exactly what a fail-closed guest does -- never
/// drops the guard, so the QUEUED TAIL IS LOST. The tail is where a fatal
/// diagnostic lives, so the one line that explains the failure is the one line
/// reliably discarded.
///
/// MEASURED 2026-08-25 on c-programs/dbt-unsupported-syscall under `--verify`:
///   non-blocking  18 runs, diagnostic present 0 times, logs truncated at a
///                 RANDOM syscall each run (#5,#6,#10,#11,#16,#23,#27,#31...)
///                 and random sizes (5,482 / 7,632 / 10,123 / 15,546 bytes)
///   synchronous    4 runs, diagnostic present 4 times, every run reaching
///                 syscall #32 and producing an IDENTICAL 16,171-byte log
/// The randomness was the tell: a deterministic guest under a deterministic
/// engine cannot genuinely die at a different syscall each time.
///
/// COST, measured rather than assumed, on a 477 KB verify log:
/// synchronous 1.77s versus non-blocking 1.73s -- inside noise, identical
/// output size. The queue was buying nothing and losing the diagnostic.
fn sync_file_subscriber<W: Write + Send + 'static>(level: LevelFilter, f: W) -> impl Subscriber {
    let filter = env_filter(level);
    tracing_subscriber::fmt()
        .with_env_filter(filter)
        .with_writer(std::sync::Mutex::new(f))
        .with_ansi(false)
        .finish()
}

/// Initializes SYNCHRONOUS tracing to `f`, for logs whose tail must survive a
/// crash. See [`sync_file_subscriber`].
pub fn init_sync_file_tracing<W: Write + Send + 'static>(
    level: Option<LevelFilter>,
    f: W,
    backend: Backend,
) -> TracingGuard {
    // Match the PID/TID slot consumed by file tracing's worker while keeping
    // verification evidence synchronous and the namespace single-threaded.
    align_tracing_pid_baseline(backend);
    let level = level.unwrap_or(DEFAULT_TRACE_LEVEL);
    let subscriber = sync_file_subscriber(level, f);
    subscriber
        .try_init()
        .expect("global tracing subscriber to install");
    TracingGuard::synchronous()
}

/// Initializes tracing to the given file `f`.
///
/// NOTE: Writes to `f` are unbuffered, so this may be slow.
#[must_use = "This function returns a guard that should not be immediately dropped"]
pub fn init_file_tracing<W: Write + Send + 'static>(
    level: Option<LevelFilter>,
    f: W,
) -> TracingGuard {
    let level = level.unwrap_or(DEFAULT_TRACE_LEVEL);

    let (subscriber, guard) = file_subscriber(level, f);

    subscriber
        .try_init()
        .expect("global tracing subscriber to install");

    TracingGuard {
        _worker: Some(guard),
    }
}

/// Preserve the public file logger while synchronously duplicating INFO-and-
/// higher events into a private evidence descriptor. This creates only the
/// public logger's existing nonblocking worker; the evidence writer is direct.
pub fn init_file_tracing_with_evidence<P, E>(
    level: Option<LevelFilter>,
    public: P,
    evidence: E,
) -> TracingGuard
where
    P: Write + Send + 'static,
    E: Write + Send + 'static,
{
    let (public_writer, guard) = tracing_appender::non_blocking(public);
    let public_layer = tracing_subscriber::fmt::layer()
        .with_writer(public_writer)
        .with_ansi(false)
        .with_filter(env_filter(level.unwrap_or(DEFAULT_TRACE_LEVEL)));
    let evidence_layer = tracing_subscriber::fmt::layer()
        .with_writer(std::sync::Mutex::new(evidence))
        .with_ansi(false)
        .with_filter(LevelFilter::INFO);
    tracing_subscriber::registry()
        .with(public_layer)
        .with(evidence_layer)
        .try_init()
        .expect("global tracing subscriber to install");
    TracingGuard {
        _worker: Some(guard),
    }
}

/// Make the next guest task use the same identity as the asynchronous logger.
///
/// `init_file_tracing` starts a nonblocking writer thread inside the fresh PID
/// namespace. The namespace init is PID 1, that worker consumes PID 2, and the
/// root guest consequently starts at [`detcore::ROOT_DETPID`] (PID 3). The
/// synchronous and stderr subscribers have no worker, but their guests still
/// need the same identity baseline.
///
/// Creating and joining a dummy host thread to consume PID 2 is not equivalent:
/// the thread participates in the parent's signal/timer state and made KVM's
/// `setitimer` determinism case die from SIGALRM. A forked process has separate
/// signal and interval-timer state, and it exits before any backend starts.
/// Outside a PID-namespace init (notably `--no-namespace`) this is a deliberate
/// no-op; Hermit must not create a process merely to alter the host namespace.
fn align_tracing_pid_baseline(backend: Backend) {
    // KVM supplies the deterministic root identity explicitly rather than
    // deriving it from a Linux task in this namespace. Allocating PID 2 for
    // that backend is both unnecessary and harmful: it makes the setitimer
    // sandbox die from SIGALRM before a comparison can run.
    if backend == Backend::Kvm {
        return;
    }

    // SAFETY: getpid has no preconditions and cannot fail.
    if unsafe { libc::getpid() } != 1 {
        return;
    }

    // SAFETY: the child returns through `_exit` immediately and touches no Rust
    // allocation, lock, or destructor inherited across fork. POSIX specifies
    // that interval timers are not inherited by a fork child.
    let child = unsafe { libc::fork() };
    assert!(
        child >= 0,
        "failed to reserve the tracing PID slot: {}",
        io::Error::last_os_error()
    );
    if child == 0 {
        unsafe { libc::_exit(0) };
    }

    let mut status = 0;
    loop {
        let waited = unsafe { libc::waitpid(child, &mut status, 0) };
        if waited == child {
            break;
        }
        let error = io::Error::last_os_error();
        if waited < 0 && error.kind() == io::ErrorKind::Interrupted {
            continue;
        }
        panic!("failed to reap tracing PID-slot process {child}: {error}");
    }
    assert!(
        libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0,
        "tracing PID-slot process {child} exited unexpectedly: {status:#x}"
    );
    assert_eq!(
        child,
        detcore::ROOT_DETPID.as_raw() - 1,
        "fresh PID namespace did not reserve the expected tracing slot"
    );
}

/// Preserve the public stderr logger while synchronously duplicating INFO-and-
/// higher events into a private evidence descriptor. PID alignment uses the
/// short-lived process reservation above and does not create a host thread.
pub fn init_stderr_tracing_with_evidence<E>(
    level: Option<LevelFilter>,
    evidence: E,
    backend: Backend,
) -> TracingGuard
where
    E: Write + Send + 'static,
{
    align_tracing_pid_baseline(backend);
    let public_layer = tracing_subscriber::fmt::layer()
        .with_writer(|| detcore::util::RetryingStderr)
        .with_ansi(stderr().is_terminal())
        .with_filter(env_filter(level.unwrap_or(DEFAULT_TRACE_LEVEL)));
    let evidence_layer = tracing_subscriber::fmt::layer()
        .with_writer(std::sync::Mutex::new(evidence))
        .with_ansi(false)
        .with_filter(LevelFilter::INFO);
    tracing_subscriber::registry()
        .with(public_layer)
        .with(evidence_layer)
        .try_init()
        .expect("global tracing subscriber to install");
    TracingGuard::synchronous()
}

/// Returns a tracing subscriber that logs to `stderr`.
///
/// NOTE: Writes to stderr are unbuffered, so this may be slow.
pub fn stderr_subscriber(level: Option<LevelFilter>) -> impl Subscriber {
    let level = level.unwrap_or(DEFAULT_TRACE_LEVEL);

    let filter = env_filter(level);
    tracing_subscriber::fmt()
        .with_env_filter(filter)
        // NOT `io::stderr`: a guest can set O_NONBLOCK on the inherited fd 2,
        // after which a full pipe makes these writes fail with EAGAIN. The fmt
        // layer discards the write error, so log lines would vanish with no
        // marker at all. `RetryingStderr` waits for the reader instead of
        // dropping, and does not alter the flag the guest set.
        .with_writer(|| detcore::util::RetryingStderr)
        .with_ansi(stderr().is_terminal())
        .finish()
}

/// Initializes tracing to `stderr`.
///
/// NOTE: Writes to stderr are unbuffered, so this may be slow.
pub fn init_stderr_tracing(level: Option<LevelFilter>, backend: Backend) {
    align_tracing_pid_baseline(backend);

    stderr_subscriber(level)
        .try_init()
        .expect("global tracing subscriber to install")
}

#[cfg(test)]
mod tests {
    use std::sync::Mutex;

    use reverie::process::Container;
    use reverie::process::Mount;
    use reverie::process::Namespace;

    use super::*;

    /// `log_max_bytes` reads the process environment, which libtest's threads
    /// share.
    static ENV_LOCK: Mutex<()> = Mutex::new(());

    fn pid_namespace() -> Container {
        let mut container = Container::new();
        container
            .unshare(Namespace::PID)
            .map_root()
            .mount(Mount::proc().allow_readonly_fallback());
        container
    }

    fn fork_and_observe_child_pid() -> i32 {
        // SAFETY: the child makes only getpid and _exit calls. The parent waits
        // synchronously, so no child is left behind in the test namespace.
        let child = unsafe { libc::fork() };
        assert!(child >= 0, "fork failed: {}", io::Error::last_os_error());
        if child == 0 {
            let observed = unsafe { libc::getpid() };
            unsafe {
                libc::_exit(if observed == detcore::ROOT_DETPID.as_raw() {
                    0
                } else {
                    1
                })
            };
        }
        let mut status = 0;
        assert_eq!(unsafe { libc::waitpid(child, &mut status, 0) }, child);
        assert!(libc::WIFEXITED(status));
        assert_eq!(libc::WEXITSTATUS(status), 0, "child observed the wrong PID");
        child
    }

    fn deny_host_thread_creation() -> io::Result<()> {
        let denied = 0x0005_0000 | libc::EPERM as u32; // SECCOMP_RET_ERRNO
        let mut filter = [
            libc::sock_filter {
                code: 0x20, // BPF_LD | BPF_W | BPF_ABS
                jt: 0,
                jf: 0,
                k: 0, // offsetof(seccomp_data, nr)
            },
            libc::sock_filter {
                code: 0x15, // BPF_JMP | BPF_JEQ | BPF_K
                jt: 4,
                jf: 0,
                k: libc::SYS_clone3 as u32,
            },
            libc::sock_filter {
                code: 0x15,
                jt: 0,
                jf: 2,
                k: libc::SYS_clone as u32,
            },
            libc::sock_filter {
                code: 0x20, // BPF_LD | BPF_W | BPF_ABS
                jt: 0,
                jf: 0,
                k: 16, // offsetof(seccomp_data, args[0])
            },
            libc::sock_filter {
                code: 0x45, // BPF_JMP | BPF_JSET | BPF_K
                jt: 1,
                jf: 0,
                k: libc::CLONE_THREAD as u32,
            },
            libc::sock_filter {
                code: 0x06, // BPF_RET | BPF_K
                jt: 0,
                jf: 0,
                k: 0x7fff_0000, // SECCOMP_RET_ALLOW
            },
            libc::sock_filter {
                code: 0x06, // BPF_RET | BPF_K
                jt: 0,
                jf: 0,
                k: denied,
            },
        ];
        let program = libc::sock_fprog {
            len: filter.len() as u16,
            filter: filter.as_mut_ptr(),
        };
        // SAFETY: `program` describes the live filter array above. The filter
        // allows process creation but rejects clone3 and clone(CLONE_THREAD),
        // the two paths std::thread uses. It is installed only in the
        // disposable namespace child used by the test.
        unsafe {
            if libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) == -1 {
                return Err(io::Error::last_os_error());
            }
            if libc::prctl(
                libc::PR_SET_SECCOMP,
                libc::SECCOMP_MODE_FILTER,
                &program as *const libc::sock_fprog,
            ) == -1
            {
                return Err(io::Error::last_os_error());
            }
        }
        Ok(())
    }

    #[test]
    fn synchronous_pid_alignment_matches_the_file_appender_baseline() {
        let synchronous_pid = pid_namespace()
            .run(|| {
                assert_eq!(unsafe { libc::getpid() }, 1);
                align_tracing_pid_baseline(Backend::Ptrace);
                fork_and_observe_child_pid()
            })
            .expect("synchronous baseline namespace failed");

        let appender_pid = pid_namespace()
            .run(|| {
                assert_eq!(unsafe { libc::getpid() }, 1);
                let (_subscriber, guard) = file_subscriber(DEFAULT_TRACE_LEVEL, io::sink());
                let child = fork_and_observe_child_pid();
                drop(guard);
                child
            })
            .expect("file-appender baseline namespace failed");

        assert_eq!(synchronous_pid, detcore::ROOT_DETPID.as_raw());
        assert_eq!(synchronous_pid, appender_pid);
    }

    #[test]
    fn pid_alignment_succeeds_when_host_thread_creation_is_denied() {
        let guest_pid = pid_namespace()
            .run(|| {
                assert_eq!(unsafe { libc::getpid() }, 1);
                deny_host_thread_creation().expect("cannot install host-thread deny filter");
                align_tracing_pid_baseline(Backend::Ptrace);
                fork_and_observe_child_pid()
            })
            .expect("thread-free alignment namespace failed");

        assert_eq!(guest_pid, detcore::ROOT_DETPID.as_raw());
    }

    #[test]
    fn kvm_pid_alignment_allocates_no_host_task() {
        let next_pid = pid_namespace()
            .run(|| {
                assert_eq!(unsafe { libc::getpid() }, 1);
                align_tracing_pid_baseline(Backend::Kvm);
                // No task was allocated by alignment, so this first actual
                // child must receive PID 2 rather than PID 3.
                let child = unsafe { libc::fork() };
                assert!(child >= 0, "fork failed: {}", io::Error::last_os_error());
                if child == 0 {
                    unsafe { libc::_exit(0) };
                }
                let mut status = 0;
                assert_eq!(unsafe { libc::waitpid(child, &mut status, 0) }, child);
                assert!(libc::WIFEXITED(status));
                assert_eq!(libc::WEXITSTATUS(status), 0);
                child
            })
            .expect("KVM no-allocation namespace failed");

        assert_eq!(next_pid, 2);
    }

    struct FailingWriter {
        fail_write: bool,
    }

    impl Write for FailingWriter {
        fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
            if self.fail_write {
                Err(io::Error::other("injected write failure"))
            } else {
                Ok(buf.len())
            }
        }

        fn flush(&mut self) -> io::Result<()> {
            if self.fail_write {
                Ok(())
            } else {
                Err(io::Error::other("injected flush failure"))
            }
        }
    }

    #[test]
    fn private_writer_latches_write_and_flush_errors() {
        let write_latch = WriteErrorLatch::new().unwrap();
        let mut write_fails =
            LatchedWriter::new(FailingWriter { fail_write: true }, write_latch.clone());
        assert!(write_fails.write_all(b"record").is_err());
        assert!(write_latch.failed());

        let flush_latch = WriteErrorLatch::new().unwrap();
        let mut flush_fails =
            LatchedWriter::new(FailingWriter { fail_write: false }, flush_latch.clone());
        assert!(flush_fails.flush().is_err());
        assert!(flush_latch.failed());
    }

    #[test]
    fn private_writer_latch_is_visible_across_fork() {
        let latch = WriteErrorLatch::new().unwrap();
        // SAFETY: the child performs only an atomic store and _exit; it does no
        // allocation and acquires no process-local lock after this threaded
        // test harness forks.
        let pid = unsafe { libc::fork() };
        assert!(pid >= 0, "fork failed");
        if pid == 0 {
            latch.record_failure();
            unsafe { libc::_exit(0) };
        }
        let mut status = 0;
        assert_eq!(unsafe { libc::waitpid(pid, &mut status, 0) }, pid);
        assert_eq!(status, 0);
        assert!(
            latch.failed(),
            "a heap-only latch would lose the child writer failure"
        );
    }

    /// Bracketed both ways on purpose: a bound that always fires would silently
    /// truncate ordinary diagnostic logs, which is the failure mode opposite to
    /// the one being fixed and just as damaging to an investigation.
    #[test]
    fn bound_truncates_and_announces_exactly_once() {
        let mut sink: Vec<u8> = Vec::new();
        {
            let mut writer = BoundedWriter::new(&mut sink, 100);
            for _ in 0..50 {
                writer.write_all(&[b'x'; 10]).unwrap();
            }
            writer.flush().unwrap();
        }
        let body = sink.iter().filter(|byte| **byte == b'x').count();
        let text = String::from_utf8_lossy(&sink);
        assert_eq!(body, 100, "wrote past the bound");
        assert_eq!(
            text.matches("HERMIT LOG TRUNCATED").count(),
            1,
            "truncation must be announced exactly once, not per write"
        );
    }

    #[test]
    fn under_the_bound_nothing_is_truncated_or_announced() {
        let mut sink: Vec<u8> = Vec::new();
        {
            let mut writer = BoundedWriter::new(&mut sink, 1000);
            for _ in 0..50 {
                writer.write_all(&[b'x'; 10]).unwrap();
            }
        }
        assert_eq!(sink.len(), 500);
        assert!(!String::from_utf8_lossy(&sink).contains("HERMIT LOG TRUNCATED"));
    }

    #[test]
    fn a_zero_limit_disables_the_bound() {
        let mut sink: Vec<u8> = Vec::new();
        {
            let mut writer = BoundedWriter::new(&mut sink, 0);
            writer.write_all(&[b'z'; 4096]).unwrap();
        }
        assert_eq!(sink.len(), 4096);
    }

    /// The regression the unit tests originally missed and an end-to-end run
    /// caught: a log whose FINAL write crosses the bound must still announce
    /// itself. Deferring the marker to the next write left a 100-byte-bounded
    /// log at exactly 100 bytes with no marker at all.
    #[test]
    fn a_single_straddling_write_announces_without_a_following_write() {
        let mut sink: Vec<u8> = Vec::new();
        {
            let mut writer = BoundedWriter::new(&mut sink, 10);
            writer.write_all(&[b'q'; 40]).unwrap();
            // deliberately no further write
        }
        let text = String::from_utf8_lossy(&sink);
        assert_eq!(
            text.matches("HERMIT LOG TRUNCATED").count(),
            1,
            "a truncated final write must announce itself: {text}"
        );
        assert_eq!(sink.iter().filter(|b| **b == b'q').count(), 10);
    }

    /// What this writer produces must be what the comparator recognizes.
    ///
    /// `detcore::logdiff` refuses to return a comparison verdict for a
    /// truncated log. If the bytes written here ever drift from what
    /// [`detcore::logdiff::log_was_truncated`] accepts, that refusal stops
    /// firing and a truncated pair silently returns to comparing only its
    /// prefix -- with nothing failing to say so. This test is the binding
    /// between them.
    ///
    /// It runs the REAL writer's output through the REAL predicate rather than
    /// comparing two constants, so it binds the anchoring as well as the text:
    /// the marker must survive as a whole line at end of file, which is what
    /// the comparator actually keys on. Drift in this file's literal or in
    /// detcore's constant or predicate all fail it.
    #[test]
    fn the_written_marker_is_the_text_the_comparator_matches() {
        let mut sink: Vec<u8> = Vec::new();
        {
            let mut writer = BoundedWriter::new(&mut sink, 64);
            // A realistic tail: whole "lines", the last of which straddles the
            // bound, so the marker has to terminate a partial line.
            for _ in 0..20 {
                writer
                    .write_all(b"2022-09-06T14:15:47.000000Z INFO detcore: DETLOG x\n")
                    .unwrap();
            }
            writer.flush().unwrap();
        }
        let written = String::from_utf8(sink).unwrap();
        assert!(
            detcore::logdiff::log_was_truncated(&written),
            "the bounded writer produced {written:?}, which detcore::logdiff does not classify \
             as truncated; a truncated pair would silently be compared on its prefix"
        );

        // The other direction, so this cannot pass by the predicate accepting
        // everything: the same content under a bound it never reaches is NOT
        // truncated.
        let mut unbounded: Vec<u8> = Vec::new();
        {
            let mut writer = BoundedWriter::new(&mut unbounded, 0);
            writer
                .write_all(b"2022-09-06T14:15:47.000000Z INFO detcore: DETLOG x\n")
                .unwrap();
        }
        assert!(
            !detcore::logdiff::log_was_truncated(&String::from_utf8(unbounded).unwrap()),
            "an untruncated log must not be classified as truncated"
        );
    }

    /// A malformed bound must be an error, not a silent 1 GiB.
    ///
    /// `HERMIT_LOG_MAX_BYTES=0` is the documented way to disable the bound, so
    /// a value that fails to parse must not quietly become the default: that
    /// would re-enable the bound for anyone who typed `unlimited`, `none`, or
    /// `1GiB` and reasonably believed it was off.
    ///
    /// Serialized against the other env-var cases in this file because the
    /// process environment is global; `log_max_bytes` reads it directly.
    #[test]
    fn a_malformed_bound_is_an_error_not_a_silent_default() {
        let _lock = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
        let restore = std::env::var(LOG_MAX_BYTES_ENV).ok();

        // SAFETY: guarded by ENV_LOCK; no other thread in this test binary
        // reads the environment concurrently.
        unsafe {
            for bad in ["unlimited", "none", "1GiB", "-1", "", "1 000"] {
                std::env::set_var(LOG_MAX_BYTES_ENV, bad);
                assert!(
                    log_max_bytes().is_err(),
                    "{bad:?} must be rejected, not silently read as {DEFAULT_LOG_MAX_BYTES}"
                );
            }
            // Bracket the accepting direction, including the documented
            // "disable" value, so this is not a check that rejects everything.
            for (good, expected) in [("0", 0u64), ("4096", 4096), (" 4096 ", 4096)] {
                std::env::set_var(LOG_MAX_BYTES_ENV, good);
                assert_eq!(
                    log_max_bytes().unwrap(),
                    expected,
                    "{good:?} must be accepted"
                );
            }
            std::env::remove_var(LOG_MAX_BYTES_ENV);
            assert_eq!(
                log_max_bytes().unwrap(),
                DEFAULT_LOG_MAX_BYTES,
                "an unset variable is the only thing that means the default"
            );
            match restore {
                Some(v) => std::env::set_var(LOG_MAX_BYTES_ENV, v),
                None => std::env::remove_var(LOG_MAX_BYTES_ENV),
            }
        }
    }

    /// The non-obvious correctness point: `tracing_appender` treats a short
    /// write as an I/O error, so a discarding writer must still report the
    /// caller's whole buffer as consumed.
    #[test]
    fn a_write_straddling_the_bound_reports_full_consumption() {
        let mut sink: Vec<u8> = Vec::new();
        let mut writer = BoundedWriter::new(&mut sink, 5);
        assert_eq!(writer.write(&[b'y'; 40]).unwrap(), 40);
    }
}