Skip to main content

readcon_core/
conbin.rs

1//! CON-bin: one page of header, then every frame's positions as one run
2//! of little-endian values, then the velocities the same way.
3//!
4//! [`map_file`] maps that file and faults it in place. The runs are the
5//! columns: the caller does not allocate a second buffer, so the read does
6//! not take a write fault per destination page. [`read_into_threads`] is the
7//! `pread` fill of a caller-owned slice, kept so a measurement can pair the
8//! two on the same file.
9
10use std::fs::File;
11use std::io::{Read, Seek, SeekFrom, Write};
12#[cfg(unix)]
13use std::os::fd::AsRawFd;
14use std::path::Path;
15
16use crate::error::ParseError;
17use crate::iterators::ConFrameIterator;
18
19const MAGIC: &[u8; 8] = b"CONBIN01";
20const VERSION: u32 = 1;
21const FLAG_F32: u32 = 1;
22const FLAG_VELOCITY: u32 = 2;
23
24/// Magic, version, flags, frame count, atom count, payload offset.
25/// 8 + 4 + 4 + 8 + 8 + 8.
26const FIELD_BYTES: usize = 40;
27
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
29pub struct ConBinHeader {
30    pub n_frames: u64,
31    pub n_atoms: u64,
32    pub binary32: bool,
33    pub velocities: bool,
34}
35
36impl ConBinHeader {
37    pub fn position_len(&self) -> Result<usize, ParseError> {
38        (self.n_frames as usize)
39            .checked_mul(self.n_atoms as usize)
40            .and_then(|n| n.checked_mul(3))
41            .ok_or_else(|| ParseError::InvalidCount("CON-bin column length overflows".into()))
42    }
43}
44
45/// How the mapped pages are made resident before the caller reads them.
46#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47pub enum Prefetch {
48    /// One `MADV_POPULATE_READ` per worker, covering that worker's span.
49    Span,
50    /// `MADV_POPULATE_READ` in queue-sized pieces of each worker's span.
51    ChunkPopulate,
52    /// `posix_fadvise(POSIX_FADV_WILLNEED)` of the next queue-sized piece
53    /// on a private file descriptor, issued before the caller reads.
54    ChunkWillneed,
55    /// Will-need the next piece, then `MADV_POPULATE_READ` the current one.
56    Hybrid,
57}
58
59/// Limits read from the block queue that holds `path`.
60///
61/// `max_request` is `max_sectors_kb` times 1024, the largest request that
62/// queue accepts
63/// (<https://docs.kernel.org/admin-guide/abi-stable.html#abi-sys-block-disk-queue-max-sectors-kb>).
64/// `nr_requests` is how many requests the queue can hold
65/// (<https://docs.kernel.org/admin-guide/abi-stable.html#abi-sys-block-disk-queue-nr-requests>).
66/// `read_ahead` is `read_ahead_kb`
67/// times 1024, the kernel readahead window. A partition is walked to its
68/// parent disk, which is the sysfs node that holds `queue/`. A missing
69/// queue file yields the page size, the unit `madvise` accepts, and a
70/// missing `nr_requests` yields 1. `rotational` is recorded and does not
71/// select the plan.
72#[derive(Debug, Clone, Copy)]
73pub struct QueueLimits {
74    pub page: usize,
75    pub logical_block: usize,
76    pub max_request: usize,
77    pub nr_requests: usize,
78    pub read_ahead: usize,
79    /// Hardware queues. blk-mq applies `nr_requests` to each one.
80    pub nr_hw_queues: usize,
81    /// `queue/rotational` is 1 when the queue file says so.
82    /// The flag is recorded. It does not select the read plan.
83    pub rotational: bool,
84}
85
86impl QueueLimits {
87    /// Page-sized stand-in used when the file has no block queue.
88    ///
89    /// btrfs, tmpfs, overlayfs and NFS have no `queue/` directory.
90    /// The page is the unit `madvise` accepts, so one request of one
91    /// page is the stand-in.
92    pub fn page_default(page: usize) -> Self {
93        Self {
94            page,
95            logical_block: page,
96            max_request: page,
97            nr_requests: 1,
98            read_ahead: page,
99            nr_hw_queues: 1,
100            rotational: false,
101        }
102    }
103
104    pub fn of(path: &Path) -> Result<Self, ParseError> {
105        let page = page_size()?;
106        #[cfg(target_os = "linux")]
107        {
108            let file =
109                File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
110            let dev = std::os::unix::fs::MetadataExt::dev(
111                &file
112                    .metadata()
113                    .map_err(|err| ParseError::ValidationError(err.to_string()))?,
114            );
115            Ok(Self::from_sysfs(dev, page))
116        }
117        #[cfg(not(target_os = "linux"))]
118        {
119            let _ = path;
120            Ok(Self::page_default(page))
121        }
122    }
123
124    #[cfg(target_os = "linux")]
125    fn from_sysfs(dev: u64, page: usize) -> Self {
126        let num =
127            |name: &str| queue_text(dev, name).and_then(|text| text.trim().parse::<usize>().ok());
128        let logical = num("logical_block_size").unwrap_or(page).max(1);
129        let max_request = num("max_sectors_kb")
130            .map(|kb| kb.saturating_mul(1024))
131            .unwrap_or(page)
132            .max(page);
133        let nr_requests = num("nr_requests").unwrap_or(1).max(1);
134        let read_ahead = num("read_ahead_kb")
135            .map(|kb| kb.saturating_mul(1024))
136            .unwrap_or(page)
137            .max(page);
138        let rotational = num("rotational").unwrap_or(0) != 0;
139        Self {
140            page,
141            logical_block: logical,
142            max_request,
143            nr_requests,
144            read_ahead,
145            nr_hw_queues: hw_queue_count(dev),
146            rotational,
147        }
148    }
149
150    /// Bytes one worker submits while the others share the same queue.
151    ///
152    /// The queue holds `nr_requests` requests of `max_request` bytes.
153    /// `workers` share it, so each gets `nr_requests / workers` requests,
154    /// or one request when the queue is shorter than the worker count.
155    /// The span is at least the kernel readahead window, because a shorter
156    /// hint still pulls that window, and it is aligned down to the page.
157    pub fn worker_span(&self, workers: usize) -> usize {
158        let workers = workers.max(1);
159        let requests = (self.nr_requests / workers).max(1);
160        let queued = requests.saturating_mul(self.max_request);
161        let span = queued.max(self.read_ahead).max(self.page);
162        let aligned = span & !(self.page - 1);
163        aligned.max(self.page)
164    }
165}
166
167#[cfg(target_os = "linux")]
168pub(crate) fn dev_nums(dev: u64) -> (u64, u64) {
169    // glibc `sys/sysmacros.h`, the encoding major(3) documents.
170    let major = ((dev >> 32) & 0xffff_f000) | ((dev >> 8) & 0xfff);
171    let minor = ((dev >> 12) & 0xffff_ff00) | (dev & 0xff);
172    (major, minor)
173}
174
175/// `queue/nr_hw_queues` when the file exists. Otherwise the number of
176/// directories under `mq/`, which blk-mq creates, one per hardware queue.
177/// A missing tree means one queue.
178#[cfg(target_os = "linux")]
179fn hw_queue_count(dev: u64) -> usize {
180    if let Some(text) = queue_text(dev, "nr_hw_queues")
181        && let Ok(count) = text.trim().parse::<usize>()
182    {
183        return count.max(1);
184    }
185    let (major, minor) = dev_nums(dev);
186    let mut dir = std::path::PathBuf::from(format!("/sys/dev/block/{major}:{minor}"));
187    if dir.join("partition").is_file() {
188        dir.push("..");
189    }
190    let mq = dir.join("mq");
191    let count = std::fs::read_dir(&mq)
192        .ok()
193        .map(|entries| entries.filter_map(|entry| entry.ok()).count())
194        .unwrap_or(0);
195    count.max(1)
196}
197
198/// Read one queue file. A partition node has `partition` and no `queue`.
199/// Its parent is the disk, and that directory holds `queue/`.
200/// `Documentation/admin-guide/devices.rst` describes the block sysfs tree.
201#[cfg(target_os = "linux")]
202fn queue_text(dev: u64, name: &str) -> Option<String> {
203    let (major, minor) = dev_nums(dev);
204    let dir = std::path::PathBuf::from(format!("/sys/dev/block/{major}:{minor}"));
205    let own = dir.join("queue").join(name);
206    if let Ok(text) = std::fs::read_to_string(&own) {
207        return Some(text);
208    }
209    if dir.join("partition").is_file() {
210        let parent = dir.join("..").join("queue").join(name);
211        if let Ok(text) = std::fs::read_to_string(&parent) {
212            return Some(text);
213        }
214    }
215    None
216}
217
218/// Host page size. `madvise` rejects an address that is not a multiple of it.
219fn page_size() -> Result<usize, ParseError> {
220    #[cfg(unix)]
221    let page = {
222        let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
223        if page <= 0 {
224            return Err(ParseError::ValidationError(
225                "sysconf(SC_PAGESIZE) failed".into(),
226            ));
227        }
228        page as usize
229    };
230    #[cfg(windows)]
231    let page = {
232        // dwPageSize is the DWORD after the 4-byte architecture union in
233        // SYSTEM_INFO, on both 32-bit and 64-bit Windows.
234        // https://learn.microsoft.com/windows/win32/api/sysinfoapi/ns-sysinfoapi-system_info
235        #[link(name = "kernel32")]
236        unsafe extern "system" {
237            fn GetSystemInfo(info: *mut std::ffi::c_void);
238        }
239        let mut info = [0u8; 64];
240        unsafe { GetSystemInfo(info.as_mut_ptr().cast()) };
241        u32::from_ne_bytes(info[4..8].try_into().unwrap()) as usize
242    };
243    if !page.is_power_of_two() || page == 0 {
244        return Err(ParseError::ValidationError(
245            "the host page size is not a power of two".into(),
246        ));
247    }
248    Ok(page)
249}
250
251fn header_fields(header: &ConBinHeader, payload_off: u64) -> [u8; FIELD_BYTES] {
252    let mut out = [0u8; FIELD_BYTES];
253    out[..8].copy_from_slice(MAGIC);
254    out[8..12].copy_from_slice(&VERSION.to_le_bytes());
255    let mut flags = 0u32;
256    if header.binary32 {
257        flags |= FLAG_F32;
258    }
259    if header.velocities {
260        flags |= FLAG_VELOCITY;
261    }
262    out[12..16].copy_from_slice(&flags.to_le_bytes());
263    out[16..24].copy_from_slice(&header.n_frames.to_le_bytes());
264    out[24..32].copy_from_slice(&header.n_atoms.to_le_bytes());
265    out[32..40].copy_from_slice(&payload_off.to_le_bytes());
266    out
267}
268
269fn parse_fields(bytes: &[u8]) -> Result<(ConBinHeader, u64), ParseError> {
270    if bytes.len() < FIELD_BYTES || &bytes[..8] != MAGIC {
271        return Err(ParseError::ValidationError(
272            "CON-bin header is not CONBIN01".into(),
273        ));
274    }
275    let version = u32::from_le_bytes(bytes[8..12].try_into().unwrap());
276    if version != VERSION {
277        return Err(ParseError::ValidationError(format!(
278            "CON-bin version {version} is not {VERSION}"
279        )));
280    }
281    let flags = u32::from_le_bytes(bytes[12..16].try_into().unwrap());
282    let offset = u64::from_le_bytes(bytes[32..40].try_into().unwrap());
283    Ok((
284        ConBinHeader {
285            n_frames: u64::from_le_bytes(bytes[16..24].try_into().unwrap()),
286            n_atoms: u64::from_le_bytes(bytes[24..32].try_into().unwrap()),
287            binary32: flags & FLAG_F32 != 0,
288            velocities: flags & FLAG_VELOCITY != 0,
289        },
290        offset,
291    ))
292}
293
294/// Payload start stored in the header. Any 8-byte alignment inside the
295/// file is accepted, so a file written on a 4 KiB host still opens when
296/// the host page is 16 KiB or 64 KiB. An offset of 0 is not a stand-in
297/// for the host page.
298fn checked_payload_offset(stored: u64, file_len: usize) -> Result<usize, ParseError> {
299    let off = usize::try_from(stored)
300        .map_err(|_| ParseError::ValidationError("CON-bin payload offset does not fit".into()))?;
301    if off < FIELD_BYTES || off > file_len || off % 8 != 0 {
302        return Err(ParseError::ValidationError(
303            "CON-bin payload offset is not an 8-byte aligned address inside the file".into(),
304        ));
305    }
306    Ok(off)
307}
308
309#[derive(Clone, Copy)]
310struct F64Payload {
311    offset: usize,
312    end: usize,
313}
314
315/// Binary64 columns with a velocity run. Binary32 and a missing velocity
316/// run are errors: `sum_mapped` reads those two binary64 runs.
317fn f64_payload(header_bytes: &[u8], file_len: usize) -> Result<F64Payload, ParseError> {
318    let (header, stored) = parse_fields(header_bytes)?;
319    if header.binary32 {
320        return Err(ParseError::ValidationError(
321            "CON-bin file is binary32; sum_mapped reads binary64".into(),
322        ));
323    }
324    if !header.velocities {
325        return Err(ParseError::ValidationError(
326            "CON-bin file has no velocity run".into(),
327        ));
328    }
329    let offset = checked_payload_offset(stored, file_len)?;
330    let values = header
331        .position_len()?
332        .checked_mul(2)
333        .ok_or_else(|| ParseError::InvalidCount("CON-bin column length overflows".into()))?;
334    let nbytes = values
335        .checked_mul(8)
336        .ok_or_else(|| ParseError::InvalidCount("CON-bin column byte length overflows".into()))?;
337    let end = offset
338        .checked_add(nbytes)
339        .ok_or_else(|| ParseError::InvalidCount("CON-bin payload offset overflows".into()))?;
340    if end > file_len {
341        return Err(ParseError::IncompleteFrame);
342    }
343    Ok(F64Payload { offset, end })
344}
345
346/// A mapped CON-bin. The position run and the velocity run are the file.
347pub struct ConBinMap {
348    /// `MAP_PRIVATE` copy-on-write (`MmapOptions::map_copy`). A store copies
349    /// the page and does not write the file.
350    pub map: memmap2::MmapMut,
351    pub header: ConBinHeader,
352    pub payload_off: usize,
353}
354
355impl ConBinMap {
356    pub fn positions_f64(&self) -> Result<&[f64], ParseError> {
357        self.run(std::mem::size_of::<f64>(), 0)
358    }
359
360    pub fn positions_f64_mut(&mut self) -> Result<&mut [f64], ParseError> {
361        self.run_mut(std::mem::size_of::<f64>(), 0)
362    }
363
364    pub fn velocities_f64(&self) -> Result<&[f64], ParseError> {
365        let n = self.header.position_len()?;
366        self.run(std::mem::size_of::<f64>(), n)
367    }
368
369    pub fn positions_f32(&self) -> Result<&[f32], ParseError> {
370        self.run(std::mem::size_of::<f32>(), 0)
371    }
372
373    pub fn velocities_f32(&self) -> Result<&[f32], ParseError> {
374        let n = self.header.position_len()?;
375        self.run(std::mem::size_of::<f32>(), n)
376    }
377
378    fn run<T>(&self, width: usize, column: usize) -> Result<&[T], ParseError> {
379        let n = self.header.position_len()?;
380        let start = self
381            .payload_off
382            .checked_add(column.checked_mul(width).ok_or_else(|| {
383                ParseError::InvalidCount("CON-bin column byte length overflows".into())
384            })?)
385            .ok_or_else(|| ParseError::InvalidCount("CON-bin payload offset overflows".into()))?;
386        let nbytes = n.checked_mul(width).ok_or_else(|| {
387            ParseError::InvalidCount("CON-bin column byte length overflows".into())
388        })?;
389        let end = start.checked_add(nbytes).ok_or_else(|| {
390            ParseError::InvalidCount("CON-bin column extends past the file".into())
391        })?;
392        if end > self.map.len() {
393            return Err(ParseError::IncompleteFrame);
394        }
395        let bytes = &self.map[start..end];
396        if !cfg!(target_endian = "little") {
397            return Err(ParseError::ValidationError(
398                "CON-bin is little-endian; this host cannot view it in place".into(),
399            ));
400        }
401        if bytes.as_ptr().align_offset(std::mem::align_of::<T>()) != 0 {
402            return Err(ParseError::ValidationError(
403                "CON-bin payload is not aligned for its element type".into(),
404            ));
405        }
406        // SAFETY: `bytes` is inside the map, its length is a multiple of `T`,
407        // and the pointer is aligned for `T`. The file is little-endian and
408        // the host is little-endian, so the bit pattern is the value.
409        // The map stays borrowed for the returned slice.
410        Ok(unsafe { std::slice::from_raw_parts(bytes.as_ptr().cast::<T>(), n) })
411    }
412
413    fn run_mut<T>(&mut self, width: usize, column: usize) -> Result<&mut [T], ParseError> {
414        let n = self.header.position_len()?;
415        let start = self
416            .payload_off
417            .checked_add(column.checked_mul(width).ok_or_else(|| {
418                ParseError::InvalidCount("CON-bin column byte length overflows".into())
419            })?)
420            .ok_or_else(|| ParseError::InvalidCount("CON-bin payload offset overflows".into()))?;
421        let nbytes = n.checked_mul(width).ok_or_else(|| {
422            ParseError::InvalidCount("CON-bin column byte length overflows".into())
423        })?;
424        let end = start.checked_add(nbytes).ok_or_else(|| {
425            ParseError::InvalidCount("CON-bin column extends past the file".into())
426        })?;
427        if end > self.map.len() {
428            return Err(ParseError::IncompleteFrame);
429        }
430        if !cfg!(target_endian = "little") {
431            return Err(ParseError::ValidationError(
432                "CON-bin is little-endian; this host cannot view it in place".into(),
433            ));
434        }
435        let ptr = self.map[start..end].as_mut_ptr();
436        if ptr.align_offset(std::mem::align_of::<T>()) != 0 {
437            return Err(ParseError::ValidationError(
438                "CON-bin payload is not aligned for its element type".into(),
439            ));
440        }
441        // SAFETY: same alignment and length rules as `run`. The map is
442        // `MAP_PRIVATE`, so stores copy a page and do not write the file.
443        Ok(unsafe { std::slice::from_raw_parts_mut(ptr.cast::<T>(), n) })
444    }
445}
446
447/// Sum both binary64 columns of `path`.
448///
449/// The default is one `readahead` stream.
450/// Two sessions on one boot compared one stream with a stream per span.
451/// Neither plan won at every thread count in both.
452/// The host cache was not controlled.
453/// The default stays one stream, the simpler plan.
454/// `nr_requests` is the depth of each hardware queue (`nr_hw_queues`
455/// of them, blk-mq), not one device-wide pool.
456/// Workers sum a piece after that piece is faulted in, and the piece
457/// sums are added in file order. The header's payload offset, length
458/// and dtype select the bytes. Binary32 and a missing velocity run are
459/// errors. Other platforms map the file and sum it with no `readahead`.
460pub fn sum_mapped(path: &Path, threads: usize) -> Result<f64, ParseError> {
461    #[cfg(target_os = "linux")]
462    {
463        let limits = QueueLimits::of(path)?;
464        sequential_resident_sum(path, threads.max(1), &limits)
465    }
466    #[cfg(not(target_os = "linux"))]
467    {
468        let _ = threads;
469        plain_sum(path)
470    }
471}
472
473#[cfg(not(target_os = "linux"))]
474fn plain_sum(path: &Path) -> Result<f64, ParseError> {
475    let file = File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
476    let map = unsafe { memmap2::Mmap::map(&file) }
477        .map_err(|err| ParseError::ValidationError(err.to_string()))?;
478    if map.len() < FIELD_BYTES {
479        return Err(ParseError::ValidationError(
480            "CON-bin header is not CONBIN01".into(),
481        ));
482    }
483    let payload = f64_payload(&map[..FIELD_BYTES], map.len())?;
484    if !cfg!(target_endian = "little") {
485        return Err(ParseError::ValidationError(
486            "CON-bin is little-endian; this host cannot view it in place".into(),
487        ));
488    }
489    let bytes = &map[payload.offset..payload.end];
490    let ptr = bytes.as_ptr();
491    if ptr.align_offset(std::mem::align_of::<f64>()) != 0 {
492        return Err(ParseError::ValidationError(
493            "CON-bin payload is not aligned for its element type".into(),
494        ));
495    }
496    let data = unsafe { std::slice::from_raw_parts(ptr.cast::<f64>(), bytes.len() / 8) };
497    Ok(data.iter().copied().sum())
498}
499
500#[cfg(target_os = "linux")]
501fn sequential_resident_sum(
502    path: &Path,
503    threads: usize,
504    limits: &QueueLimits,
505) -> Result<f64, ParseError> {
506    let owned = open_map(path)?;
507    let page = limits.page;
508    let len = owned.len;
509    let base = owned.ptr as usize;
510    let header =
511        unsafe { std::slice::from_raw_parts(owned.ptr.cast::<u8>(), FIELD_BYTES.min(len)) };
512    let payload = f64_payload(header, len)?;
513    let request = (limits.max_request & !(page - 1)).max(page);
514    let pieces = piece_list(0, len, request);
515    if pages_all_resident(owned.ptr, len, page)? {
516        return Ok(sum_pieces_in_order(
517            base,
518            payload.offset,
519            payload.end,
520            threads,
521            &pieces,
522        ));
523    }
524    // One stream fills the read side of the queue.
525    let depth = limits.nr_requests.max(1);
526    let (tx, rx) = std::sync::mpsc::sync_channel::<usize>(depth);
527    let (err_tx, err_rx) = std::sync::mpsc::channel::<String>();
528    let rx = std::sync::Arc::new(std::sync::Mutex::new(rx));
529    let addr = owned.ptr as usize;
530    let found = std::thread::scope(|scope| {
531        let queued = pieces.clone();
532        scope.spawn(move || {
533            let fd = match File::open(path) {
534                Ok(fd) => fd,
535                Err(err) => {
536                    let _ = err_tx.send(err.to_string());
537                    return;
538                }
539            };
540            // Keep `depth` requests in flight. Submit the next piece only
541            // after the oldest one has been waited on, so the count does not
542            // become depth + 1.
543            let mut submitted = 0usize;
544            while submitted < depth.min(queued.len()) {
545                if let Err(err) = advise_readahead(&fd, queued[submitted].0, queued[submitted].1) {
546                    let _ = err_tx.send(err);
547                    return;
548                }
549                submitted += 1;
550            }
551            for i in 0..queued.len() {
552                let (off, n) = queued[i];
553                if let Err(err) = wait_resident(addr, off, n, page) {
554                    let _ = err_tx.send(err);
555                    return;
556                }
557                let _ = tx.send(i);
558                if submitted < queued.len() {
559                    if let Err(err) =
560                        advise_readahead(&fd, queued[submitted].0, queued[submitted].1)
561                    {
562                        let _ = err_tx.send(err);
563                        return;
564                    }
565                    submitted += 1;
566                }
567            }
568        });
569        let slots = std::sync::Arc::new(std::sync::Mutex::new(vec![0.0f64; pieces.len()]));
570        let mut handles = Vec::with_capacity(threads);
571        for _ in 0..threads {
572            let rx = std::sync::Arc::clone(&rx);
573            let slots = std::sync::Arc::clone(&slots);
574            let pieces = pieces.clone();
575            handles.push(scope.spawn(move || {
576                loop {
577                    let msg = rx.lock().expect("piece queue").recv();
578                    let Ok(index) = msg else { break };
579                    let (off, n) = pieces[index];
580                    let part = sum_range(base, off, off + n, payload.offset, payload.end);
581                    slots.lock().expect("piece sums")[index] = part;
582                }
583            }));
584        }
585        for handle in handles {
586            handle.join().expect("column sum");
587        }
588        let slots = slots.lock().expect("piece sums");
589        slots.iter().copied().sum::<f64>()
590    });
591    if let Ok(err) = err_rx.try_recv() {
592        return Err(ParseError::ValidationError(err));
593    }
594    Ok(found)
595}
596
597/// One `readahead` stream per worker, each over a disjoint span.
598///
599/// The piece is `max_sectors_kb` times 1024. `nr_requests` is the depth
600/// of each hardware queue, so each worker keeps
601/// `max(1, nr_requests / workers)` pieces in flight.
602/// Two sessions on one boot compared one stream with a stream per span.
603/// Neither plan won at every thread count in both.
604/// The host cache was not controlled.
605/// The default stays one stream, the simpler plan.
606#[cfg(target_os = "linux")]
607pub fn sum_split_streams(path: &Path, threads: usize) -> Result<f64, ParseError> {
608    let limits = QueueLimits::of(path)?;
609    split_stream_sum(path, threads.max(1), &limits)
610}
611
612#[cfg(not(target_os = "linux"))]
613pub fn sum_split_streams(path: &Path, threads: usize) -> Result<f64, ParseError> {
614    let _ = threads;
615    plain_sum(path)
616}
617
618#[cfg(target_os = "linux")]
619fn split_stream_sum(path: &Path, threads: usize, limits: &QueueLimits) -> Result<f64, ParseError> {
620    let owned = open_map(path)?;
621    let page = limits.page;
622    let len = owned.len;
623    let base = owned.ptr as usize;
624    let header =
625        unsafe { std::slice::from_raw_parts(owned.ptr.cast::<u8>(), FIELD_BYTES.min(len)) };
626    let payload = f64_payload(header, len)?;
627    let request = (limits.max_request & !(page - 1)).max(page);
628    let ranges = worker_ranges(len, threads, page);
629    if pages_all_resident(owned.ptr, len, page)? {
630        let pieces = piece_list(0, len, request);
631        return Ok(sum_pieces_in_order(
632            base,
633            payload.offset,
634            payload.end,
635            threads,
636            &pieces,
637        ));
638    }
639    let depth = (limits.nr_requests / ranges.len()).max(1);
640    let (err_tx, err_rx) = std::sync::mpsc::channel::<String>();
641    let sum = std::thread::scope(|scope| {
642        let mut handles = Vec::with_capacity(ranges.len());
643        for (span, end) in ranges {
644            let err_tx = err_tx.clone();
645            handles.push(scope.spawn(move || {
646                match stream_one_span(
647                    path,
648                    SpanGeom {
649                        base,
650                        span,
651                        end,
652                        page,
653                        payload_off: payload.offset,
654                        payload_end: payload.end,
655                        request,
656                        depth,
657                    },
658                ) {
659                    Ok(part) => part,
660                    Err(err) => {
661                        let _ = err_tx.send(err);
662                        0.0
663                    }
664                }
665            }));
666        }
667        drop(err_tx);
668        handles
669            .into_iter()
670            .map(|handle| handle.join().expect("split stream"))
671            .sum::<f64>()
672    });
673    if let Ok(err) = err_rx.try_recv() {
674        return Err(ParseError::ValidationError(err));
675    }
676    Ok(sum)
677}
678
679#[cfg(target_os = "linux")]
680struct SpanGeom {
681    base: usize,
682    span: usize,
683    end: usize,
684    page: usize,
685    payload_off: usize,
686    payload_end: usize,
687    request: usize,
688    depth: usize,
689}
690
691#[cfg(target_os = "linux")]
692fn stream_one_span(path: &Path, geom: SpanGeom) -> Result<f64, String> {
693    let SpanGeom {
694        base,
695        span,
696        end,
697        page,
698        payload_off,
699        payload_end,
700        request,
701        depth,
702    } = geom;
703    let fd = File::open(path).map_err(|err| err.to_string())?;
704    let pieces = piece_list(span, end, request);
705    let depth = depth.max(1);
706    let mut next_submit = 0usize;
707    let mut next_sum = 0usize;
708    let mut acc = 0.0f64;
709    while next_sum < pieces.len() {
710        while next_submit < pieces.len() && next_submit - next_sum < depth {
711            let (off, n) = pieces[next_submit];
712            advise_readahead(&fd, off, n)?;
713            next_submit += 1;
714        }
715        let (off, n) = pieces[next_sum];
716        wait_resident(base, off, n, page)?;
717        acc += sum_range(base, off, off + n, payload_off, payload_end);
718        next_sum += 1;
719    }
720    Ok(acc)
721}
722
723#[cfg(target_os = "linux")]
724fn sum_pieces_in_order(
725    base: usize,
726    payload_off: usize,
727    payload_end: usize,
728    threads: usize,
729    pieces: &[(usize, usize)],
730) -> f64 {
731    let mut slots = vec![0.0f64; pieces.len()];
732    let next = std::sync::atomic::AtomicUsize::new(0);
733    std::thread::scope(|scope| {
734        let mut handles = Vec::with_capacity(threads);
735        for _ in 0..threads {
736            handles.push(scope.spawn(|| {
737                let mut local = Vec::new();
738                loop {
739                    let index = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
740                    if index >= pieces.len() {
741                        break;
742                    }
743                    let (off, n) = pieces[index];
744                    local.push((
745                        index,
746                        sum_range(base, off, off + n, payload_off, payload_end),
747                    ));
748                }
749                local
750            }));
751        }
752        for handle in handles {
753            for (index, part) in handle.join().expect("column sum") {
754                slots[index] = part;
755            }
756        }
757    });
758    slots.iter().copied().sum()
759}
760
761#[cfg(target_os = "linux")]
762fn pages_all_resident(ptr: *mut libc::c_void, len: usize, page: usize) -> Result<bool, ParseError> {
763    if len == 0 {
764        return Ok(true);
765    }
766    let pages = len.div_ceil(page);
767    let mut vec = vec![0u8; pages];
768    let rc = unsafe { libc::mincore(ptr, len, vec.as_mut_ptr()) };
769    if rc != 0 {
770        return Err(ParseError::ValidationError(
771            std::io::Error::last_os_error().to_string(),
772        ));
773    }
774    Ok(vec.iter().all(|byte| byte & 1 == 1))
775}
776
777/// `Ok` means the range is resident. `Err` is the errno from `madvise`.
778/// Only `EINVAL` means this kernel has no `MADV_POPULATE_READ`, so the
779/// caller may fault the pages itself. `EFAULT`, `EIO`, `EHWPOISON` and
780/// `ENOMEM` are failures (man 2 madvise).
781#[cfg(target_os = "linux")]
782fn populate_followup(code: i32) -> Result<(), i32> {
783    if code == libc::EINVAL {
784        Ok(())
785    } else {
786        Err(code)
787    }
788}
789
790#[cfg(target_os = "linux")]
791fn touch_pages(addr: usize, off: usize, len: usize, page: usize) {
792    let pages = len.div_ceil(page);
793    for index in 0..pages {
794        let at = index * page;
795        if at >= len {
796            break;
797        }
798        let byte = (addr + off + at) as *const u8;
799        std::hint::black_box(unsafe { std::ptr::read_volatile(byte) });
800    }
801}
802
803#[cfg(target_os = "linux")]
804fn wait_resident(addr: usize, off: usize, len: usize, page: usize) -> Result<(), String> {
805    // One readahead has already been submitted. MADV_POPULATE_READ (Linux
806    // 5.14) faults this range and waits. The address is checked before the
807    // call, so EINVAL is the old-kernel miss and one byte of each page is
808    // read. EAGAIN is a transient kernel resource (man 2 madvise) and is
809    // retried once per page of the range. Any other errno is returned.
810    if page == 0 || !(addr + off).is_multiple_of(page) {
811        return Err("madvise address is not page-aligned".into());
812    }
813    let start = (addr + off) as *mut libc::c_void;
814    let pages = len.div_ceil(page).max(1);
815    let mut tried = 0usize;
816    loop {
817        let rc = unsafe { libc::madvise(start, len, libc::MADV_POPULATE_READ) };
818        if rc == 0 {
819            return Ok(());
820        }
821        let err = std::io::Error::last_os_error();
822        let code = err.raw_os_error().unwrap_or(0);
823        if code == libc::EAGAIN && tried < pages {
824            tried += 1;
825            continue;
826        }
827        if populate_followup(code).is_ok() {
828            touch_pages(addr, off, len, page);
829            return Ok(());
830        }
831        return Err(err.to_string());
832    }
833}
834
835#[cfg(target_os = "linux")]
836fn advise_readahead(file: &File, off: usize, len: usize) -> Result<(), String> {
837    let rc = unsafe { libc::readahead(file.as_raw_fd(), off as i64, len) };
838    if rc != 0 {
839        Err(std::io::Error::last_os_error().to_string())
840    } else {
841        Ok(())
842    }
843}
844
845#[cfg(target_os = "linux")]
846struct OwnedMap {
847    ptr: *mut libc::c_void,
848    len: usize,
849}
850
851#[cfg(target_os = "linux")]
852impl Drop for OwnedMap {
853    fn drop(&mut self) {
854        unsafe { libc::munmap(self.ptr, self.len) };
855    }
856}
857
858#[cfg(target_os = "linux")]
859fn open_map(path: &Path) -> Result<OwnedMap, ParseError> {
860    let file = File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
861    let len = file
862        .metadata()
863        .map_err(|err| ParseError::ValidationError(err.to_string()))?
864        .len() as usize;
865    let ptr = unsafe {
866        libc::mmap(
867            std::ptr::null_mut(),
868            len,
869            libc::PROT_READ,
870            libc::MAP_PRIVATE,
871            file.as_raw_fd(),
872            0,
873        )
874    };
875    if ptr == libc::MAP_FAILED {
876        return Err(ParseError::ValidationError(
877            std::io::Error::last_os_error().to_string(),
878        ));
879    }
880    unsafe {
881        libc::madvise(ptr, len, libc::MADV_SEQUENTIAL);
882        libc::madvise(ptr, len, libc::MADV_HUGEPAGE);
883    }
884    Ok(OwnedMap { ptr, len })
885}
886
887#[cfg(target_os = "linux")]
888fn sum_range(base: usize, start: usize, end: usize, payload_off: usize, payload_end: usize) -> f64 {
889    let from = start.max(payload_off);
890    let to = end.min(payload_end);
891    if to <= from {
892        return 0.0;
893    }
894    let data = unsafe { std::slice::from_raw_parts((base + from) as *const f64, (to - from) / 8) };
895    let mut acc = 0.0f64;
896    for &value in data {
897        acc += value;
898    }
899    acc
900}
901
902/// Map `path` and fault the pages in before returning.
903///
904/// The map is `MAP_PRIVATE` and writable. A store to a page copies that
905/// page. It does not modify the file. `mmap(2)` leaves it unspecified
906/// whether a later write to the file by someone else appears in a page
907/// the caller has not stored to. Truncation can raise `SIGBUS`.
908///
909/// `MADV_SEQUENTIAL` and `MADV_WILLNEED` describe one forward pass.
910/// `MADV_HUGEPAGE` is the request used when transparent huge pages are in
911/// `madvise` mode (`CONFIG_TRANSPARENT_HUGEPAGE_MADVISE`). File-backed
912/// huge pages also need `CONFIG_READ_ONLY_THP_FOR_FS`
913/// (Documentation/admin-guide/mm/transhuge.rst). This kernel's
914/// `/proc/config.gz` has that option unset, and a mapped read showed
915/// `FilePmdMapped` 0. The advice stays, because it is the switch the
916/// kernel documents, and it is a no-op without that option.
917/// `MADV_POPULATE_READ` (Linux 5.14) waits until the pages are resident
918/// and reports an error. `MAP_POPULATE` hides errors and cannot be split
919/// across the caller's threads after `mmap` returns.
920/// `threads` workers fault page-aligned spans. A file shorter than one
921/// page per worker stays on one worker. The page is `sysconf(SC_PAGESIZE)`.
922pub fn map_file(path: &Path, threads: usize) -> Result<ConBinMap, ParseError> {
923    map_file_with(path, threads, Prefetch::Span)
924}
925
926/// [`map_file`] with an explicit prefetch. The benchmark pairs these.
927pub fn map_file_with(
928    path: &Path,
929    threads: usize,
930    prefetch: Prefetch,
931) -> Result<ConBinMap, ParseError> {
932    let file = File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
933    // MAP_PRIVATE, PROT_READ|PROT_WRITE. Writes copy the page.
934    let map = unsafe { memmap2::MmapOptions::new().map_copy(&file) }
935        .map_err(|err| ParseError::ValidationError(err.to_string()))?;
936    if map.len() < FIELD_BYTES {
937        return Err(ParseError::ValidationError(
938            "CON-bin header is not CONBIN01".into(),
939        ));
940    }
941    let (header, offset) = parse_fields(&map[..FIELD_BYTES])?;
942    let page = page_size()?;
943    let payload_off = checked_payload_offset(offset, map.len())?;
944    fault_in(path, &map, threads, page, prefetch)?;
945    Ok(ConBinMap {
946        map,
947        header,
948        payload_off,
949    })
950}
951
952fn fault_in(
953    path: &Path,
954    map: &memmap2::MmapMut,
955    threads: usize,
956    page: usize,
957    prefetch: Prefetch,
958) -> Result<(), ParseError> {
959    #[cfg(unix)]
960    let _ = map.advise(memmap2::Advice::Sequential);
961    #[cfg(target_os = "linux")]
962    let _ = map.advise(memmap2::Advice::HugePage);
963    #[cfg(target_os = "linux")]
964    {
965        match prefetch {
966            Prefetch::Span => {
967                let _ = map.advise(memmap2::Advice::WillNeed);
968                advise_spans(map, memmap2::Advice::PopulateRead, threads, page)?;
969            }
970            Prefetch::ChunkPopulate => {
971                let window = QueueLimits::of(path)?.worker_span(threads);
972                chunk_each(map, threads, page, window, ChunkOp::Populate)?;
973            }
974            Prefetch::ChunkWillneed => {
975                let window = QueueLimits::of(path)?.worker_span(threads);
976                chunk_each(map, threads, page, window, ChunkOp::Willneed { path })?;
977            }
978            Prefetch::Hybrid => {
979                let window = QueueLimits::of(path)?.worker_span(threads);
980                chunk_each(map, threads, page, window, ChunkOp::Hybrid { path })?;
981            }
982        }
983    }
984    #[cfg(not(target_os = "linux"))]
985    {
986        let _ = (path, threads, prefetch);
987        for chunk in map.chunks(page) {
988            std::hint::black_box(chunk.first().copied());
989        }
990    }
991    Ok(())
992}
993
994#[cfg(target_os = "linux")]
995enum ChunkOp<'a> {
996    Populate,
997    Willneed { path: &'a Path },
998    Hybrid { path: &'a Path },
999}
1000
1001#[cfg(target_os = "linux")]
1002fn chunk_each(
1003    map: &memmap2::MmapMut,
1004    threads: usize,
1005    page: usize,
1006    window: usize,
1007    op: ChunkOp<'_>,
1008) -> Result<(), ParseError> {
1009    let ranges = worker_ranges(map.len(), threads, page);
1010    let shared = match op {
1011        ChunkOp::Populate => ChunkOp::Populate,
1012        ChunkOp::Willneed { path } => ChunkOp::Willneed { path },
1013        ChunkOp::Hybrid { path } => ChunkOp::Hybrid { path },
1014    };
1015    std::thread::scope(|scope| {
1016        let mut handles = Vec::with_capacity(ranges.len());
1017        for (start, end) in ranges {
1018            let op = match shared {
1019                ChunkOp::Populate => ChunkOp::Populate,
1020                ChunkOp::Willneed { path } => ChunkOp::Willneed { path },
1021                ChunkOp::Hybrid { path } => ChunkOp::Hybrid { path },
1022            };
1023            handles.push(scope.spawn(move || chunk_worker(map, start, end, window, page, op)));
1024        }
1025        handles.into_iter().try_for_each(|handle| {
1026            handle.join().unwrap_or_else(|_| {
1027                Err(ParseError::ValidationError(
1028                    "CON-bin prefetch thread panicked".into(),
1029                ))
1030            })
1031        })
1032    })
1033}
1034
1035#[cfg(target_os = "linux")]
1036fn chunk_worker(
1037    map: &memmap2::MmapMut,
1038    start: usize,
1039    end: usize,
1040    window: usize,
1041    page: usize,
1042    op: ChunkOp<'_>,
1043) -> Result<(), ParseError> {
1044    let window = window.max(page);
1045    let pieces = piece_list(start, end, window);
1046    let fd = match op {
1047        ChunkOp::Willneed { path } | ChunkOp::Hybrid { path } => {
1048            Some(File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?)
1049        }
1050        ChunkOp::Populate => None,
1051    };
1052    match op {
1053        ChunkOp::Populate => {
1054            for (off, len) in pieces {
1055                populate_range(map, off, len)?;
1056            }
1057        }
1058        ChunkOp::Willneed { .. } => {
1059            let fd = fd.as_ref().expect("willneed opens a descriptor");
1060            for (off, len) in pieces {
1061                willneed(fd, off, len)?;
1062            }
1063        }
1064        ChunkOp::Hybrid { .. } => {
1065            let fd = fd.as_ref().expect("hybrid opens a descriptor");
1066            if let Some((off, len)) = pieces.first().copied() {
1067                willneed(fd, off, len)?;
1068            }
1069            for i in 0..pieces.len() {
1070                if let Some((off, len)) = pieces.get(i + 1).copied() {
1071                    willneed(fd, off, len)?;
1072                }
1073                let (off, len) = pieces[i];
1074                populate_range(map, off, len)?;
1075            }
1076        }
1077    }
1078    Ok(())
1079}
1080
1081#[cfg(target_os = "linux")]
1082fn piece_list(start: usize, end: usize, window: usize) -> Vec<(usize, usize)> {
1083    let mut pieces = Vec::new();
1084    let mut off = start;
1085    while off < end {
1086        let len = window.min(end - off);
1087        pieces.push((off, len));
1088        off += len;
1089    }
1090    pieces
1091}
1092
1093#[cfg(target_os = "linux")]
1094fn populate_range(map: &memmap2::MmapMut, off: usize, len: usize) -> Result<(), ParseError> {
1095    let page = page_size()?;
1096    wait_resident(map.as_ptr() as usize, off, len, page).map_err(ParseError::ValidationError)
1097}
1098
1099#[cfg(target_os = "linux")]
1100fn willneed(file: &File, off: usize, len: usize) -> Result<(), ParseError> {
1101    let rc = unsafe {
1102        libc::posix_fadvise(
1103            file.as_raw_fd(),
1104            off as libc::off_t,
1105            len as libc::off_t,
1106            libc::POSIX_FADV_WILLNEED,
1107        )
1108    };
1109    if rc != 0 {
1110        return Err(ParseError::ValidationError(format!(
1111            "posix_fadvise WILLNEED: {rc}"
1112        )));
1113    }
1114    Ok(())
1115}
1116
1117#[cfg(target_os = "linux")]
1118fn worker_ranges(len: usize, threads: usize, page: usize) -> Vec<(usize, usize)> {
1119    if len == 0 {
1120        return Vec::new();
1121    }
1122    let threads = threads.max(1);
1123    let workers = if len < page.saturating_mul(threads) {
1124        1
1125    } else {
1126        threads
1127    };
1128    if workers == 1 {
1129        return vec![(0, len)];
1130    }
1131    let chunk = (len / workers) & !(page - 1);
1132    let mut ranges = Vec::with_capacity(workers);
1133    let mut start = 0usize;
1134    for worker in 0..workers {
1135        let take = if worker + 1 == workers {
1136            len - start
1137        } else {
1138            chunk
1139        };
1140        ranges.push((start, start + take));
1141        start += take;
1142    }
1143    ranges
1144}
1145
1146#[cfg(target_os = "linux")]
1147fn advise_spans(
1148    map: &memmap2::MmapMut,
1149    advice: memmap2::Advice,
1150    threads: usize,
1151    page: usize,
1152) -> Result<(), ParseError> {
1153    let len = map.len();
1154    if len == 0 {
1155        return Ok(());
1156    }
1157    let threads = threads.max(1);
1158    let workers = if len < page.saturating_mul(threads) {
1159        1
1160    } else {
1161        threads
1162    };
1163    if workers == 1 {
1164        return map
1165            .advise(advice)
1166            .map_err(|err| ParseError::ValidationError(err.to_string()));
1167    }
1168    let chunk = (len / workers) & !(page - 1);
1169    std::thread::scope(|scope| {
1170        let mut start = 0usize;
1171        let mut handles = Vec::with_capacity(workers);
1172        for worker in 0..workers {
1173            let take = if worker + 1 == workers {
1174                len - start
1175            } else {
1176                chunk
1177            };
1178            let off = start;
1179            start += take;
1180            handles.push(scope.spawn(move || map.advise_range(advice, off, take)));
1181        }
1182        handles.into_iter().try_for_each(|handle| {
1183            handle
1184                .join()
1185                .unwrap_or_else(|_| Err(std::io::Error::other("CON-bin populate thread panicked")))
1186                .map_err(|err| ParseError::ValidationError(err.to_string()))
1187        })
1188    })
1189}
1190
1191pub fn read_header(path: &Path) -> Result<ConBinHeader, ParseError> {
1192    let mut file = File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
1193    let mut bytes = [0u8; FIELD_BYTES];
1194    file.read_exact(&mut bytes)
1195        .map_err(|err| ParseError::ValidationError(err.to_string()))?;
1196    parse_fields(&bytes).map(|(header, _)| header)
1197}
1198
1199fn bytes_of<T>(values: &mut [T]) -> &mut [u8] {
1200    let len = std::mem::size_of_val(values);
1201    // SAFETY: `values` is a live slice. The byte view covers exactly those
1202    // elements and is not used after `values` is borrowed again.
1203    unsafe { std::slice::from_raw_parts_mut(values.as_mut_ptr().cast::<u8>(), len) }
1204}
1205
1206/// Copy a binary64 CON-bin into caller-owned columns. One thread issues the reads.
1207pub fn read_into(
1208    path: &Path,
1209    positions: &mut [f64],
1210    velocities: Option<&mut [f64]>,
1211) -> Result<ConBinHeader, ParseError> {
1212    read_into_threads(path, positions, velocities, 1)
1213}
1214
1215/// [`read_into`] with `threads` workers. Each worker reads a page-aligned span.
1216/// A span shorter than one page stays on one worker.
1217pub fn read_into_threads(
1218    path: &Path,
1219    positions: &mut [f64],
1220    velocities: Option<&mut [f64]>,
1221    threads: usize,
1222) -> Result<ConBinHeader, ParseError> {
1223    read_columns_into(path, positions, velocities, threads, false)
1224}
1225
1226/// Copy both binary64 columns into owned vectors.
1227///
1228/// Each vector is `with_capacity`, then `pread` fills `spare_capacity_mut`
1229/// and `set_len` publishes those slots. The byte view is not a `&mut [f64]`
1230/// over uninitialised memory. A missing velocity run is an error.
1231pub fn read_owned_f64(
1232    path: &Path,
1233    threads: usize,
1234) -> Result<(ConBinHeader, Vec<f64>, Vec<f64>), ParseError> {
1235    let header = read_header(path)?;
1236    if header.binary32 {
1237        return Err(ParseError::ValidationError(
1238            "CON-bin file is binary32; use read_into_f32".into(),
1239        ));
1240    }
1241    if !header.velocities {
1242        return Err(ParseError::ValidationError(
1243            "CON-bin file has no velocity run".into(),
1244        ));
1245    }
1246    let n = header.position_len()?;
1247    let mut positions = Vec::<f64>::with_capacity(n);
1248    let mut velocities = Vec::<f64>::with_capacity(n);
1249    fill_owned_f64(path, &mut positions, &mut velocities, n, threads)?;
1250    Ok((header, positions, velocities))
1251}
1252
1253#[cfg(unix)]
1254fn spare_prefix_uninit(buf: &mut Vec<f64>, n: usize) -> &mut [std::mem::MaybeUninit<u8>] {
1255    let spare = buf.spare_capacity_mut();
1256    assert!(spare.len() >= n);
1257    let nbytes = n * std::mem::size_of::<f64>();
1258    let ptr = spare.as_mut_ptr().cast::<std::mem::MaybeUninit<u8>>();
1259    // SAFETY: `n` uninitialised f64 slots cover `nbytes` bytes. A
1260    // `MaybeUninit<u8>` slot may be uninitialised, so this slice does not
1261    // claim the bytes are initialised. `pread` writes through the raw
1262    // pointer, then `set_len` publishes the f64 slots.
1263    unsafe { std::slice::from_raw_parts_mut(ptr, nbytes) }
1264}
1265
1266fn fill_owned_f64(
1267    path: &Path,
1268    positions: &mut Vec<f64>,
1269    velocities: &mut Vec<f64>,
1270    n: usize,
1271    threads: usize,
1272) -> Result<(), ParseError> {
1273    #[cfg(not(unix))]
1274    {
1275        let _ = (path, positions, velocities, n, threads);
1276        return Err(ParseError::ValidationError(
1277            "CON-bin pread fill requires Unix".into(),
1278        ));
1279    }
1280    #[cfg(unix)]
1281    {
1282        let file = File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
1283        let mut raw = [0u8; FIELD_BYTES];
1284        (&file)
1285            .read_exact(&mut raw)
1286            .map_err(|err| ParseError::ValidationError(err.to_string()))?;
1287        let (_header, offset) = parse_fields(&raw)?;
1288        let width = std::mem::size_of::<f64>();
1289        let expect = n.checked_mul(width).ok_or_else(|| {
1290            ParseError::InvalidCount("CON-bin column byte length overflows".into())
1291        })?;
1292        let page = page_size()?;
1293        let file_len = file
1294            .metadata()
1295            .map_err(|err| ParseError::ValidationError(err.to_string()))?
1296            .len() as usize;
1297        let pos_off = checked_payload_offset(offset, file_len)? as u64;
1298        let fd = file.as_raw_fd();
1299        {
1300            let bytes = spare_prefix_uninit(positions, n);
1301            pread_parallel_uninit(fd, bytes, pos_off, threads, page)?;
1302        }
1303        // SAFETY: `pread_parallel_uninit` wrote `n` little-endian f64 values or returned.
1304        unsafe { positions.set_len(n) };
1305        {
1306            let bytes = spare_prefix_uninit(velocities, n);
1307            pread_parallel_uninit(fd, bytes, pos_off + expect as u64, threads, page)?;
1308        }
1309        unsafe { velocities.set_len(n) };
1310        Ok(())
1311    }
1312}
1313
1314/// Copy a binary32 CON-bin into caller-owned columns. One thread issues the reads.
1315pub fn read_into_f32(
1316    path: &Path,
1317    positions: &mut [f32],
1318    velocities: Option<&mut [f32]>,
1319) -> Result<ConBinHeader, ParseError> {
1320    read_into_f32_threads(path, positions, velocities, 1)
1321}
1322
1323/// [`read_into_f32`] with `threads` workers. The split is the same page rule
1324/// as [`read_into_threads`].
1325pub fn read_into_f32_threads(
1326    path: &Path,
1327    positions: &mut [f32],
1328    velocities: Option<&mut [f32]>,
1329    threads: usize,
1330) -> Result<ConBinHeader, ParseError> {
1331    read_columns_into(path, positions, velocities, threads, true)
1332}
1333
1334fn read_columns_into<T>(
1335    path: &Path,
1336    positions: &mut [T],
1337    velocities: Option<&mut [T]>,
1338    threads: usize,
1339    binary32: bool,
1340) -> Result<ConBinHeader, ParseError> {
1341    let file = File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
1342    let mut raw = [0u8; FIELD_BYTES];
1343    (&file)
1344        .read_exact(&mut raw)
1345        .map_err(|err| ParseError::ValidationError(err.to_string()))?;
1346    let (header, offset) = parse_fields(&raw)?;
1347    if header.binary32 != binary32 {
1348        return Err(ParseError::ValidationError(if binary32 {
1349            "CON-bin file is binary64; use read_into".into()
1350        } else {
1351            "CON-bin file is binary32; use read_into_f32".into()
1352        }));
1353    }
1354    let n = header.position_len()?;
1355    let width = std::mem::size_of::<T>();
1356    let expect = n
1357        .checked_mul(width)
1358        .ok_or_else(|| ParseError::InvalidCount("CON-bin column byte length overflows".into()))?;
1359    if positions.len() != n {
1360        return Err(ParseError::InvalidCount(format!(
1361            "position slice has {} values, the file has {n}",
1362            positions.len()
1363        )));
1364    }
1365    let page = page_size()?;
1366    let file_len = file
1367        .metadata()
1368        .map_err(|err| ParseError::ValidationError(err.to_string()))?
1369        .len() as usize;
1370    let pos_off = checked_payload_offset(offset, file_len)? as u64;
1371    let pos_bytes = bytes_of(positions);
1372    if pos_bytes.len() != expect {
1373        return Err(ParseError::InvalidCount(format!(
1374            "position slice has {} bytes, the file has {expect}",
1375            pos_bytes.len()
1376        )));
1377    }
1378    #[cfg(not(unix))]
1379    {
1380        let _ = (pos_bytes, pos_off, threads, page, velocities, header);
1381        return Err(ParseError::ValidationError(
1382            "CON-bin pread fill requires Unix".into(),
1383        ));
1384    }
1385    #[cfg(unix)]
1386    {
1387        let fd = file.as_raw_fd();
1388        // Sequential advice matches one pass over the payload. WILLNEED is not
1389        // used here: it starts a second readahead beside the reads themselves.
1390        // libc exposes posix_fadvise on Linux. macOS has posix_madvise instead.
1391        #[cfg(target_os = "linux")]
1392        unsafe {
1393            libc::posix_fadvise(fd, 0, 0, libc::POSIX_FADV_SEQUENTIAL);
1394        }
1395        // Linux read returns at most MAX_RW_COUNT bytes (INT_MAX aligned down to
1396        // the page, include/linux/fs.h). `pread_all` continues until the span is full.
1397        pread_parallel(fd, pos_bytes, pos_off, threads, page)?;
1398        if header.velocities {
1399            let Some(vel) = velocities else {
1400                return Err(ParseError::InvalidCount(
1401                    "CON-bin has velocities and the caller passed no velocity slice".into(),
1402                ));
1403            };
1404            if vel.len() != n {
1405                return Err(ParseError::InvalidCount(format!(
1406                    "velocity slice has {} values, the file has {n}",
1407                    vel.len()
1408                )));
1409            }
1410            let vel_bytes = bytes_of(vel);
1411            pread_parallel(fd, vel_bytes, pos_off + expect as u64, threads, page)?;
1412        }
1413        Ok(header)
1414    }
1415}
1416
1417#[cfg(unix)]
1418fn pread_parallel(
1419    fd: i32,
1420    dest: &mut [u8],
1421    offset: u64,
1422    threads: usize,
1423    page: usize,
1424) -> Result<(), ParseError> {
1425    let threads = threads.max(1);
1426    let workers = if dest.len() < page.saturating_mul(threads) {
1427        1
1428    } else {
1429        threads
1430    };
1431    if workers == 1 {
1432        return pread_all(fd, dest, offset);
1433    }
1434    let chunk = (dest.len() / workers) & !(page - 1);
1435    std::thread::scope(|scope| {
1436        let mut rest = dest;
1437        let mut off = offset;
1438        let mut handles = Vec::with_capacity(workers);
1439        for worker in 0..workers {
1440            let take = if worker + 1 == workers {
1441                rest.len()
1442            } else {
1443                chunk
1444            };
1445            let (head, tail) = rest.split_at_mut(take);
1446            rest = tail;
1447            let this = off;
1448            off += take as u64;
1449            handles.push(scope.spawn(move || pread_all(fd, head, this)));
1450        }
1451        handles
1452            .into_iter()
1453            .try_for_each(|handle| handle.join().unwrap_or(Err(ParseError::IncompleteFrame)))
1454    })
1455}
1456
1457#[cfg(unix)]
1458fn pread_raw(fd: i32, ptr: *mut u8, len: usize, mut offset: u64) -> Result<(), ParseError> {
1459    let mut done = 0;
1460    while done < len {
1461        let n = unsafe { libc::pread(fd, ptr.add(done).cast(), len - done, offset as libc::off_t) };
1462        if n < 0 {
1463            let err = std::io::Error::last_os_error();
1464            // POSIX: EINTR means the call was interrupted before a transfer.
1465            // Restart the same offset. Rust's `read_at` does the same for
1466            // `ErrorKind::Interrupted`.
1467            if err.raw_os_error() == Some(libc::EINTR) {
1468                continue;
1469            }
1470            return Err(ParseError::ValidationError(err.to_string()));
1471        }
1472        if n == 0 {
1473            return Err(ParseError::IncompleteFrame);
1474        }
1475        done += n as usize;
1476        offset += n as u64;
1477    }
1478    Ok(())
1479}
1480
1481#[cfg(unix)]
1482fn pread_all(fd: i32, dest: &mut [u8], offset: u64) -> Result<(), ParseError> {
1483    pread_raw(fd, dest.as_mut_ptr(), dest.len(), offset)
1484}
1485
1486#[cfg(unix)]
1487fn pread_uninit(
1488    fd: i32,
1489    dest: &mut [std::mem::MaybeUninit<u8>],
1490    offset: u64,
1491) -> Result<(), ParseError> {
1492    // `as_mut_ptr` of `MaybeUninit<u8>` is the address `pread` may write.
1493    // The slice type does not require the bytes to be initialised.
1494    pread_raw(fd, dest.as_mut_ptr().cast::<u8>(), dest.len(), offset)
1495}
1496
1497#[cfg(unix)]
1498fn pread_parallel_uninit(
1499    fd: i32,
1500    dest: &mut [std::mem::MaybeUninit<u8>],
1501    offset: u64,
1502    threads: usize,
1503    page: usize,
1504) -> Result<(), ParseError> {
1505    let threads = threads.max(1);
1506    let workers = if dest.len() < page.saturating_mul(threads) {
1507        1
1508    } else {
1509        threads
1510    };
1511    if workers == 1 {
1512        return pread_uninit(fd, dest, offset);
1513    }
1514    let chunk = (dest.len() / workers) & !(page - 1);
1515    std::thread::scope(|scope| {
1516        let mut rest = dest;
1517        let mut off = offset;
1518        let mut handles = Vec::with_capacity(workers);
1519        for worker in 0..workers {
1520            let take = if worker + 1 == workers {
1521                rest.len()
1522            } else {
1523                chunk
1524            };
1525            let (head, tail) = rest.split_at_mut(take);
1526            rest = tail;
1527            let this = off;
1528            off += take as u64;
1529            handles.push(scope.spawn(move || pread_uninit(fd, head, this)));
1530        }
1531        handles
1532            .into_iter()
1533            .try_for_each(|handle| handle.join().unwrap_or(Err(ParseError::IncompleteFrame)))
1534    })
1535}
1536
1537fn write_values<T>(file: &mut File, values: &[T]) -> Result<(), ParseError> {
1538    let bytes = unsafe {
1539        std::slice::from_raw_parts(values.as_ptr().cast::<u8>(), std::mem::size_of_val(values))
1540    };
1541    file.write_all(bytes)
1542        .map_err(|err| ParseError::ValidationError(err.to_string()))
1543}
1544
1545fn write_header_page(
1546    file: &mut File,
1547    header: &ConBinHeader,
1548    page: usize,
1549) -> Result<(), ParseError> {
1550    let mut header_page = vec![0u8; page];
1551    let prefix = header_fields(header, page as u64);
1552    header_page[..FIELD_BYTES].copy_from_slice(&prefix);
1553    file.write_all(&header_page)
1554        .map_err(|err| ParseError::ValidationError(err.to_string()))
1555}
1556
1557/// Stream a CON text file into a CON-bin of the same frames.
1558///
1559/// The header occupies one host page, so the payload starts on a page
1560/// boundary. Positions of every frame are written as one run, then velocities.
1561pub fn write_from_con(
1562    con_path: &Path,
1563    out_path: &Path,
1564    binary32: bool,
1565) -> Result<ConBinHeader, ParseError> {
1566    let text = std::fs::read_to_string(con_path)
1567        .map_err(|err| ParseError::ValidationError(err.to_string()))?;
1568    let mut file =
1569        File::create(out_path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
1570    let page = page_size()?;
1571    write_header_page(
1572        &mut file,
1573        &ConBinHeader {
1574            n_frames: 0,
1575            n_atoms: 0,
1576            binary32,
1577            velocities: true,
1578        },
1579        page,
1580    )?;
1581
1582    let mut n_frames = 0u64;
1583    let mut n_atoms = 0u64;
1584    let mut positions: Vec<f64> = Vec::new();
1585    let mut velocities: Vec<f64> = Vec::new();
1586    for frame in ConFrameIterator::new(&text) {
1587        let frame = frame?;
1588        let n = frame.atom_data.len();
1589        if n_frames == 0 {
1590            n_atoms = n as u64;
1591        } else if n as u64 != n_atoms {
1592            return Err(ParseError::InvalidCount(
1593                "CON-bin requires the same atom count in every frame".into(),
1594            ));
1595        }
1596        for atom in &frame.atom_data {
1597            positions.extend_from_slice(&[atom.x, atom.y, atom.z]);
1598            velocities.extend_from_slice(&atom.velocity().unwrap_or([0.0; 3]));
1599        }
1600        n_frames += 1;
1601    }
1602    if binary32 {
1603        let pos32: Vec<f32> = positions.iter().map(|v| *v as f32).collect();
1604        let vel32: Vec<f32> = velocities.iter().map(|v| *v as f32).collect();
1605        write_values(&mut file, &pos32)?;
1606        write_values(&mut file, &vel32)?;
1607    } else {
1608        write_values(&mut file, &positions)?;
1609        write_values(&mut file, &velocities)?;
1610    }
1611    let header = ConBinHeader {
1612        n_frames,
1613        n_atoms,
1614        binary32,
1615        velocities: true,
1616    };
1617    file.seek(SeekFrom::Start(0))
1618        .map_err(|err| ParseError::ValidationError(err.to_string()))?;
1619    write_header_page(&mut file, &header, page)?;
1620    Ok(header)
1621}
1622
1623#[cfg(test)]
1624mod tests {
1625    use super::*;
1626
1627    fn write_known(path: &Path, frames: usize, atoms: usize) {
1628        let page = page_size().unwrap();
1629        let mut file = File::create(path).unwrap();
1630        let header = ConBinHeader {
1631            n_frames: frames as u64,
1632            n_atoms: atoms as u64,
1633            binary32: false,
1634            velocities: true,
1635        };
1636        write_header_page(&mut file, &header, page).unwrap();
1637        let n = frames * atoms * 3;
1638        let pos: Vec<f64> = (0..n).map(|i| i as f64 * 0.25).collect();
1639        let vel: Vec<f64> = (0..n).map(|i| i as f64 * -0.5).collect();
1640        write_values(&mut file, &pos).unwrap();
1641        write_values(&mut file, &vel).unwrap();
1642    }
1643
1644    #[cfg(target_os = "linux")]
1645    #[test]
1646    fn worker_span_shares_the_queue() {
1647        let limits = QueueLimits {
1648            page: 4096,
1649            logical_block: 512,
1650            max_request: 1280 * 1024,
1651            nr_requests: 256,
1652            read_ahead: 128 * 1024,
1653            nr_hw_queues: 1,
1654            rotational: true,
1655        };
1656        assert_eq!(limits.worker_span(8), 32 * 1280 * 1024);
1657        assert_eq!(limits.worker_span(1), 256 * 1280 * 1024);
1658        // /dev/shm is tmpfs, so it has no block queue. The sum must still run.
1659        let dir = std::path::PathBuf::from(format!("/dev/shm/conbin-sum-{}", std::process::id()));
1660        let _ = std::fs::remove_dir_all(&dir);
1661        std::fs::create_dir_all(&dir).unwrap();
1662        let path = dir.join("known.conbin");
1663        write_known(&path, 4, 8);
1664        let mapped = map_file(&path, 1).unwrap();
1665        let expect = mapped.positions_f64().unwrap().iter().sum::<f64>()
1666            + mapped.velocities_f64().unwrap().iter().sum::<f64>();
1667        let got = sum_mapped(&path, 2).unwrap();
1668        let got_split = sum_split_streams(&path, 2).unwrap();
1669        assert_eq!(got, expect, "parallel sum {got} serial sum {expect}");
1670        assert_eq!(
1671            got_split, expect,
1672            "split sum {got_split} serial sum {expect}"
1673        );
1674        let _ = std::fs::remove_dir_all(&dir);
1675        let short = QueueLimits {
1676            nr_requests: 2,
1677            read_ahead: 4096,
1678            ..limits
1679        };
1680        assert_eq!(short.worker_span(8), 1280 * 1024);
1681        let shm = std::path::PathBuf::from(format!("/dev/shm/conbin-queue-{}", std::process::id()));
1682        let _ = std::fs::remove_dir_all(&shm);
1683        std::fs::create_dir_all(&shm).unwrap();
1684        let qpath = shm.join("known.conbin");
1685        write_known(&qpath, 1, 1);
1686        let got = QueueLimits::of(&qpath).unwrap();
1687        assert!(got.page.is_power_of_two());
1688        assert_eq!(got.max_request, got.page);
1689        assert_eq!(got.nr_requests, 1);
1690        assert!(!got.rotational);
1691        let _ = std::fs::remove_dir_all(&shm);
1692    }
1693
1694    #[test]
1695    fn mapped_runs_match_the_pread_fill() {
1696        let dir = std::env::temp_dir().join(format!("conbin-map-{}", std::process::id()));
1697        let _ = std::fs::remove_dir_all(&dir);
1698        std::fs::create_dir_all(&dir).unwrap();
1699        let path = dir.join("known.conbin");
1700        write_known(&path, 4, 3);
1701        let mapped = map_file(&path, 2).unwrap();
1702        let pos = mapped.positions_f64().unwrap();
1703        let vel = mapped.velocities_f64().unwrap();
1704        assert_eq!(pos.len(), 4 * 3 * 3);
1705        assert_eq!(pos[0], 0.0);
1706        assert_eq!(pos[1], 0.25);
1707        assert_eq!(vel[1], -0.5);
1708        let mut got_pos = vec![0.0; pos.len()];
1709        let mut got_vel = vec![0.0; vel.len()];
1710        read_into_threads(&path, &mut got_pos, Some(&mut got_vel), 2).unwrap();
1711        assert_eq!(got_pos, pos);
1712        assert_eq!(got_vel, vel);
1713        let _ = std::fs::remove_dir_all(&dir);
1714    }
1715
1716    #[test]
1717    fn write_from_con_round_trips_text() {
1718        let dir = std::env::temp_dir().join(format!("conbin-text-{}", std::process::id()));
1719        let _ = std::fs::remove_dir_all(&dir);
1720        std::fs::create_dir_all(&dir).unwrap();
1721        let con = dir.join("two.con");
1722        std::fs::write(
1723            &con,
1724            include_str!("../resources/test/tiny_cuh2.con").repeat(2),
1725        )
1726        .unwrap();
1727        let bin = dir.join("two.conbin");
1728        let header = write_from_con(&con, &bin, false).unwrap();
1729        assert_eq!(header.n_frames, 2);
1730        assert_eq!(header.n_atoms, 4);
1731        let mapped = map_file(&bin, 1).unwrap();
1732        let pos = mapped.positions_f64().unwrap();
1733        assert_eq!(pos.len(), 2 * 4 * 3);
1734        assert!(pos[0].is_finite());
1735        let _ = std::fs::remove_dir_all(&dir);
1736    }
1737
1738    #[test]
1739    fn a_store_copies_the_page_and_leaves_the_file() {
1740        let dir = std::env::temp_dir().join(format!("conbin-cow-{}", std::process::id()));
1741        let _ = std::fs::remove_dir_all(&dir);
1742        std::fs::create_dir_all(&dir).unwrap();
1743        let path = dir.join("known.conbin");
1744        write_known(&path, 2, 4);
1745        let before = std::fs::read(&path).unwrap();
1746        let mut mapped = map_file(&path, 1).unwrap();
1747        {
1748            let pos = mapped.positions_f64_mut().unwrap();
1749            let original = pos[0];
1750            pos[0] = original + 1.0;
1751            assert_eq!(pos[0], original + 1.0);
1752        }
1753        let after = std::fs::read(&path).unwrap();
1754        assert_eq!(before, after);
1755        let reread = map_file(&path, 1).unwrap();
1756        assert_eq!(reread.positions_f64().unwrap()[0], 0.0);
1757        let _ = std::fs::remove_dir_all(&dir);
1758    }
1759
1760    #[test]
1761    fn a_4k_offset_opens_on_a_16k_or_64k_host() {
1762        // The stored offset is 4096, and the file is shorter than a 16 KiB
1763        // or 64 KiB page. The reader keeps 4096. It does not replace that
1764        // offset with the host page.
1765        let header = ConBinHeader {
1766            n_frames: 1,
1767            n_atoms: 1,
1768            binary32: false,
1769            velocities: true,
1770        };
1771        let mut bytes = header_fields(&header, 4096).to_vec();
1772        bytes.resize(4096 + 48, 0);
1773        bytes[4096..4096 + 8].copy_from_slice(&1.25f64.to_le_bytes());
1774        bytes[4096 + 24..4096 + 32].copy_from_slice(&(-2.5f64).to_le_bytes());
1775        assert!(bytes.len() < 16 * 1024);
1776        assert!(bytes.len() < 64 * 1024);
1777        let off = checked_payload_offset(4096, bytes.len()).unwrap();
1778        assert_eq!(off, 4096);
1779        assert_ne!(off, 16 * 1024);
1780        assert_ne!(off, 64 * 1024);
1781        let dir = std::env::temp_dir().join(format!("conbin-off-{}", std::process::id()));
1782        let _ = std::fs::remove_dir_all(&dir);
1783        std::fs::create_dir_all(&dir).unwrap();
1784        let path = dir.join("known.conbin");
1785        std::fs::write(&path, &bytes).unwrap();
1786        let mapped = map_file(&path, 1).unwrap();
1787        assert_eq!(mapped.payload_off, 4096);
1788        assert_eq!(mapped.positions_f64().unwrap()[0], 1.25);
1789        assert_eq!(mapped.velocities_f64().unwrap()[0], -2.5);
1790        let _ = std::fs::remove_dir_all(&dir);
1791    }
1792
1793    fn write_typed(path: &Path, binary32: bool, velocities: bool) {
1794        let header = ConBinHeader {
1795            n_frames: 1,
1796            n_atoms: 1,
1797            binary32,
1798            velocities,
1799        };
1800        let mut bytes = header_fields(&header, 64).to_vec();
1801        bytes.resize(64 + 48, 0);
1802        std::fs::write(path, bytes).unwrap();
1803    }
1804
1805    #[test]
1806    fn sum_mapped_rejects_binary32_and_a_missing_velocity_run() {
1807        let dir = std::env::temp_dir().join(format!("conbin-kind-{}", std::process::id()));
1808        let _ = std::fs::remove_dir_all(&dir);
1809        std::fs::create_dir_all(&dir).unwrap();
1810        let binary32 = dir.join("f32.conbin");
1811        write_typed(&binary32, true, true);
1812        let err = sum_mapped(&binary32, 1).unwrap_err();
1813        assert!(err.to_string().contains("binary32"), "{err}");
1814        let bare = dir.join("nov.conbin");
1815        write_typed(&bare, false, false);
1816        let err = sum_mapped(&bare, 1).unwrap_err();
1817        assert!(err.to_string().contains("no velocity"), "{err}");
1818        let err = read_owned_f64(&bare, 1).unwrap_err();
1819        assert!(err.to_string().contains("no velocity"), "{err}");
1820        let known = dir.join("known.conbin");
1821        write_known(&known, 1, 2);
1822        let (_header, pos, vel) = read_owned_f64(&known, 1).unwrap();
1823        let mapped = map_file(&known, 1).unwrap();
1824        assert_eq!(pos.as_slice(), mapped.positions_f64().unwrap());
1825        assert_eq!(vel.as_slice(), mapped.velocities_f64().unwrap());
1826        let _ = std::fs::remove_dir_all(&dir);
1827    }
1828
1829    #[cfg(target_os = "linux")]
1830    fn dir_is_tmpfs(path: &Path) -> bool {
1831        let text = path.to_string_lossy();
1832        let Ok(c) = std::ffi::CString::new(text.as_bytes()) else {
1833            return false;
1834        };
1835        let mut stat: libc::statfs = unsafe { std::mem::zeroed() };
1836        let rc = unsafe { libc::statfs(c.as_ptr(), &mut stat) };
1837        // include/uapi/linux/magic.h TMPFS_MAGIC
1838        rc == 0 && stat.f_type == 0x01021994
1839    }
1840
1841    #[cfg(target_os = "linux")]
1842    fn streaming_dir() -> Option<std::path::PathBuf> {
1843        let mut candidates = vec![std::env::temp_dir()];
1844        if let Ok(cwd) = std::env::current_dir() {
1845            candidates.push(cwd);
1846        }
1847        candidates.push(std::path::PathBuf::from("/var/tmp"));
1848        for base in candidates {
1849            if base.is_dir() && !dir_is_tmpfs(&base) {
1850                let dir = base.join(format!("conbin-stream-{}", std::process::id()));
1851                let _ = std::fs::remove_dir_all(&dir);
1852                if std::fs::create_dir_all(&dir).is_ok() {
1853                    return Some(dir);
1854                }
1855            }
1856        }
1857        None
1858    }
1859
1860    #[cfg(target_os = "linux")]
1861    #[test]
1862    fn sum_mapped_streams_when_pages_are_not_resident() {
1863        let Some(dir) = streaming_dir() else {
1864            panic!(
1865                "sum_mapped_streams_when_pages_are_not_resident: temp, the working directory and /var/tmp are tmpfs, and POSIX_FADV_DONTNEED leaves tmpfs pages resident"
1866            );
1867        };
1868        let _ = std::fs::remove_dir_all(&dir);
1869        std::fs::create_dir_all(&dir).unwrap();
1870        let path = dir.join("known.conbin");
1871        write_known(&path, 2, 32);
1872        let file = File::open(&path).unwrap();
1873        file.sync_all().unwrap();
1874        let rc = unsafe { libc::posix_fadvise(file.as_raw_fd(), 0, 0, libc::POSIX_FADV_DONTNEED) };
1875        assert_eq!(rc, 0);
1876        let len = file.metadata().unwrap().len() as usize;
1877        let ptr = unsafe {
1878            libc::mmap(
1879                std::ptr::null_mut(),
1880                len,
1881                libc::PROT_READ,
1882                libc::MAP_PRIVATE,
1883                file.as_raw_fd(),
1884                0,
1885            )
1886        };
1887        assert_ne!(ptr, libc::MAP_FAILED);
1888        let page = page_size().unwrap();
1889        let pages = len.div_ceil(page);
1890        let mut vec = vec![0u8; pages];
1891        let rc = unsafe { libc::mincore(ptr, len, vec.as_mut_ptr()) };
1892        assert_eq!(rc, 0);
1893        assert!(
1894            vec.iter().any(|byte| byte & 1 == 0),
1895            "every page stayed resident"
1896        );
1897        unsafe { libc::munmap(ptr, len) };
1898        let got = sum_mapped(&path, 2).unwrap();
1899        let mapped = map_file(&path, 1).unwrap();
1900        let expect = mapped.positions_f64().unwrap().iter().sum::<f64>()
1901            + mapped.velocities_f64().unwrap().iter().sum::<f64>();
1902        assert_eq!(got, expect, "{got} {expect}");
1903        let _ = std::fs::remove_dir_all(&dir);
1904    }
1905
1906    /// Fills both columns with `pread` into spare capacity.
1907    /// Miri runs this with `-Zmiri-disable-isolation`.
1908    /// Miri does not model file-backed mmap, `mincore`, `readahead`
1909    /// or `MADV_POPULATE_READ`.
1910    #[cfg(unix)]
1911    #[test]
1912    fn read_owned_f64_fills_spare_capacity_by_pread() {
1913        let dir = tempfile::tempdir().unwrap();
1914        let path = dir.path().join("known.conbin");
1915        write_known(&path, 2, 8);
1916        for threads in [1usize, 2] {
1917            let (header, pos, vel) = read_owned_f64(&path, threads).unwrap();
1918            let n = header.position_len().unwrap();
1919            assert_eq!(pos.len(), n);
1920            assert_eq!(vel.len(), n);
1921            let mut raw = Vec::new();
1922            std::io::Read::read_to_end(&mut File::open(&path).unwrap(), &mut raw).unwrap();
1923            let off = raw.len() - 2 * n * 8;
1924            let want: Vec<f64> = raw[off..]
1925                .as_chunks::<8>()
1926                .0
1927                .iter()
1928                .map(|chunk| f64::from_le_bytes(*chunk))
1929                .collect();
1930            assert_eq!(pos.as_slice(), &want[..n]);
1931            assert_eq!(vel.as_slice(), &want[n..]);
1932        }
1933    }
1934
1935    #[cfg(target_os = "linux")]
1936    #[test]
1937    fn populate_followup_touches_only_after_einval() {
1938        assert!(populate_followup(libc::EINVAL).is_ok());
1939        for code in [libc::EFAULT, libc::EIO, libc::EHWPOISON, libc::ENOMEM] {
1940            assert_eq!(populate_followup(code).unwrap_err(), code);
1941        }
1942    }
1943
1944    #[cfg(target_os = "linux")]
1945    #[test]
1946    fn wait_resident_rejects_an_unaligned_address() {
1947        let page = page_size().unwrap();
1948        let ptr = unsafe {
1949            libc::mmap(
1950                std::ptr::null_mut(),
1951                page,
1952                libc::PROT_READ | libc::PROT_WRITE,
1953                libc::MAP_PRIVATE | libc::MAP_ANONYMOUS,
1954                -1,
1955                0,
1956            )
1957        };
1958        assert_ne!(ptr, libc::MAP_FAILED);
1959        let err = wait_resident(ptr as usize + 1, 0, 1, page).unwrap_err();
1960        assert!(err.contains("page-aligned"), "{err}");
1961        unsafe { libc::munmap(ptr, page) };
1962    }
1963
1964    #[cfg(target_os = "linux")]
1965    #[test]
1966    fn wait_resident_returns_an_unmapped_populate_error() {
1967        let page = page_size().unwrap();
1968        let ptr = unsafe {
1969            libc::mmap(
1970                std::ptr::null_mut(),
1971                page,
1972                libc::PROT_NONE,
1973                libc::MAP_PRIVATE | libc::MAP_ANONYMOUS,
1974                -1,
1975                0,
1976            )
1977        };
1978        assert_ne!(ptr, libc::MAP_FAILED);
1979        unsafe { libc::munmap(ptr, page) };
1980        let err = wait_resident(ptr as usize, 0, page, page).unwrap_err();
1981        assert!(err.contains("os error"), "{err}");
1982    }
1983}