1use std::fs::File;
11use std::io::{Read, Seek, SeekFrom, Write};
12#[cfg(unix)]
13use std::os::fd::AsRawFd;
14use std::path::Path;
15
16use crate::error::ParseError;
17use crate::iterators::ConFrameIterator;
18
19const MAGIC: &[u8; 8] = b"CONBIN01";
20const VERSION: u32 = 1;
21const FLAG_F32: u32 = 1;
22const FLAG_VELOCITY: u32 = 2;
23
24const FIELD_BYTES: usize = 40;
27
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
29pub struct ConBinHeader {
30 pub n_frames: u64,
31 pub n_atoms: u64,
32 pub binary32: bool,
33 pub velocities: bool,
34}
35
36impl ConBinHeader {
37 pub fn position_len(&self) -> Result<usize, ParseError> {
38 (self.n_frames as usize)
39 .checked_mul(self.n_atoms as usize)
40 .and_then(|n| n.checked_mul(3))
41 .ok_or_else(|| ParseError::InvalidCount("CON-bin column length overflows".into()))
42 }
43}
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47pub enum Prefetch {
48 Span,
50 ChunkPopulate,
52 ChunkWillneed,
55 Hybrid,
57}
58
59#[derive(Debug, Clone, Copy)]
73pub struct QueueLimits {
74 pub page: usize,
75 pub logical_block: usize,
76 pub max_request: usize,
77 pub nr_requests: usize,
78 pub read_ahead: usize,
79 pub nr_hw_queues: usize,
81 pub rotational: bool,
84}
85
86impl QueueLimits {
87 pub fn page_default(page: usize) -> Self {
93 Self {
94 page,
95 logical_block: page,
96 max_request: page,
97 nr_requests: 1,
98 read_ahead: page,
99 nr_hw_queues: 1,
100 rotational: false,
101 }
102 }
103
104 pub fn of(path: &Path) -> Result<Self, ParseError> {
105 let page = page_size()?;
106 #[cfg(target_os = "linux")]
107 {
108 let file =
109 File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
110 let dev = std::os::unix::fs::MetadataExt::dev(
111 &file
112 .metadata()
113 .map_err(|err| ParseError::ValidationError(err.to_string()))?,
114 );
115 Ok(Self::from_sysfs(dev, page))
116 }
117 #[cfg(not(target_os = "linux"))]
118 {
119 let _ = path;
120 Ok(Self::page_default(page))
121 }
122 }
123
124 #[cfg(target_os = "linux")]
125 fn from_sysfs(dev: u64, page: usize) -> Self {
126 let num =
127 |name: &str| queue_text(dev, name).and_then(|text| text.trim().parse::<usize>().ok());
128 let logical = num("logical_block_size").unwrap_or(page).max(1);
129 let max_request = num("max_sectors_kb")
130 .map(|kb| kb.saturating_mul(1024))
131 .unwrap_or(page)
132 .max(page);
133 let nr_requests = num("nr_requests").unwrap_or(1).max(1);
134 let read_ahead = num("read_ahead_kb")
135 .map(|kb| kb.saturating_mul(1024))
136 .unwrap_or(page)
137 .max(page);
138 let rotational = num("rotational").unwrap_or(0) != 0;
139 Self {
140 page,
141 logical_block: logical,
142 max_request,
143 nr_requests,
144 read_ahead,
145 nr_hw_queues: hw_queue_count(dev),
146 rotational,
147 }
148 }
149
150 pub fn worker_span(&self, workers: usize) -> usize {
158 let workers = workers.max(1);
159 let requests = (self.nr_requests / workers).max(1);
160 let queued = requests.saturating_mul(self.max_request);
161 let span = queued.max(self.read_ahead).max(self.page);
162 let aligned = span & !(self.page - 1);
163 aligned.max(self.page)
164 }
165}
166
167#[cfg(target_os = "linux")]
168pub(crate) fn dev_nums(dev: u64) -> (u64, u64) {
169 let major = ((dev >> 32) & 0xffff_f000) | ((dev >> 8) & 0xfff);
171 let minor = ((dev >> 12) & 0xffff_ff00) | (dev & 0xff);
172 (major, minor)
173}
174
175#[cfg(target_os = "linux")]
179fn hw_queue_count(dev: u64) -> usize {
180 if let Some(text) = queue_text(dev, "nr_hw_queues")
181 && let Ok(count) = text.trim().parse::<usize>()
182 {
183 return count.max(1);
184 }
185 let (major, minor) = dev_nums(dev);
186 let mut dir = std::path::PathBuf::from(format!("/sys/dev/block/{major}:{minor}"));
187 if dir.join("partition").is_file() {
188 dir.push("..");
189 }
190 let mq = dir.join("mq");
191 let count = std::fs::read_dir(&mq)
192 .ok()
193 .map(|entries| entries.filter_map(|entry| entry.ok()).count())
194 .unwrap_or(0);
195 count.max(1)
196}
197
198#[cfg(target_os = "linux")]
202fn queue_text(dev: u64, name: &str) -> Option<String> {
203 let (major, minor) = dev_nums(dev);
204 let dir = std::path::PathBuf::from(format!("/sys/dev/block/{major}:{minor}"));
205 let own = dir.join("queue").join(name);
206 if let Ok(text) = std::fs::read_to_string(&own) {
207 return Some(text);
208 }
209 if dir.join("partition").is_file() {
210 let parent = dir.join("..").join("queue").join(name);
211 if let Ok(text) = std::fs::read_to_string(&parent) {
212 return Some(text);
213 }
214 }
215 None
216}
217
218fn page_size() -> Result<usize, ParseError> {
220 #[cfg(unix)]
221 let page = {
222 let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
223 if page <= 0 {
224 return Err(ParseError::ValidationError(
225 "sysconf(SC_PAGESIZE) failed".into(),
226 ));
227 }
228 page as usize
229 };
230 #[cfg(windows)]
231 let page = {
232 #[link(name = "kernel32")]
236 unsafe extern "system" {
237 fn GetSystemInfo(info: *mut std::ffi::c_void);
238 }
239 let mut info = [0u8; 64];
240 unsafe { GetSystemInfo(info.as_mut_ptr().cast()) };
241 u32::from_ne_bytes(info[4..8].try_into().unwrap()) as usize
242 };
243 if !page.is_power_of_two() || page == 0 {
244 return Err(ParseError::ValidationError(
245 "the host page size is not a power of two".into(),
246 ));
247 }
248 Ok(page)
249}
250
251fn header_fields(header: &ConBinHeader, payload_off: u64) -> [u8; FIELD_BYTES] {
252 let mut out = [0u8; FIELD_BYTES];
253 out[..8].copy_from_slice(MAGIC);
254 out[8..12].copy_from_slice(&VERSION.to_le_bytes());
255 let mut flags = 0u32;
256 if header.binary32 {
257 flags |= FLAG_F32;
258 }
259 if header.velocities {
260 flags |= FLAG_VELOCITY;
261 }
262 out[12..16].copy_from_slice(&flags.to_le_bytes());
263 out[16..24].copy_from_slice(&header.n_frames.to_le_bytes());
264 out[24..32].copy_from_slice(&header.n_atoms.to_le_bytes());
265 out[32..40].copy_from_slice(&payload_off.to_le_bytes());
266 out
267}
268
269fn parse_fields(bytes: &[u8]) -> Result<(ConBinHeader, u64), ParseError> {
270 if bytes.len() < FIELD_BYTES || &bytes[..8] != MAGIC {
271 return Err(ParseError::ValidationError(
272 "CON-bin header is not CONBIN01".into(),
273 ));
274 }
275 let version = u32::from_le_bytes(bytes[8..12].try_into().unwrap());
276 if version != VERSION {
277 return Err(ParseError::ValidationError(format!(
278 "CON-bin version {version} is not {VERSION}"
279 )));
280 }
281 let flags = u32::from_le_bytes(bytes[12..16].try_into().unwrap());
282 let offset = u64::from_le_bytes(bytes[32..40].try_into().unwrap());
283 Ok((
284 ConBinHeader {
285 n_frames: u64::from_le_bytes(bytes[16..24].try_into().unwrap()),
286 n_atoms: u64::from_le_bytes(bytes[24..32].try_into().unwrap()),
287 binary32: flags & FLAG_F32 != 0,
288 velocities: flags & FLAG_VELOCITY != 0,
289 },
290 offset,
291 ))
292}
293
294fn checked_payload_offset(stored: u64, file_len: usize) -> Result<usize, ParseError> {
299 let off = usize::try_from(stored)
300 .map_err(|_| ParseError::ValidationError("CON-bin payload offset does not fit".into()))?;
301 if off < FIELD_BYTES || off > file_len || off % 8 != 0 {
302 return Err(ParseError::ValidationError(
303 "CON-bin payload offset is not an 8-byte aligned address inside the file".into(),
304 ));
305 }
306 Ok(off)
307}
308
309#[derive(Clone, Copy)]
310struct F64Payload {
311 offset: usize,
312 end: usize,
313}
314
315fn f64_payload(header_bytes: &[u8], file_len: usize) -> Result<F64Payload, ParseError> {
318 let (header, stored) = parse_fields(header_bytes)?;
319 if header.binary32 {
320 return Err(ParseError::ValidationError(
321 "CON-bin file is binary32; sum_mapped reads binary64".into(),
322 ));
323 }
324 if !header.velocities {
325 return Err(ParseError::ValidationError(
326 "CON-bin file has no velocity run".into(),
327 ));
328 }
329 let offset = checked_payload_offset(stored, file_len)?;
330 let values = header
331 .position_len()?
332 .checked_mul(2)
333 .ok_or_else(|| ParseError::InvalidCount("CON-bin column length overflows".into()))?;
334 let nbytes = values
335 .checked_mul(8)
336 .ok_or_else(|| ParseError::InvalidCount("CON-bin column byte length overflows".into()))?;
337 let end = offset
338 .checked_add(nbytes)
339 .ok_or_else(|| ParseError::InvalidCount("CON-bin payload offset overflows".into()))?;
340 if end > file_len {
341 return Err(ParseError::IncompleteFrame);
342 }
343 Ok(F64Payload { offset, end })
344}
345
346pub struct ConBinMap {
348 pub map: memmap2::MmapMut,
351 pub header: ConBinHeader,
352 pub payload_off: usize,
353}
354
355impl ConBinMap {
356 pub fn positions_f64(&self) -> Result<&[f64], ParseError> {
357 self.run(std::mem::size_of::<f64>(), 0)
358 }
359
360 pub fn positions_f64_mut(&mut self) -> Result<&mut [f64], ParseError> {
361 self.run_mut(std::mem::size_of::<f64>(), 0)
362 }
363
364 pub fn velocities_f64(&self) -> Result<&[f64], ParseError> {
365 let n = self.header.position_len()?;
366 self.run(std::mem::size_of::<f64>(), n)
367 }
368
369 pub fn positions_f32(&self) -> Result<&[f32], ParseError> {
370 self.run(std::mem::size_of::<f32>(), 0)
371 }
372
373 pub fn velocities_f32(&self) -> Result<&[f32], ParseError> {
374 let n = self.header.position_len()?;
375 self.run(std::mem::size_of::<f32>(), n)
376 }
377
378 fn run<T>(&self, width: usize, column: usize) -> Result<&[T], ParseError> {
379 let n = self.header.position_len()?;
380 let start = self
381 .payload_off
382 .checked_add(column.checked_mul(width).ok_or_else(|| {
383 ParseError::InvalidCount("CON-bin column byte length overflows".into())
384 })?)
385 .ok_or_else(|| ParseError::InvalidCount("CON-bin payload offset overflows".into()))?;
386 let nbytes = n.checked_mul(width).ok_or_else(|| {
387 ParseError::InvalidCount("CON-bin column byte length overflows".into())
388 })?;
389 let end = start.checked_add(nbytes).ok_or_else(|| {
390 ParseError::InvalidCount("CON-bin column extends past the file".into())
391 })?;
392 if end > self.map.len() {
393 return Err(ParseError::IncompleteFrame);
394 }
395 let bytes = &self.map[start..end];
396 if !cfg!(target_endian = "little") {
397 return Err(ParseError::ValidationError(
398 "CON-bin is little-endian; this host cannot view it in place".into(),
399 ));
400 }
401 if bytes.as_ptr().align_offset(std::mem::align_of::<T>()) != 0 {
402 return Err(ParseError::ValidationError(
403 "CON-bin payload is not aligned for its element type".into(),
404 ));
405 }
406 Ok(unsafe { std::slice::from_raw_parts(bytes.as_ptr().cast::<T>(), n) })
411 }
412
413 fn run_mut<T>(&mut self, width: usize, column: usize) -> Result<&mut [T], ParseError> {
414 let n = self.header.position_len()?;
415 let start = self
416 .payload_off
417 .checked_add(column.checked_mul(width).ok_or_else(|| {
418 ParseError::InvalidCount("CON-bin column byte length overflows".into())
419 })?)
420 .ok_or_else(|| ParseError::InvalidCount("CON-bin payload offset overflows".into()))?;
421 let nbytes = n.checked_mul(width).ok_or_else(|| {
422 ParseError::InvalidCount("CON-bin column byte length overflows".into())
423 })?;
424 let end = start.checked_add(nbytes).ok_or_else(|| {
425 ParseError::InvalidCount("CON-bin column extends past the file".into())
426 })?;
427 if end > self.map.len() {
428 return Err(ParseError::IncompleteFrame);
429 }
430 if !cfg!(target_endian = "little") {
431 return Err(ParseError::ValidationError(
432 "CON-bin is little-endian; this host cannot view it in place".into(),
433 ));
434 }
435 let ptr = self.map[start..end].as_mut_ptr();
436 if ptr.align_offset(std::mem::align_of::<T>()) != 0 {
437 return Err(ParseError::ValidationError(
438 "CON-bin payload is not aligned for its element type".into(),
439 ));
440 }
441 Ok(unsafe { std::slice::from_raw_parts_mut(ptr.cast::<T>(), n) })
444 }
445}
446
447pub fn sum_mapped(path: &Path, threads: usize) -> Result<f64, ParseError> {
461 #[cfg(target_os = "linux")]
462 {
463 let limits = QueueLimits::of(path)?;
464 sequential_resident_sum(path, threads.max(1), &limits)
465 }
466 #[cfg(not(target_os = "linux"))]
467 {
468 let _ = threads;
469 plain_sum(path)
470 }
471}
472
473#[cfg(not(target_os = "linux"))]
474fn plain_sum(path: &Path) -> Result<f64, ParseError> {
475 let file = File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
476 let map = unsafe { memmap2::Mmap::map(&file) }
477 .map_err(|err| ParseError::ValidationError(err.to_string()))?;
478 if map.len() < FIELD_BYTES {
479 return Err(ParseError::ValidationError(
480 "CON-bin header is not CONBIN01".into(),
481 ));
482 }
483 let payload = f64_payload(&map[..FIELD_BYTES], map.len())?;
484 if !cfg!(target_endian = "little") {
485 return Err(ParseError::ValidationError(
486 "CON-bin is little-endian; this host cannot view it in place".into(),
487 ));
488 }
489 let bytes = &map[payload.offset..payload.end];
490 let ptr = bytes.as_ptr();
491 if ptr.align_offset(std::mem::align_of::<f64>()) != 0 {
492 return Err(ParseError::ValidationError(
493 "CON-bin payload is not aligned for its element type".into(),
494 ));
495 }
496 let data = unsafe { std::slice::from_raw_parts(ptr.cast::<f64>(), bytes.len() / 8) };
497 Ok(data.iter().copied().sum())
498}
499
500#[cfg(target_os = "linux")]
501fn sequential_resident_sum(
502 path: &Path,
503 threads: usize,
504 limits: &QueueLimits,
505) -> Result<f64, ParseError> {
506 let owned = open_map(path)?;
507 let page = limits.page;
508 let len = owned.len;
509 let base = owned.ptr as usize;
510 let header =
511 unsafe { std::slice::from_raw_parts(owned.ptr.cast::<u8>(), FIELD_BYTES.min(len)) };
512 let payload = f64_payload(header, len)?;
513 let request = (limits.max_request & !(page - 1)).max(page);
514 let pieces = piece_list(0, len, request);
515 if pages_all_resident(owned.ptr, len, page)? {
516 return Ok(sum_pieces_in_order(
517 base,
518 payload.offset,
519 payload.end,
520 threads,
521 &pieces,
522 ));
523 }
524 let depth = limits.nr_requests.max(1);
526 let (tx, rx) = std::sync::mpsc::sync_channel::<usize>(depth);
527 let (err_tx, err_rx) = std::sync::mpsc::channel::<String>();
528 let rx = std::sync::Arc::new(std::sync::Mutex::new(rx));
529 let addr = owned.ptr as usize;
530 let found = std::thread::scope(|scope| {
531 let queued = pieces.clone();
532 scope.spawn(move || {
533 let fd = match File::open(path) {
534 Ok(fd) => fd,
535 Err(err) => {
536 let _ = err_tx.send(err.to_string());
537 return;
538 }
539 };
540 let mut submitted = 0usize;
544 while submitted < depth.min(queued.len()) {
545 if let Err(err) = advise_readahead(&fd, queued[submitted].0, queued[submitted].1) {
546 let _ = err_tx.send(err);
547 return;
548 }
549 submitted += 1;
550 }
551 for i in 0..queued.len() {
552 let (off, n) = queued[i];
553 if let Err(err) = wait_resident(addr, off, n, page) {
554 let _ = err_tx.send(err);
555 return;
556 }
557 let _ = tx.send(i);
558 if submitted < queued.len() {
559 if let Err(err) =
560 advise_readahead(&fd, queued[submitted].0, queued[submitted].1)
561 {
562 let _ = err_tx.send(err);
563 return;
564 }
565 submitted += 1;
566 }
567 }
568 });
569 let slots = std::sync::Arc::new(std::sync::Mutex::new(vec![0.0f64; pieces.len()]));
570 let mut handles = Vec::with_capacity(threads);
571 for _ in 0..threads {
572 let rx = std::sync::Arc::clone(&rx);
573 let slots = std::sync::Arc::clone(&slots);
574 let pieces = pieces.clone();
575 handles.push(scope.spawn(move || {
576 loop {
577 let msg = rx.lock().expect("piece queue").recv();
578 let Ok(index) = msg else { break };
579 let (off, n) = pieces[index];
580 let part = sum_range(base, off, off + n, payload.offset, payload.end);
581 slots.lock().expect("piece sums")[index] = part;
582 }
583 }));
584 }
585 for handle in handles {
586 handle.join().expect("column sum");
587 }
588 let slots = slots.lock().expect("piece sums");
589 slots.iter().copied().sum::<f64>()
590 });
591 if let Ok(err) = err_rx.try_recv() {
592 return Err(ParseError::ValidationError(err));
593 }
594 Ok(found)
595}
596
597#[cfg(target_os = "linux")]
607pub fn sum_split_streams(path: &Path, threads: usize) -> Result<f64, ParseError> {
608 let limits = QueueLimits::of(path)?;
609 split_stream_sum(path, threads.max(1), &limits)
610}
611
612#[cfg(not(target_os = "linux"))]
613pub fn sum_split_streams(path: &Path, threads: usize) -> Result<f64, ParseError> {
614 let _ = threads;
615 plain_sum(path)
616}
617
618#[cfg(target_os = "linux")]
619fn split_stream_sum(path: &Path, threads: usize, limits: &QueueLimits) -> Result<f64, ParseError> {
620 let owned = open_map(path)?;
621 let page = limits.page;
622 let len = owned.len;
623 let base = owned.ptr as usize;
624 let header =
625 unsafe { std::slice::from_raw_parts(owned.ptr.cast::<u8>(), FIELD_BYTES.min(len)) };
626 let payload = f64_payload(header, len)?;
627 let request = (limits.max_request & !(page - 1)).max(page);
628 let ranges = worker_ranges(len, threads, page);
629 if pages_all_resident(owned.ptr, len, page)? {
630 let pieces = piece_list(0, len, request);
631 return Ok(sum_pieces_in_order(
632 base,
633 payload.offset,
634 payload.end,
635 threads,
636 &pieces,
637 ));
638 }
639 let depth = (limits.nr_requests / ranges.len()).max(1);
640 let (err_tx, err_rx) = std::sync::mpsc::channel::<String>();
641 let sum = std::thread::scope(|scope| {
642 let mut handles = Vec::with_capacity(ranges.len());
643 for (span, end) in ranges {
644 let err_tx = err_tx.clone();
645 handles.push(scope.spawn(move || {
646 match stream_one_span(
647 path,
648 SpanGeom {
649 base,
650 span,
651 end,
652 page,
653 payload_off: payload.offset,
654 payload_end: payload.end,
655 request,
656 depth,
657 },
658 ) {
659 Ok(part) => part,
660 Err(err) => {
661 let _ = err_tx.send(err);
662 0.0
663 }
664 }
665 }));
666 }
667 drop(err_tx);
668 handles
669 .into_iter()
670 .map(|handle| handle.join().expect("split stream"))
671 .sum::<f64>()
672 });
673 if let Ok(err) = err_rx.try_recv() {
674 return Err(ParseError::ValidationError(err));
675 }
676 Ok(sum)
677}
678
679#[cfg(target_os = "linux")]
680struct SpanGeom {
681 base: usize,
682 span: usize,
683 end: usize,
684 page: usize,
685 payload_off: usize,
686 payload_end: usize,
687 request: usize,
688 depth: usize,
689}
690
691#[cfg(target_os = "linux")]
692fn stream_one_span(path: &Path, geom: SpanGeom) -> Result<f64, String> {
693 let SpanGeom {
694 base,
695 span,
696 end,
697 page,
698 payload_off,
699 payload_end,
700 request,
701 depth,
702 } = geom;
703 let fd = File::open(path).map_err(|err| err.to_string())?;
704 let pieces = piece_list(span, end, request);
705 let depth = depth.max(1);
706 let mut next_submit = 0usize;
707 let mut next_sum = 0usize;
708 let mut acc = 0.0f64;
709 while next_sum < pieces.len() {
710 while next_submit < pieces.len() && next_submit - next_sum < depth {
711 let (off, n) = pieces[next_submit];
712 advise_readahead(&fd, off, n)?;
713 next_submit += 1;
714 }
715 let (off, n) = pieces[next_sum];
716 wait_resident(base, off, n, page)?;
717 acc += sum_range(base, off, off + n, payload_off, payload_end);
718 next_sum += 1;
719 }
720 Ok(acc)
721}
722
723#[cfg(target_os = "linux")]
724fn sum_pieces_in_order(
725 base: usize,
726 payload_off: usize,
727 payload_end: usize,
728 threads: usize,
729 pieces: &[(usize, usize)],
730) -> f64 {
731 let mut slots = vec![0.0f64; pieces.len()];
732 let next = std::sync::atomic::AtomicUsize::new(0);
733 std::thread::scope(|scope| {
734 let mut handles = Vec::with_capacity(threads);
735 for _ in 0..threads {
736 handles.push(scope.spawn(|| {
737 let mut local = Vec::new();
738 loop {
739 let index = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
740 if index >= pieces.len() {
741 break;
742 }
743 let (off, n) = pieces[index];
744 local.push((
745 index,
746 sum_range(base, off, off + n, payload_off, payload_end),
747 ));
748 }
749 local
750 }));
751 }
752 for handle in handles {
753 for (index, part) in handle.join().expect("column sum") {
754 slots[index] = part;
755 }
756 }
757 });
758 slots.iter().copied().sum()
759}
760
761#[cfg(target_os = "linux")]
762fn pages_all_resident(ptr: *mut libc::c_void, len: usize, page: usize) -> Result<bool, ParseError> {
763 if len == 0 {
764 return Ok(true);
765 }
766 let pages = len.div_ceil(page);
767 let mut vec = vec![0u8; pages];
768 let rc = unsafe { libc::mincore(ptr, len, vec.as_mut_ptr()) };
769 if rc != 0 {
770 return Err(ParseError::ValidationError(
771 std::io::Error::last_os_error().to_string(),
772 ));
773 }
774 Ok(vec.iter().all(|byte| byte & 1 == 1))
775}
776
777#[cfg(target_os = "linux")]
782fn populate_followup(code: i32) -> Result<(), i32> {
783 if code == libc::EINVAL {
784 Ok(())
785 } else {
786 Err(code)
787 }
788}
789
790#[cfg(target_os = "linux")]
791fn touch_pages(addr: usize, off: usize, len: usize, page: usize) {
792 let pages = len.div_ceil(page);
793 for index in 0..pages {
794 let at = index * page;
795 if at >= len {
796 break;
797 }
798 let byte = (addr + off + at) as *const u8;
799 std::hint::black_box(unsafe { std::ptr::read_volatile(byte) });
800 }
801}
802
803#[cfg(target_os = "linux")]
804fn wait_resident(addr: usize, off: usize, len: usize, page: usize) -> Result<(), String> {
805 if page == 0 || !(addr + off).is_multiple_of(page) {
811 return Err("madvise address is not page-aligned".into());
812 }
813 let start = (addr + off) as *mut libc::c_void;
814 let pages = len.div_ceil(page).max(1);
815 let mut tried = 0usize;
816 loop {
817 let rc = unsafe { libc::madvise(start, len, libc::MADV_POPULATE_READ) };
818 if rc == 0 {
819 return Ok(());
820 }
821 let err = std::io::Error::last_os_error();
822 let code = err.raw_os_error().unwrap_or(0);
823 if code == libc::EAGAIN && tried < pages {
824 tried += 1;
825 continue;
826 }
827 if populate_followup(code).is_ok() {
828 touch_pages(addr, off, len, page);
829 return Ok(());
830 }
831 return Err(err.to_string());
832 }
833}
834
835#[cfg(target_os = "linux")]
836fn advise_readahead(file: &File, off: usize, len: usize) -> Result<(), String> {
837 let rc = unsafe { libc::readahead(file.as_raw_fd(), off as i64, len) };
838 if rc != 0 {
839 Err(std::io::Error::last_os_error().to_string())
840 } else {
841 Ok(())
842 }
843}
844
845#[cfg(target_os = "linux")]
846struct OwnedMap {
847 ptr: *mut libc::c_void,
848 len: usize,
849}
850
851#[cfg(target_os = "linux")]
852impl Drop for OwnedMap {
853 fn drop(&mut self) {
854 unsafe { libc::munmap(self.ptr, self.len) };
855 }
856}
857
858#[cfg(target_os = "linux")]
859fn open_map(path: &Path) -> Result<OwnedMap, ParseError> {
860 let file = File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
861 let len = file
862 .metadata()
863 .map_err(|err| ParseError::ValidationError(err.to_string()))?
864 .len() as usize;
865 let ptr = unsafe {
866 libc::mmap(
867 std::ptr::null_mut(),
868 len,
869 libc::PROT_READ,
870 libc::MAP_PRIVATE,
871 file.as_raw_fd(),
872 0,
873 )
874 };
875 if ptr == libc::MAP_FAILED {
876 return Err(ParseError::ValidationError(
877 std::io::Error::last_os_error().to_string(),
878 ));
879 }
880 unsafe {
881 libc::madvise(ptr, len, libc::MADV_SEQUENTIAL);
882 libc::madvise(ptr, len, libc::MADV_HUGEPAGE);
883 }
884 Ok(OwnedMap { ptr, len })
885}
886
887#[cfg(target_os = "linux")]
888fn sum_range(base: usize, start: usize, end: usize, payload_off: usize, payload_end: usize) -> f64 {
889 let from = start.max(payload_off);
890 let to = end.min(payload_end);
891 if to <= from {
892 return 0.0;
893 }
894 let data = unsafe { std::slice::from_raw_parts((base + from) as *const f64, (to - from) / 8) };
895 let mut acc = 0.0f64;
896 for &value in data {
897 acc += value;
898 }
899 acc
900}
901
902pub fn map_file(path: &Path, threads: usize) -> Result<ConBinMap, ParseError> {
923 map_file_with(path, threads, Prefetch::Span)
924}
925
926pub fn map_file_with(
928 path: &Path,
929 threads: usize,
930 prefetch: Prefetch,
931) -> Result<ConBinMap, ParseError> {
932 let file = File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
933 let map = unsafe { memmap2::MmapOptions::new().map_copy(&file) }
935 .map_err(|err| ParseError::ValidationError(err.to_string()))?;
936 if map.len() < FIELD_BYTES {
937 return Err(ParseError::ValidationError(
938 "CON-bin header is not CONBIN01".into(),
939 ));
940 }
941 let (header, offset) = parse_fields(&map[..FIELD_BYTES])?;
942 let page = page_size()?;
943 let payload_off = checked_payload_offset(offset, map.len())?;
944 fault_in(path, &map, threads, page, prefetch)?;
945 Ok(ConBinMap {
946 map,
947 header,
948 payload_off,
949 })
950}
951
952fn fault_in(
953 path: &Path,
954 map: &memmap2::MmapMut,
955 threads: usize,
956 page: usize,
957 prefetch: Prefetch,
958) -> Result<(), ParseError> {
959 #[cfg(unix)]
960 let _ = map.advise(memmap2::Advice::Sequential);
961 #[cfg(target_os = "linux")]
962 let _ = map.advise(memmap2::Advice::HugePage);
963 #[cfg(target_os = "linux")]
964 {
965 match prefetch {
966 Prefetch::Span => {
967 let _ = map.advise(memmap2::Advice::WillNeed);
968 advise_spans(map, memmap2::Advice::PopulateRead, threads, page)?;
969 }
970 Prefetch::ChunkPopulate => {
971 let window = QueueLimits::of(path)?.worker_span(threads);
972 chunk_each(map, threads, page, window, ChunkOp::Populate)?;
973 }
974 Prefetch::ChunkWillneed => {
975 let window = QueueLimits::of(path)?.worker_span(threads);
976 chunk_each(map, threads, page, window, ChunkOp::Willneed { path })?;
977 }
978 Prefetch::Hybrid => {
979 let window = QueueLimits::of(path)?.worker_span(threads);
980 chunk_each(map, threads, page, window, ChunkOp::Hybrid { path })?;
981 }
982 }
983 }
984 #[cfg(not(target_os = "linux"))]
985 {
986 let _ = (path, threads, prefetch);
987 for chunk in map.chunks(page) {
988 std::hint::black_box(chunk.first().copied());
989 }
990 }
991 Ok(())
992}
993
994#[cfg(target_os = "linux")]
995enum ChunkOp<'a> {
996 Populate,
997 Willneed { path: &'a Path },
998 Hybrid { path: &'a Path },
999}
1000
1001#[cfg(target_os = "linux")]
1002fn chunk_each(
1003 map: &memmap2::MmapMut,
1004 threads: usize,
1005 page: usize,
1006 window: usize,
1007 op: ChunkOp<'_>,
1008) -> Result<(), ParseError> {
1009 let ranges = worker_ranges(map.len(), threads, page);
1010 let shared = match op {
1011 ChunkOp::Populate => ChunkOp::Populate,
1012 ChunkOp::Willneed { path } => ChunkOp::Willneed { path },
1013 ChunkOp::Hybrid { path } => ChunkOp::Hybrid { path },
1014 };
1015 std::thread::scope(|scope| {
1016 let mut handles = Vec::with_capacity(ranges.len());
1017 for (start, end) in ranges {
1018 let op = match shared {
1019 ChunkOp::Populate => ChunkOp::Populate,
1020 ChunkOp::Willneed { path } => ChunkOp::Willneed { path },
1021 ChunkOp::Hybrid { path } => ChunkOp::Hybrid { path },
1022 };
1023 handles.push(scope.spawn(move || chunk_worker(map, start, end, window, page, op)));
1024 }
1025 handles.into_iter().try_for_each(|handle| {
1026 handle.join().unwrap_or_else(|_| {
1027 Err(ParseError::ValidationError(
1028 "CON-bin prefetch thread panicked".into(),
1029 ))
1030 })
1031 })
1032 })
1033}
1034
1035#[cfg(target_os = "linux")]
1036fn chunk_worker(
1037 map: &memmap2::MmapMut,
1038 start: usize,
1039 end: usize,
1040 window: usize,
1041 page: usize,
1042 op: ChunkOp<'_>,
1043) -> Result<(), ParseError> {
1044 let window = window.max(page);
1045 let pieces = piece_list(start, end, window);
1046 let fd = match op {
1047 ChunkOp::Willneed { path } | ChunkOp::Hybrid { path } => {
1048 Some(File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?)
1049 }
1050 ChunkOp::Populate => None,
1051 };
1052 match op {
1053 ChunkOp::Populate => {
1054 for (off, len) in pieces {
1055 populate_range(map, off, len)?;
1056 }
1057 }
1058 ChunkOp::Willneed { .. } => {
1059 let fd = fd.as_ref().expect("willneed opens a descriptor");
1060 for (off, len) in pieces {
1061 willneed(fd, off, len)?;
1062 }
1063 }
1064 ChunkOp::Hybrid { .. } => {
1065 let fd = fd.as_ref().expect("hybrid opens a descriptor");
1066 if let Some((off, len)) = pieces.first().copied() {
1067 willneed(fd, off, len)?;
1068 }
1069 for i in 0..pieces.len() {
1070 if let Some((off, len)) = pieces.get(i + 1).copied() {
1071 willneed(fd, off, len)?;
1072 }
1073 let (off, len) = pieces[i];
1074 populate_range(map, off, len)?;
1075 }
1076 }
1077 }
1078 Ok(())
1079}
1080
1081#[cfg(target_os = "linux")]
1082fn piece_list(start: usize, end: usize, window: usize) -> Vec<(usize, usize)> {
1083 let mut pieces = Vec::new();
1084 let mut off = start;
1085 while off < end {
1086 let len = window.min(end - off);
1087 pieces.push((off, len));
1088 off += len;
1089 }
1090 pieces
1091}
1092
1093#[cfg(target_os = "linux")]
1094fn populate_range(map: &memmap2::MmapMut, off: usize, len: usize) -> Result<(), ParseError> {
1095 let page = page_size()?;
1096 wait_resident(map.as_ptr() as usize, off, len, page).map_err(ParseError::ValidationError)
1097}
1098
1099#[cfg(target_os = "linux")]
1100fn willneed(file: &File, off: usize, len: usize) -> Result<(), ParseError> {
1101 let rc = unsafe {
1102 libc::posix_fadvise(
1103 file.as_raw_fd(),
1104 off as libc::off_t,
1105 len as libc::off_t,
1106 libc::POSIX_FADV_WILLNEED,
1107 )
1108 };
1109 if rc != 0 {
1110 return Err(ParseError::ValidationError(format!(
1111 "posix_fadvise WILLNEED: {rc}"
1112 )));
1113 }
1114 Ok(())
1115}
1116
1117#[cfg(target_os = "linux")]
1118fn worker_ranges(len: usize, threads: usize, page: usize) -> Vec<(usize, usize)> {
1119 if len == 0 {
1120 return Vec::new();
1121 }
1122 let threads = threads.max(1);
1123 let workers = if len < page.saturating_mul(threads) {
1124 1
1125 } else {
1126 threads
1127 };
1128 if workers == 1 {
1129 return vec![(0, len)];
1130 }
1131 let chunk = (len / workers) & !(page - 1);
1132 let mut ranges = Vec::with_capacity(workers);
1133 let mut start = 0usize;
1134 for worker in 0..workers {
1135 let take = if worker + 1 == workers {
1136 len - start
1137 } else {
1138 chunk
1139 };
1140 ranges.push((start, start + take));
1141 start += take;
1142 }
1143 ranges
1144}
1145
1146#[cfg(target_os = "linux")]
1147fn advise_spans(
1148 map: &memmap2::MmapMut,
1149 advice: memmap2::Advice,
1150 threads: usize,
1151 page: usize,
1152) -> Result<(), ParseError> {
1153 let len = map.len();
1154 if len == 0 {
1155 return Ok(());
1156 }
1157 let threads = threads.max(1);
1158 let workers = if len < page.saturating_mul(threads) {
1159 1
1160 } else {
1161 threads
1162 };
1163 if workers == 1 {
1164 return map
1165 .advise(advice)
1166 .map_err(|err| ParseError::ValidationError(err.to_string()));
1167 }
1168 let chunk = (len / workers) & !(page - 1);
1169 std::thread::scope(|scope| {
1170 let mut start = 0usize;
1171 let mut handles = Vec::with_capacity(workers);
1172 for worker in 0..workers {
1173 let take = if worker + 1 == workers {
1174 len - start
1175 } else {
1176 chunk
1177 };
1178 let off = start;
1179 start += take;
1180 handles.push(scope.spawn(move || map.advise_range(advice, off, take)));
1181 }
1182 handles.into_iter().try_for_each(|handle| {
1183 handle
1184 .join()
1185 .unwrap_or_else(|_| Err(std::io::Error::other("CON-bin populate thread panicked")))
1186 .map_err(|err| ParseError::ValidationError(err.to_string()))
1187 })
1188 })
1189}
1190
1191pub fn read_header(path: &Path) -> Result<ConBinHeader, ParseError> {
1192 let mut file = File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
1193 let mut bytes = [0u8; FIELD_BYTES];
1194 file.read_exact(&mut bytes)
1195 .map_err(|err| ParseError::ValidationError(err.to_string()))?;
1196 parse_fields(&bytes).map(|(header, _)| header)
1197}
1198
1199fn bytes_of<T>(values: &mut [T]) -> &mut [u8] {
1200 let len = std::mem::size_of_val(values);
1201 unsafe { std::slice::from_raw_parts_mut(values.as_mut_ptr().cast::<u8>(), len) }
1204}
1205
1206pub fn read_into(
1208 path: &Path,
1209 positions: &mut [f64],
1210 velocities: Option<&mut [f64]>,
1211) -> Result<ConBinHeader, ParseError> {
1212 read_into_threads(path, positions, velocities, 1)
1213}
1214
1215pub fn read_into_threads(
1218 path: &Path,
1219 positions: &mut [f64],
1220 velocities: Option<&mut [f64]>,
1221 threads: usize,
1222) -> Result<ConBinHeader, ParseError> {
1223 read_columns_into(path, positions, velocities, threads, false)
1224}
1225
1226pub fn read_owned_f64(
1232 path: &Path,
1233 threads: usize,
1234) -> Result<(ConBinHeader, Vec<f64>, Vec<f64>), ParseError> {
1235 let header = read_header(path)?;
1236 if header.binary32 {
1237 return Err(ParseError::ValidationError(
1238 "CON-bin file is binary32; use read_into_f32".into(),
1239 ));
1240 }
1241 if !header.velocities {
1242 return Err(ParseError::ValidationError(
1243 "CON-bin file has no velocity run".into(),
1244 ));
1245 }
1246 let n = header.position_len()?;
1247 let mut positions = Vec::<f64>::with_capacity(n);
1248 let mut velocities = Vec::<f64>::with_capacity(n);
1249 fill_owned_f64(path, &mut positions, &mut velocities, n, threads)?;
1250 Ok((header, positions, velocities))
1251}
1252
1253#[cfg(unix)]
1254fn spare_prefix_uninit(buf: &mut Vec<f64>, n: usize) -> &mut [std::mem::MaybeUninit<u8>] {
1255 let spare = buf.spare_capacity_mut();
1256 assert!(spare.len() >= n);
1257 let nbytes = n * std::mem::size_of::<f64>();
1258 let ptr = spare.as_mut_ptr().cast::<std::mem::MaybeUninit<u8>>();
1259 unsafe { std::slice::from_raw_parts_mut(ptr, nbytes) }
1264}
1265
1266fn fill_owned_f64(
1267 path: &Path,
1268 positions: &mut Vec<f64>,
1269 velocities: &mut Vec<f64>,
1270 n: usize,
1271 threads: usize,
1272) -> Result<(), ParseError> {
1273 #[cfg(not(unix))]
1274 {
1275 let _ = (path, positions, velocities, n, threads);
1276 return Err(ParseError::ValidationError(
1277 "CON-bin pread fill requires Unix".into(),
1278 ));
1279 }
1280 #[cfg(unix)]
1281 {
1282 let file = File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
1283 let mut raw = [0u8; FIELD_BYTES];
1284 (&file)
1285 .read_exact(&mut raw)
1286 .map_err(|err| ParseError::ValidationError(err.to_string()))?;
1287 let (_header, offset) = parse_fields(&raw)?;
1288 let width = std::mem::size_of::<f64>();
1289 let expect = n.checked_mul(width).ok_or_else(|| {
1290 ParseError::InvalidCount("CON-bin column byte length overflows".into())
1291 })?;
1292 let page = page_size()?;
1293 let file_len = file
1294 .metadata()
1295 .map_err(|err| ParseError::ValidationError(err.to_string()))?
1296 .len() as usize;
1297 let pos_off = checked_payload_offset(offset, file_len)? as u64;
1298 let fd = file.as_raw_fd();
1299 {
1300 let bytes = spare_prefix_uninit(positions, n);
1301 pread_parallel_uninit(fd, bytes, pos_off, threads, page)?;
1302 }
1303 unsafe { positions.set_len(n) };
1305 {
1306 let bytes = spare_prefix_uninit(velocities, n);
1307 pread_parallel_uninit(fd, bytes, pos_off + expect as u64, threads, page)?;
1308 }
1309 unsafe { velocities.set_len(n) };
1310 Ok(())
1311 }
1312}
1313
1314pub fn read_into_f32(
1316 path: &Path,
1317 positions: &mut [f32],
1318 velocities: Option<&mut [f32]>,
1319) -> Result<ConBinHeader, ParseError> {
1320 read_into_f32_threads(path, positions, velocities, 1)
1321}
1322
1323pub fn read_into_f32_threads(
1326 path: &Path,
1327 positions: &mut [f32],
1328 velocities: Option<&mut [f32]>,
1329 threads: usize,
1330) -> Result<ConBinHeader, ParseError> {
1331 read_columns_into(path, positions, velocities, threads, true)
1332}
1333
1334fn read_columns_into<T>(
1335 path: &Path,
1336 positions: &mut [T],
1337 velocities: Option<&mut [T]>,
1338 threads: usize,
1339 binary32: bool,
1340) -> Result<ConBinHeader, ParseError> {
1341 let file = File::open(path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
1342 let mut raw = [0u8; FIELD_BYTES];
1343 (&file)
1344 .read_exact(&mut raw)
1345 .map_err(|err| ParseError::ValidationError(err.to_string()))?;
1346 let (header, offset) = parse_fields(&raw)?;
1347 if header.binary32 != binary32 {
1348 return Err(ParseError::ValidationError(if binary32 {
1349 "CON-bin file is binary64; use read_into".into()
1350 } else {
1351 "CON-bin file is binary32; use read_into_f32".into()
1352 }));
1353 }
1354 let n = header.position_len()?;
1355 let width = std::mem::size_of::<T>();
1356 let expect = n
1357 .checked_mul(width)
1358 .ok_or_else(|| ParseError::InvalidCount("CON-bin column byte length overflows".into()))?;
1359 if positions.len() != n {
1360 return Err(ParseError::InvalidCount(format!(
1361 "position slice has {} values, the file has {n}",
1362 positions.len()
1363 )));
1364 }
1365 let page = page_size()?;
1366 let file_len = file
1367 .metadata()
1368 .map_err(|err| ParseError::ValidationError(err.to_string()))?
1369 .len() as usize;
1370 let pos_off = checked_payload_offset(offset, file_len)? as u64;
1371 let pos_bytes = bytes_of(positions);
1372 if pos_bytes.len() != expect {
1373 return Err(ParseError::InvalidCount(format!(
1374 "position slice has {} bytes, the file has {expect}",
1375 pos_bytes.len()
1376 )));
1377 }
1378 #[cfg(not(unix))]
1379 {
1380 let _ = (pos_bytes, pos_off, threads, page, velocities, header);
1381 return Err(ParseError::ValidationError(
1382 "CON-bin pread fill requires Unix".into(),
1383 ));
1384 }
1385 #[cfg(unix)]
1386 {
1387 let fd = file.as_raw_fd();
1388 #[cfg(target_os = "linux")]
1392 unsafe {
1393 libc::posix_fadvise(fd, 0, 0, libc::POSIX_FADV_SEQUENTIAL);
1394 }
1395 pread_parallel(fd, pos_bytes, pos_off, threads, page)?;
1398 if header.velocities {
1399 let Some(vel) = velocities else {
1400 return Err(ParseError::InvalidCount(
1401 "CON-bin has velocities and the caller passed no velocity slice".into(),
1402 ));
1403 };
1404 if vel.len() != n {
1405 return Err(ParseError::InvalidCount(format!(
1406 "velocity slice has {} values, the file has {n}",
1407 vel.len()
1408 )));
1409 }
1410 let vel_bytes = bytes_of(vel);
1411 pread_parallel(fd, vel_bytes, pos_off + expect as u64, threads, page)?;
1412 }
1413 Ok(header)
1414 }
1415}
1416
1417#[cfg(unix)]
1418fn pread_parallel(
1419 fd: i32,
1420 dest: &mut [u8],
1421 offset: u64,
1422 threads: usize,
1423 page: usize,
1424) -> Result<(), ParseError> {
1425 let threads = threads.max(1);
1426 let workers = if dest.len() < page.saturating_mul(threads) {
1427 1
1428 } else {
1429 threads
1430 };
1431 if workers == 1 {
1432 return pread_all(fd, dest, offset);
1433 }
1434 let chunk = (dest.len() / workers) & !(page - 1);
1435 std::thread::scope(|scope| {
1436 let mut rest = dest;
1437 let mut off = offset;
1438 let mut handles = Vec::with_capacity(workers);
1439 for worker in 0..workers {
1440 let take = if worker + 1 == workers {
1441 rest.len()
1442 } else {
1443 chunk
1444 };
1445 let (head, tail) = rest.split_at_mut(take);
1446 rest = tail;
1447 let this = off;
1448 off += take as u64;
1449 handles.push(scope.spawn(move || pread_all(fd, head, this)));
1450 }
1451 handles
1452 .into_iter()
1453 .try_for_each(|handle| handle.join().unwrap_or(Err(ParseError::IncompleteFrame)))
1454 })
1455}
1456
1457#[cfg(unix)]
1458fn pread_raw(fd: i32, ptr: *mut u8, len: usize, mut offset: u64) -> Result<(), ParseError> {
1459 let mut done = 0;
1460 while done < len {
1461 let n = unsafe { libc::pread(fd, ptr.add(done).cast(), len - done, offset as libc::off_t) };
1462 if n < 0 {
1463 let err = std::io::Error::last_os_error();
1464 if err.raw_os_error() == Some(libc::EINTR) {
1468 continue;
1469 }
1470 return Err(ParseError::ValidationError(err.to_string()));
1471 }
1472 if n == 0 {
1473 return Err(ParseError::IncompleteFrame);
1474 }
1475 done += n as usize;
1476 offset += n as u64;
1477 }
1478 Ok(())
1479}
1480
1481#[cfg(unix)]
1482fn pread_all(fd: i32, dest: &mut [u8], offset: u64) -> Result<(), ParseError> {
1483 pread_raw(fd, dest.as_mut_ptr(), dest.len(), offset)
1484}
1485
1486#[cfg(unix)]
1487fn pread_uninit(
1488 fd: i32,
1489 dest: &mut [std::mem::MaybeUninit<u8>],
1490 offset: u64,
1491) -> Result<(), ParseError> {
1492 pread_raw(fd, dest.as_mut_ptr().cast::<u8>(), dest.len(), offset)
1495}
1496
1497#[cfg(unix)]
1498fn pread_parallel_uninit(
1499 fd: i32,
1500 dest: &mut [std::mem::MaybeUninit<u8>],
1501 offset: u64,
1502 threads: usize,
1503 page: usize,
1504) -> Result<(), ParseError> {
1505 let threads = threads.max(1);
1506 let workers = if dest.len() < page.saturating_mul(threads) {
1507 1
1508 } else {
1509 threads
1510 };
1511 if workers == 1 {
1512 return pread_uninit(fd, dest, offset);
1513 }
1514 let chunk = (dest.len() / workers) & !(page - 1);
1515 std::thread::scope(|scope| {
1516 let mut rest = dest;
1517 let mut off = offset;
1518 let mut handles = Vec::with_capacity(workers);
1519 for worker in 0..workers {
1520 let take = if worker + 1 == workers {
1521 rest.len()
1522 } else {
1523 chunk
1524 };
1525 let (head, tail) = rest.split_at_mut(take);
1526 rest = tail;
1527 let this = off;
1528 off += take as u64;
1529 handles.push(scope.spawn(move || pread_uninit(fd, head, this)));
1530 }
1531 handles
1532 .into_iter()
1533 .try_for_each(|handle| handle.join().unwrap_or(Err(ParseError::IncompleteFrame)))
1534 })
1535}
1536
1537fn write_values<T>(file: &mut File, values: &[T]) -> Result<(), ParseError> {
1538 let bytes = unsafe {
1539 std::slice::from_raw_parts(values.as_ptr().cast::<u8>(), std::mem::size_of_val(values))
1540 };
1541 file.write_all(bytes)
1542 .map_err(|err| ParseError::ValidationError(err.to_string()))
1543}
1544
1545fn write_header_page(
1546 file: &mut File,
1547 header: &ConBinHeader,
1548 page: usize,
1549) -> Result<(), ParseError> {
1550 let mut header_page = vec![0u8; page];
1551 let prefix = header_fields(header, page as u64);
1552 header_page[..FIELD_BYTES].copy_from_slice(&prefix);
1553 file.write_all(&header_page)
1554 .map_err(|err| ParseError::ValidationError(err.to_string()))
1555}
1556
1557pub fn write_from_con(
1562 con_path: &Path,
1563 out_path: &Path,
1564 binary32: bool,
1565) -> Result<ConBinHeader, ParseError> {
1566 let text = std::fs::read_to_string(con_path)
1567 .map_err(|err| ParseError::ValidationError(err.to_string()))?;
1568 let mut file =
1569 File::create(out_path).map_err(|err| ParseError::ValidationError(err.to_string()))?;
1570 let page = page_size()?;
1571 write_header_page(
1572 &mut file,
1573 &ConBinHeader {
1574 n_frames: 0,
1575 n_atoms: 0,
1576 binary32,
1577 velocities: true,
1578 },
1579 page,
1580 )?;
1581
1582 let mut n_frames = 0u64;
1583 let mut n_atoms = 0u64;
1584 let mut positions: Vec<f64> = Vec::new();
1585 let mut velocities: Vec<f64> = Vec::new();
1586 for frame in ConFrameIterator::new(&text) {
1587 let frame = frame?;
1588 let n = frame.atom_data.len();
1589 if n_frames == 0 {
1590 n_atoms = n as u64;
1591 } else if n as u64 != n_atoms {
1592 return Err(ParseError::InvalidCount(
1593 "CON-bin requires the same atom count in every frame".into(),
1594 ));
1595 }
1596 for atom in &frame.atom_data {
1597 positions.extend_from_slice(&[atom.x, atom.y, atom.z]);
1598 velocities.extend_from_slice(&atom.velocity().unwrap_or([0.0; 3]));
1599 }
1600 n_frames += 1;
1601 }
1602 if binary32 {
1603 let pos32: Vec<f32> = positions.iter().map(|v| *v as f32).collect();
1604 let vel32: Vec<f32> = velocities.iter().map(|v| *v as f32).collect();
1605 write_values(&mut file, &pos32)?;
1606 write_values(&mut file, &vel32)?;
1607 } else {
1608 write_values(&mut file, &positions)?;
1609 write_values(&mut file, &velocities)?;
1610 }
1611 let header = ConBinHeader {
1612 n_frames,
1613 n_atoms,
1614 binary32,
1615 velocities: true,
1616 };
1617 file.seek(SeekFrom::Start(0))
1618 .map_err(|err| ParseError::ValidationError(err.to_string()))?;
1619 write_header_page(&mut file, &header, page)?;
1620 Ok(header)
1621}
1622
1623#[cfg(test)]
1624mod tests {
1625 use super::*;
1626
1627 fn write_known(path: &Path, frames: usize, atoms: usize) {
1628 let page = page_size().unwrap();
1629 let mut file = File::create(path).unwrap();
1630 let header = ConBinHeader {
1631 n_frames: frames as u64,
1632 n_atoms: atoms as u64,
1633 binary32: false,
1634 velocities: true,
1635 };
1636 write_header_page(&mut file, &header, page).unwrap();
1637 let n = frames * atoms * 3;
1638 let pos: Vec<f64> = (0..n).map(|i| i as f64 * 0.25).collect();
1639 let vel: Vec<f64> = (0..n).map(|i| i as f64 * -0.5).collect();
1640 write_values(&mut file, &pos).unwrap();
1641 write_values(&mut file, &vel).unwrap();
1642 }
1643
1644 #[cfg(target_os = "linux")]
1645 #[test]
1646 fn worker_span_shares_the_queue() {
1647 let limits = QueueLimits {
1648 page: 4096,
1649 logical_block: 512,
1650 max_request: 1280 * 1024,
1651 nr_requests: 256,
1652 read_ahead: 128 * 1024,
1653 nr_hw_queues: 1,
1654 rotational: true,
1655 };
1656 assert_eq!(limits.worker_span(8), 32 * 1280 * 1024);
1657 assert_eq!(limits.worker_span(1), 256 * 1280 * 1024);
1658 let dir = std::path::PathBuf::from(format!("/dev/shm/conbin-sum-{}", std::process::id()));
1660 let _ = std::fs::remove_dir_all(&dir);
1661 std::fs::create_dir_all(&dir).unwrap();
1662 let path = dir.join("known.conbin");
1663 write_known(&path, 4, 8);
1664 let mapped = map_file(&path, 1).unwrap();
1665 let expect = mapped.positions_f64().unwrap().iter().sum::<f64>()
1666 + mapped.velocities_f64().unwrap().iter().sum::<f64>();
1667 let got = sum_mapped(&path, 2).unwrap();
1668 let got_split = sum_split_streams(&path, 2).unwrap();
1669 assert_eq!(got, expect, "parallel sum {got} serial sum {expect}");
1670 assert_eq!(
1671 got_split, expect,
1672 "split sum {got_split} serial sum {expect}"
1673 );
1674 let _ = std::fs::remove_dir_all(&dir);
1675 let short = QueueLimits {
1676 nr_requests: 2,
1677 read_ahead: 4096,
1678 ..limits
1679 };
1680 assert_eq!(short.worker_span(8), 1280 * 1024);
1681 let shm = std::path::PathBuf::from(format!("/dev/shm/conbin-queue-{}", std::process::id()));
1682 let _ = std::fs::remove_dir_all(&shm);
1683 std::fs::create_dir_all(&shm).unwrap();
1684 let qpath = shm.join("known.conbin");
1685 write_known(&qpath, 1, 1);
1686 let got = QueueLimits::of(&qpath).unwrap();
1687 assert!(got.page.is_power_of_two());
1688 assert_eq!(got.max_request, got.page);
1689 assert_eq!(got.nr_requests, 1);
1690 assert!(!got.rotational);
1691 let _ = std::fs::remove_dir_all(&shm);
1692 }
1693
1694 #[test]
1695 fn mapped_runs_match_the_pread_fill() {
1696 let dir = std::env::temp_dir().join(format!("conbin-map-{}", std::process::id()));
1697 let _ = std::fs::remove_dir_all(&dir);
1698 std::fs::create_dir_all(&dir).unwrap();
1699 let path = dir.join("known.conbin");
1700 write_known(&path, 4, 3);
1701 let mapped = map_file(&path, 2).unwrap();
1702 let pos = mapped.positions_f64().unwrap();
1703 let vel = mapped.velocities_f64().unwrap();
1704 assert_eq!(pos.len(), 4 * 3 * 3);
1705 assert_eq!(pos[0], 0.0);
1706 assert_eq!(pos[1], 0.25);
1707 assert_eq!(vel[1], -0.5);
1708 let mut got_pos = vec![0.0; pos.len()];
1709 let mut got_vel = vec![0.0; vel.len()];
1710 read_into_threads(&path, &mut got_pos, Some(&mut got_vel), 2).unwrap();
1711 assert_eq!(got_pos, pos);
1712 assert_eq!(got_vel, vel);
1713 let _ = std::fs::remove_dir_all(&dir);
1714 }
1715
1716 #[test]
1717 fn write_from_con_round_trips_text() {
1718 let dir = std::env::temp_dir().join(format!("conbin-text-{}", std::process::id()));
1719 let _ = std::fs::remove_dir_all(&dir);
1720 std::fs::create_dir_all(&dir).unwrap();
1721 let con = dir.join("two.con");
1722 std::fs::write(
1723 &con,
1724 include_str!("../resources/test/tiny_cuh2.con").repeat(2),
1725 )
1726 .unwrap();
1727 let bin = dir.join("two.conbin");
1728 let header = write_from_con(&con, &bin, false).unwrap();
1729 assert_eq!(header.n_frames, 2);
1730 assert_eq!(header.n_atoms, 4);
1731 let mapped = map_file(&bin, 1).unwrap();
1732 let pos = mapped.positions_f64().unwrap();
1733 assert_eq!(pos.len(), 2 * 4 * 3);
1734 assert!(pos[0].is_finite());
1735 let _ = std::fs::remove_dir_all(&dir);
1736 }
1737
1738 #[test]
1739 fn a_store_copies_the_page_and_leaves_the_file() {
1740 let dir = std::env::temp_dir().join(format!("conbin-cow-{}", std::process::id()));
1741 let _ = std::fs::remove_dir_all(&dir);
1742 std::fs::create_dir_all(&dir).unwrap();
1743 let path = dir.join("known.conbin");
1744 write_known(&path, 2, 4);
1745 let before = std::fs::read(&path).unwrap();
1746 let mut mapped = map_file(&path, 1).unwrap();
1747 {
1748 let pos = mapped.positions_f64_mut().unwrap();
1749 let original = pos[0];
1750 pos[0] = original + 1.0;
1751 assert_eq!(pos[0], original + 1.0);
1752 }
1753 let after = std::fs::read(&path).unwrap();
1754 assert_eq!(before, after);
1755 let reread = map_file(&path, 1).unwrap();
1756 assert_eq!(reread.positions_f64().unwrap()[0], 0.0);
1757 let _ = std::fs::remove_dir_all(&dir);
1758 }
1759
1760 #[test]
1761 fn a_4k_offset_opens_on_a_16k_or_64k_host() {
1762 let header = ConBinHeader {
1766 n_frames: 1,
1767 n_atoms: 1,
1768 binary32: false,
1769 velocities: true,
1770 };
1771 let mut bytes = header_fields(&header, 4096).to_vec();
1772 bytes.resize(4096 + 48, 0);
1773 bytes[4096..4096 + 8].copy_from_slice(&1.25f64.to_le_bytes());
1774 bytes[4096 + 24..4096 + 32].copy_from_slice(&(-2.5f64).to_le_bytes());
1775 assert!(bytes.len() < 16 * 1024);
1776 assert!(bytes.len() < 64 * 1024);
1777 let off = checked_payload_offset(4096, bytes.len()).unwrap();
1778 assert_eq!(off, 4096);
1779 assert_ne!(off, 16 * 1024);
1780 assert_ne!(off, 64 * 1024);
1781 let dir = std::env::temp_dir().join(format!("conbin-off-{}", std::process::id()));
1782 let _ = std::fs::remove_dir_all(&dir);
1783 std::fs::create_dir_all(&dir).unwrap();
1784 let path = dir.join("known.conbin");
1785 std::fs::write(&path, &bytes).unwrap();
1786 let mapped = map_file(&path, 1).unwrap();
1787 assert_eq!(mapped.payload_off, 4096);
1788 assert_eq!(mapped.positions_f64().unwrap()[0], 1.25);
1789 assert_eq!(mapped.velocities_f64().unwrap()[0], -2.5);
1790 let _ = std::fs::remove_dir_all(&dir);
1791 }
1792
1793 fn write_typed(path: &Path, binary32: bool, velocities: bool) {
1794 let header = ConBinHeader {
1795 n_frames: 1,
1796 n_atoms: 1,
1797 binary32,
1798 velocities,
1799 };
1800 let mut bytes = header_fields(&header, 64).to_vec();
1801 bytes.resize(64 + 48, 0);
1802 std::fs::write(path, bytes).unwrap();
1803 }
1804
1805 #[test]
1806 fn sum_mapped_rejects_binary32_and_a_missing_velocity_run() {
1807 let dir = std::env::temp_dir().join(format!("conbin-kind-{}", std::process::id()));
1808 let _ = std::fs::remove_dir_all(&dir);
1809 std::fs::create_dir_all(&dir).unwrap();
1810 let binary32 = dir.join("f32.conbin");
1811 write_typed(&binary32, true, true);
1812 let err = sum_mapped(&binary32, 1).unwrap_err();
1813 assert!(err.to_string().contains("binary32"), "{err}");
1814 let bare = dir.join("nov.conbin");
1815 write_typed(&bare, false, false);
1816 let err = sum_mapped(&bare, 1).unwrap_err();
1817 assert!(err.to_string().contains("no velocity"), "{err}");
1818 let err = read_owned_f64(&bare, 1).unwrap_err();
1819 assert!(err.to_string().contains("no velocity"), "{err}");
1820 let known = dir.join("known.conbin");
1821 write_known(&known, 1, 2);
1822 let (_header, pos, vel) = read_owned_f64(&known, 1).unwrap();
1823 let mapped = map_file(&known, 1).unwrap();
1824 assert_eq!(pos.as_slice(), mapped.positions_f64().unwrap());
1825 assert_eq!(vel.as_slice(), mapped.velocities_f64().unwrap());
1826 let _ = std::fs::remove_dir_all(&dir);
1827 }
1828
1829 #[cfg(target_os = "linux")]
1830 fn dir_is_tmpfs(path: &Path) -> bool {
1831 let text = path.to_string_lossy();
1832 let Ok(c) = std::ffi::CString::new(text.as_bytes()) else {
1833 return false;
1834 };
1835 let mut stat: libc::statfs = unsafe { std::mem::zeroed() };
1836 let rc = unsafe { libc::statfs(c.as_ptr(), &mut stat) };
1837 rc == 0 && stat.f_type == 0x01021994
1839 }
1840
1841 #[cfg(target_os = "linux")]
1842 fn streaming_dir() -> Option<std::path::PathBuf> {
1843 let mut candidates = vec![std::env::temp_dir()];
1844 if let Ok(cwd) = std::env::current_dir() {
1845 candidates.push(cwd);
1846 }
1847 candidates.push(std::path::PathBuf::from("/var/tmp"));
1848 for base in candidates {
1849 if base.is_dir() && !dir_is_tmpfs(&base) {
1850 let dir = base.join(format!("conbin-stream-{}", std::process::id()));
1851 let _ = std::fs::remove_dir_all(&dir);
1852 if std::fs::create_dir_all(&dir).is_ok() {
1853 return Some(dir);
1854 }
1855 }
1856 }
1857 None
1858 }
1859
1860 #[cfg(target_os = "linux")]
1861 #[test]
1862 fn sum_mapped_streams_when_pages_are_not_resident() {
1863 let Some(dir) = streaming_dir() else {
1864 panic!(
1865 "sum_mapped_streams_when_pages_are_not_resident: temp, the working directory and /var/tmp are tmpfs, and POSIX_FADV_DONTNEED leaves tmpfs pages resident"
1866 );
1867 };
1868 let _ = std::fs::remove_dir_all(&dir);
1869 std::fs::create_dir_all(&dir).unwrap();
1870 let path = dir.join("known.conbin");
1871 write_known(&path, 2, 32);
1872 let file = File::open(&path).unwrap();
1873 file.sync_all().unwrap();
1874 let rc = unsafe { libc::posix_fadvise(file.as_raw_fd(), 0, 0, libc::POSIX_FADV_DONTNEED) };
1875 assert_eq!(rc, 0);
1876 let len = file.metadata().unwrap().len() as usize;
1877 let ptr = unsafe {
1878 libc::mmap(
1879 std::ptr::null_mut(),
1880 len,
1881 libc::PROT_READ,
1882 libc::MAP_PRIVATE,
1883 file.as_raw_fd(),
1884 0,
1885 )
1886 };
1887 assert_ne!(ptr, libc::MAP_FAILED);
1888 let page = page_size().unwrap();
1889 let pages = len.div_ceil(page);
1890 let mut vec = vec![0u8; pages];
1891 let rc = unsafe { libc::mincore(ptr, len, vec.as_mut_ptr()) };
1892 assert_eq!(rc, 0);
1893 assert!(
1894 vec.iter().any(|byte| byte & 1 == 0),
1895 "every page stayed resident"
1896 );
1897 unsafe { libc::munmap(ptr, len) };
1898 let got = sum_mapped(&path, 2).unwrap();
1899 let mapped = map_file(&path, 1).unwrap();
1900 let expect = mapped.positions_f64().unwrap().iter().sum::<f64>()
1901 + mapped.velocities_f64().unwrap().iter().sum::<f64>();
1902 assert_eq!(got, expect, "{got} {expect}");
1903 let _ = std::fs::remove_dir_all(&dir);
1904 }
1905
1906 #[cfg(unix)]
1911 #[test]
1912 fn read_owned_f64_fills_spare_capacity_by_pread() {
1913 let dir = tempfile::tempdir().unwrap();
1914 let path = dir.path().join("known.conbin");
1915 write_known(&path, 2, 8);
1916 for threads in [1usize, 2] {
1917 let (header, pos, vel) = read_owned_f64(&path, threads).unwrap();
1918 let n = header.position_len().unwrap();
1919 assert_eq!(pos.len(), n);
1920 assert_eq!(vel.len(), n);
1921 let mut raw = Vec::new();
1922 std::io::Read::read_to_end(&mut File::open(&path).unwrap(), &mut raw).unwrap();
1923 let off = raw.len() - 2 * n * 8;
1924 let want: Vec<f64> = raw[off..]
1925 .as_chunks::<8>()
1926 .0
1927 .iter()
1928 .map(|chunk| f64::from_le_bytes(*chunk))
1929 .collect();
1930 assert_eq!(pos.as_slice(), &want[..n]);
1931 assert_eq!(vel.as_slice(), &want[n..]);
1932 }
1933 }
1934
1935 #[cfg(target_os = "linux")]
1936 #[test]
1937 fn populate_followup_touches_only_after_einval() {
1938 assert!(populate_followup(libc::EINVAL).is_ok());
1939 for code in [libc::EFAULT, libc::EIO, libc::EHWPOISON, libc::ENOMEM] {
1940 assert_eq!(populate_followup(code).unwrap_err(), code);
1941 }
1942 }
1943
1944 #[cfg(target_os = "linux")]
1945 #[test]
1946 fn wait_resident_rejects_an_unaligned_address() {
1947 let page = page_size().unwrap();
1948 let ptr = unsafe {
1949 libc::mmap(
1950 std::ptr::null_mut(),
1951 page,
1952 libc::PROT_READ | libc::PROT_WRITE,
1953 libc::MAP_PRIVATE | libc::MAP_ANONYMOUS,
1954 -1,
1955 0,
1956 )
1957 };
1958 assert_ne!(ptr, libc::MAP_FAILED);
1959 let err = wait_resident(ptr as usize + 1, 0, 1, page).unwrap_err();
1960 assert!(err.contains("page-aligned"), "{err}");
1961 unsafe { libc::munmap(ptr, page) };
1962 }
1963
1964 #[cfg(target_os = "linux")]
1965 #[test]
1966 fn wait_resident_returns_an_unmapped_populate_error() {
1967 let page = page_size().unwrap();
1968 let ptr = unsafe {
1969 libc::mmap(
1970 std::ptr::null_mut(),
1971 page,
1972 libc::PROT_NONE,
1973 libc::MAP_PRIVATE | libc::MAP_ANONYMOUS,
1974 -1,
1975 0,
1976 )
1977 };
1978 assert_ne!(ptr, libc::MAP_FAILED);
1979 unsafe { libc::munmap(ptr, page) };
1980 let err = wait_resident(ptr as usize, 0, page, page).unwrap_err();
1981 assert!(err.contains("os error"), "{err}");
1982 }
1983}