1use std::path::{Path, PathBuf};
14use std::sync::atomic::{AtomicBool, Ordering};
15use std::sync::{Arc, Mutex, MutexGuard};
16
17use store_io_core::error::{OsError, OsErrorSource};
18use store_io_core::evidence::{
19 Bus, DeviceFacts, Evidence, FsFacts, FsKind, OsCacheMode, PlatformKind, Privilege, Tri,
20};
21use store_io_platform::{
22 Completion, CompletionBuf, DeviceIdentity, FileIdentity, FileMode, FileName, IoBuf, IoOp,
23 Platform, Queue, QueueConfig, RangeState, RawResult, Rejected, ReleaseHow, Wake,
24};
25
26use crate::world::{DeviceKind, EBADF, EINVAL, EIO, FileId, SimConfig, World};
27
28#[derive(Clone, Debug)]
30pub struct SimPlatform {
31 world: Arc<Mutex<World>>,
32}
33
34fn lock(w: &Mutex<World>) -> MutexGuard<'_, World> {
35 w.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
38}
39
40impl SimPlatform {
41 #[must_use]
43 pub fn new(cfg: SimConfig) -> Self {
44 Self {
45 world: Arc::new(Mutex::new(World::new(cfg))),
46 }
47 }
48
49 pub fn with_world<R>(&self, f: impl FnOnce(&mut World) -> R) -> R {
51 f(&mut lock(&self.world))
52 }
53
54 #[must_use]
72 pub fn fork(&self) -> Self {
73 let w = lock(&self.world);
74 let open = w.live_handles();
75 assert!(
76 open == 0,
77 "SimPlatform::fork: {open} handles of this world are open (files, \
78 directories, locks or queues): drop every store, sink, source and \
79 queue before forking"
80 );
81 Self {
82 world: Arc::new(Mutex::new(w.forked())),
83 }
84 }
85}
86
87#[derive(Debug)]
89pub struct SimFile {
90 _live: crate::world::Live,
91 id: FileId,
92 epoch: u64,
93 opened: u64,
96 writable: bool,
97}
98
99impl SimFile {
100 #[must_use]
102 pub fn id(&self) -> FileId {
103 self.id
104 }
105
106 fn check(&self, w: &World, write: bool) -> RawResult<()> {
107 if w.epoch() != self.epoch || (write && !self.writable) {
108 Err(EBADF)
109 } else if w.handle_lost(self.id, self.opened) {
110 Err(EIO)
111 } else {
112 Ok(())
113 }
114 }
115}
116
117#[derive(Debug, Clone)]
119pub struct SimDir {
120 _live: crate::world::Live,
121 path: PathBuf,
122 opened: u64,
125}
126
127impl SimDir {
128 fn check(&self, w: &World) -> RawResult<()> {
129 if w.dir_lost(&self.path, self.opened) {
130 Err(EIO)
131 } else {
132 Ok(())
133 }
134 }
135}
136
137#[derive(Debug)]
139pub struct SimLock {
140 _live: crate::world::Live,
141 world: Arc<Mutex<World>>,
142 file: FileId,
143 epoch: u64,
144}
145
146impl Drop for SimLock {
147 fn drop(&mut self) {
148 let mut w = lock(&self.world);
149 if w.epoch() == self.epoch {
150 w.unlock(self.file);
151 }
152 }
153}
154
155enum Pending {
156 Write {
157 file: FileId,
158 epoch: u64,
159 opened: u64,
160 offset: u64,
161 buf: IoBuf,
162 dsync: bool,
163 },
164 Read {
165 file: FileId,
166 epoch: u64,
167 opened: u64,
168 offset: u64,
169 buf: IoBuf,
170 },
171 Flush {
172 file: FileId,
173 epoch: u64,
174 opened: u64,
175 cover: u64,
176 },
177}
178
179pub struct SimQueue {
181 _live: crate::world::Live,
182 world: Arc<Mutex<World>>,
183 in_flight: Vec<(u64, Pending)>,
184 depth: usize,
185 woken: Arc<AtomicBool>,
186}
187
188#[derive(Debug, Clone)]
192pub struct SimWaker(Arc<AtomicBool>);
193
194impl Wake for SimWaker {
195 fn wake(&self) {
196 self.0.store(true, Ordering::Release);
197 }
198}
199
200impl SimQueue {
201 fn reap_now(&mut self, out: &mut CompletionBuf) -> usize {
204 let mut w = lock(&self.world);
205 let reorder = w.config().reorder;
206 let mut added = 0;
207 let mut i = 0;
208 while i < self.in_flight.len() && out.room() > 0 {
209 let pick = if reorder {
210 if !w.rng().chance(500_000) {
211 i += 1;
212 continue;
213 }
214 i
215 } else {
216 0
217 };
218 let (tag, p) = self.in_flight.remove(pick);
219 if out.push(complete(&mut w, tag, p)).is_err() {
222 break;
223 }
224 added += 1;
225 if !reorder {
226 i = 0;
227 }
228 }
229 added
230 }
231
232 fn accept(&mut self, op: IoOp<'_, SimFile>) -> Result<Pending, Rejected> {
236 let probe = lock(&self.world).submit_probe.0.clone();
237 if let Some(probe) = probe {
238 probe();
239 }
240 if let Err(raw) = step(&self.world) {
241 let buf = match op {
242 IoOp::Write { buf, .. } | IoOp::Read { buf, .. } => Some(buf),
243 IoOp::FlushData { .. } => None,
244 };
245 return Err(Rejected {
246 buf,
247 raw: Some(raw),
248 });
249 }
250 if self.in_flight.len() >= self.depth {
251 let buf = match op {
252 IoOp::Write { buf, .. } | IoOp::Read { buf, .. } => Some(buf),
253 IoOp::FlushData { .. } => None,
254 };
255 return Err(Rejected { buf, raw: None });
256 }
257 let mut w = lock(&self.world);
258 let pending = match op {
259 IoOp::Write {
260 file,
261 offset,
262 buf,
263 dsync,
264 } => {
265 if let Some(e) = w.submit_fails() {
266 return Err(Rejected {
267 buf: Some(buf),
268 raw: Some(e),
269 });
270 }
271 if let Err(e) = file.check(&w, true) {
272 return Err(Rejected {
273 buf: Some(buf),
274 raw: Some(e),
275 });
276 }
277 Pending::Write {
278 file: file.id,
279 epoch: file.epoch,
280 opened: file.opened,
281 offset,
282 buf,
283 dsync,
284 }
285 }
286 IoOp::Read { file, offset, buf } => {
287 if let Err(e) = file.check(&w, false) {
288 return Err(Rejected {
289 buf: Some(buf),
290 raw: Some(e),
291 });
292 }
293 Pending::Read {
294 file: file.id,
295 epoch: file.epoch,
296 opened: file.opened,
297 offset,
298 buf,
299 }
300 }
301 IoOp::FlushData { file } => {
302 if let Err(e) = file.check(&w, true) {
303 return Err(Rejected {
304 buf: None,
305 raw: Some(e),
306 });
307 }
308 Pending::Flush {
310 file: file.id,
311 epoch: file.epoch,
312 opened: file.opened,
313 cover: w.flush_cover(),
314 }
315 }
316 };
317 Ok(pending)
318 }
319}
320
321impl Queue for SimQueue {
322 type File = SimFile;
323 type Waker = SimWaker;
324
325 fn submit(&mut self, op: IoOp<'_, SimFile>, tag: u64) -> Result<(), Rejected> {
326 let pending = self.accept(op)?;
327 self.in_flight.push((tag, pending));
328 Ok(())
329 }
330
331 fn performs(&self) -> bool {
335 self.in_flight.is_empty() && lock(&self.world).config().inline
336 }
337
338 fn perform<'a>(
339 &mut self,
340 op: IoOp<'a, SimFile>,
341 ) -> Result<Result<(Option<IoBuf>, RawResult<usize>), IoOp<'a, SimFile>>, Rejected> {
342 if !self.performs() {
343 return Ok(Err(op));
344 }
345 let pending = self.accept(op)?;
346 let c = complete(&mut lock(&self.world), 0, pending);
347 Ok(Ok((c.buf, c.result)))
348 }
349
350 fn reap(&mut self, out: &mut CompletionBuf) -> usize {
351 if lock(&self.world).reap_held() {
352 return 0;
353 }
354 self.reap_now(out)
355 }
356
357 fn wait(&mut self, min: usize, out: &mut CompletionBuf) -> RawResult<usize> {
358 if let Some(raw) = lock(&self.world).wait_fails() {
359 return Err(raw);
360 }
361 if self.woken.swap(false, Ordering::AcqRel) {
362 return Ok(self.reap_now(out));
363 }
364 let mut added = self.reap_now(out);
365 let mut w = lock(&self.world);
366 while added < min && !self.in_flight.is_empty() && out.room() > 0 {
367 let pick = if w.config().reorder {
368 w.rng().below(self.in_flight.len() as u64) as usize
369 } else {
370 0
371 };
372 let (tag, p) = self.in_flight.remove(pick);
373 let c = complete(&mut w, tag, p);
374 if out.push(c).is_err() {
375 break;
376 }
377 added += 1;
378 }
379 Ok(added)
380 }
381
382 fn in_flight(&self) -> usize {
383 self.in_flight.len()
384 }
385
386 fn waker(&self) -> SimWaker {
387 SimWaker(Arc::clone(&self.woken))
388 }
389}
390
391const MEM_ALIGN: usize = 4096;
394
395fn aligned(w: &World, offset: u64, buf: &IoBuf) -> bool {
397 let lb = u64::from(w.config().logical_block.max(1));
398 offset % lb == 0 && buf.len() as u64 % lb == 0 && buf.as_ptr() as usize % MEM_ALIGN == 0
399}
400
401fn complete(w: &mut World, tag: u64, p: Pending) -> Completion {
403 match p {
404 Pending::Write {
405 file,
406 epoch,
407 opened,
408 offset,
409 buf,
410 dsync,
411 } => {
412 let result = if epoch != w.epoch() {
413 Err(EBADF)
414 } else if w.powered_off() || w.handle_lost(file, opened) {
415 Err(EIO)
416 } else if !aligned(w, offset, &buf) {
417 Err(EINVAL)
418 } else {
419 w.complete_write(file, offset, buf.as_slice(), dsync)
420 };
421 Completion {
422 tag,
423 buf: Some(buf),
424 result,
425 }
426 }
427 Pending::Read {
428 file,
429 epoch,
430 opened,
431 offset,
432 mut buf,
433 } => {
434 let result = if epoch != w.epoch() {
435 Err(EBADF)
436 } else if w.powered_off() || w.handle_lost(file, opened) {
437 Err(EIO)
438 } else if !aligned(w, offset, &buf) {
439 w.count_misaligned_read();
440 Err(EINVAL)
441 } else {
442 w.complete_read(file, offset, buf.as_mut_slice())
443 };
444 Completion {
445 tag,
446 buf: Some(buf),
447 result,
448 }
449 }
450 Pending::Flush {
451 file,
452 epoch,
453 opened,
454 cover,
455 } => {
456 let result = if epoch != w.epoch() {
457 Err(EBADF)
458 } else if w.powered_off() || w.handle_lost(file, opened) {
459 Err(EIO)
460 } else {
461 w.complete_flush(file, cover)
462 };
463 Completion {
464 tag,
465 buf: None,
466 result,
467 }
468 }
469 }
470}
471
472fn step(world: &Mutex<World>) -> RawResult<()> {
475 let probe = lock(world).step_probe.0.clone();
476 let cut = probe.is_some_and(|p| p());
477 let mut w = lock(world);
478 if cut {
479 w.cut_power();
480 }
481 if w.powered_off() {
482 return Err(EIO);
483 }
484 Ok(())
485}
486
487impl SimPlatform {
488 fn lock_probe(&self) {
490 let probe = lock(&self.world).lock_probe.0.clone();
491 if let Some(probe) = probe {
492 probe();
493 }
494 }
495}
496
497impl Platform for SimPlatform {
498 type File = SimFile;
499 type Dir = SimDir;
500 type Lock = SimLock;
501 type Queue = SimQueue;
502
503 fn open_dir(&self, path: &Path, create: bool) -> RawResult<SimDir> {
504 if create {
505 step(&self.world)?;
506 }
507 let mut w = lock(&self.world);
508 w.open_dir(path.to_path_buf(), create)?;
509 Ok(SimDir {
510 _live: w.live(),
511 path: path.to_path_buf(),
512 opened: w.next_open(),
513 })
514 }
515
516 fn create_file(&self, dir: &SimDir, name: &FileName) -> RawResult<SimFile> {
517 step(&self.world)?;
518 let mut w = lock(&self.world);
519 dir.check(&w)?;
520 let id = w.create_file(&dir.path, name.as_str())?;
521 Ok(SimFile {
522 _live: w.live(),
523 id,
524 epoch: w.epoch(),
525 opened: w.next_open(),
526 writable: true,
527 })
528 }
529
530 fn open_file(&self, dir: &SimDir, name: &FileName, mode: FileMode) -> RawResult<SimFile> {
531 let mut w = lock(&self.world);
532 if let Some((d, n, code)) = &w.faults().fail_open {
533 if *d == dir.path && n == name.as_str() {
534 return Err(OsError {
535 code: *code,
536 source: OsErrorSource::Sim,
537 });
538 }
539 }
540 dir.check(&w)?;
541 let id = w.lookup(&dir.path, name.as_str())?;
542 if mode == FileMode::ReadWrite && w.faults().read_only_media {
543 return Err(crate::world::EROFS);
544 }
545 Ok(SimFile {
546 _live: w.live(),
547 id,
548 epoch: w.epoch(),
549 opened: w.next_open(),
550 writable: mode == FileMode::ReadWrite,
551 })
552 }
553
554 fn rename_replace(
555 &self,
556 dir: &SimDir,
557 from: &FileName,
558 to: &FileName,
559 _file: &SimFile,
560 ) -> RawResult<()> {
561 step(&self.world)?;
562 let mut w = lock(&self.world);
563 dir.check(&w)?;
564 w.rename(&dir.path, from.as_str(), to.as_str())
565 }
566
567 fn rename_noreplace(
568 &self,
569 dir: &SimDir,
570 from: &FileName,
571 to: &FileName,
572 _file: &SimFile,
573 ) -> RawResult<()> {
574 step(&self.world)?;
575 let mut w = lock(&self.world);
576 dir.check(&w)?;
577 w.rename_noreplace(&dir.path, from.as_str(), to.as_str())
578 }
579
580 fn unlink(&self, dir: &SimDir, name: &FileName) -> RawResult<()> {
581 step(&self.world)?;
582 let mut w = lock(&self.world);
583 dir.check(&w)?;
584 w.unlink(&dir.path, name.as_str())
585 }
586
587 fn sync_dir(&self, dir: &SimDir) -> RawResult<()> {
588 step(&self.world)?;
589 let mut w = lock(&self.world);
590 dir.check(&w)?;
591 w.sync_dir(&dir.path)
592 }
593
594 fn free_space(&self, dir: &SimDir) -> RawResult<u64> {
597 let w = lock(&self.world);
598 dir.check(&w)?;
599 Ok(w.free_space())
600 }
601
602 fn dir_is_empty(&self, dir: &SimDir) -> RawResult<bool> {
603 let w = lock(&self.world);
604 dir.check(&w)?;
605 w.dir_is_empty(&dir.path)
606 }
607
608 fn lock_exclusive(&self, file: &SimFile) -> RawResult<SimLock> {
609 self.lock_probe();
610 let mut w = lock(&self.world);
612 file.check(&w, true)?;
613 w.lock(file.id, false)?;
614 Ok(SimLock {
615 _live: w.live(),
616 world: Arc::clone(&self.world),
617 file: file.id,
618 epoch: w.epoch(),
619 })
620 }
621
622 fn monotonic_ms(&self) -> u64 {
623 lock(&self.world).clock_ms()
624 }
625
626 fn monotonic_ns(&self) -> u64 {
627 lock(&self.world).clock_ns()
628 }
629
630 fn clock(&self) -> store_io_platform::Clock {
631 let world = Arc::clone(&self.world);
632 Arc::new(move || lock(&world).clock_ns())
633 }
634
635 fn identity_scope(&self) -> u64 {
636 lock(&self.world).id()
637 }
638
639 fn flush_errors_per_handle(&self) -> bool {
640 true
643 }
644
645 fn reopen(&self, file: &SimFile) -> Option<SimFile> {
646 let mut w = lock(&self.world);
648 (file.writable && w.reopen()).then(|| SimFile {
649 _live: w.live(),
650 id: file.id,
651 epoch: file.epoch,
652 opened: file.opened,
653 writable: true,
654 })
655 }
656
657 fn io_path(&self) -> store_io_platform::IoPath {
658 store_io_platform::IoPath {
659 interface: store_io_platform::IoInterface::Simulator,
660 reason: "the deterministic simulator".to_owned(),
661 }
662 }
663
664 fn lock_shared(&self, file: &SimFile) -> RawResult<SimLock> {
665 self.lock_probe();
666 let mut w = lock(&self.world);
667 file.check(&w, false)?;
668 w.lock(file.id, true)?;
669 Ok(SimLock {
670 _live: w.live(),
671 world: Arc::clone(&self.world),
672 file: file.id,
673 epoch: w.epoch(),
674 })
675 }
676
677 fn allocate(&self, file: &SimFile, len: u64) -> RawResult<()> {
678 step(&self.world)?;
679 let mut w = lock(&self.world);
680 file.check(&w, true)?;
681 w.allocate(file.id, len)
682 }
683
684 fn set_len(&self, file: &SimFile, len: u64) -> RawResult<()> {
685 step(&self.world)?;
686 let mut w = lock(&self.world);
687 file.check(&w, true)?;
688 w.set_len(file.id, len)
689 }
690
691 fn size(&self, file: &SimFile) -> RawResult<u64> {
692 let w = lock(&self.world);
693 file.check(&w, false)?;
694 w.size(file.id)
695 }
696
697 fn release_range(
698 &self,
699 file: &SimFile,
700 offset: u64,
701 len: u64,
702 how: ReleaseHow,
703 ) -> RawResult<()> {
704 let probe = lock(&self.world).release_probe.0.clone();
705 if let Some(probe) = probe {
706 probe();
707 }
708 step(&self.world)?;
709 let mut w = lock(&self.world);
710 file.check(&w, true)?;
711 w.release(file.id, offset, len, how == ReleaseHow::Deallocate)
714 }
715
716 fn allocate_range(&self, file: &SimFile, offset: u64, len: u64) -> RawResult<()> {
717 step(&self.world)?;
718 let mut w = lock(&self.world);
719 file.check(&w, true)?;
720 w.allocate_range(file.id, offset, len)
721 }
722
723 fn range_state(&self, file: &SimFile, offset: u64, len: u64) -> RawResult<RangeState> {
724 let w = lock(&self.world);
725 file.check(&w, false)?;
726 let unwritten = w.unwritten(file.id, offset, len)?;
727 Ok(RangeState {
728 unwritten: if unwritten { Tri::Yes } else { Tri::No },
729 shared: Tri::No,
730 cached_pages: Some(if w.cached_pages() {
731 len.div_ceil(4096)
732 } else {
733 0
734 }),
735 valid_data: if w.past_valid_data(file.id, offset, len) {
736 Tri::No
737 } else {
738 Tri::Yes
739 },
740 })
741 }
742
743 fn flush_all(&self, file: &SimFile) -> RawResult<()> {
744 step(&self.world)?;
745 let mut w = lock(&self.world);
746 file.check(&w, true)?;
747 w.flush_all(file.id)
748 }
749
750 fn device_identity(&self, _dir: &SimDir, file: &SimFile) -> DeviceIdentity {
751 let w = lock(&self.world);
752 w.device_config(file.id)
753 .evidence
754 .map_or_else(DeviceIdentity::unknown, |ev| ev.device.identity)
755 }
756
757 fn file_identity(&self, file: &SimFile) -> RawResult<FileIdentity> {
758 let w = lock(&self.world);
759 file.check(&w, false)?;
760 Ok(FileIdentity {
761 volume: w.volume_of(file.id),
762 file: u128::from(file.id),
763 })
764 }
765
766 fn path_identity(&self, dir: &SimDir, name: &FileName) -> RawResult<FileIdentity> {
767 let w = lock(&self.world);
768 dir.check(&w)?;
769 let id = w.lookup(&dir.path, name.as_str())?;
770 Ok(FileIdentity {
771 volume: w.volume_of(id),
772 file: u128::from(id),
773 })
774 }
775
776 fn probe(&self, _dir: &SimDir, file: &SimFile) -> RawResult<Evidence> {
777 let w = lock(&self.world);
778 file.check(&w, false)?;
779 if let Some(code) = w.faults().fail_probe {
780 return Err(OsError {
781 code,
782 source: OsErrorSource::Sim,
783 });
784 }
785 let device = w.device_config(file.id);
786 if let Some(ev) = device.evidence {
787 return Ok(ev);
788 }
789 let power_safe = device.kind == DeviceKind::PowerSafe;
790 let mut ev = Evidence::unknown(PlatformKind::Sim);
791 ev.device = DeviceFacts {
792 model: Some("store-io simulated device".to_owned()),
793 bus: Bus::Nvme,
794 cache_present: if power_safe { Tri::No } else { Tri::Yes },
795 cache_enabled: if power_safe { Tri::No } else { Tri::Yes },
796 os_cache_mode: if power_safe {
797 OsCacheMode::WriteThrough
798 } else {
799 OsCacheMode::WriteBack
800 },
801 backup_failed: Tri::No,
802 logical_block: w.config().logical_block,
803 physical_block: w.config().logical_block,
804 max_transfer: 1 << 20,
805 ..DeviceFacts::default()
806 };
807 ev.fs = FsFacts {
808 kind: FsKind::Sim,
809 direct_io: Tri::Yes,
810 cow: Tri::No,
811 dio_mem_align: 4096,
812 dio_offset_align: w.config().logical_block,
813 device: Some(w.volume_of(file.id())),
814 ..FsFacts::default()
815 };
816 ev.privilege = Privilege::Full;
817 Ok(ev)
818 }
819
820 fn queue(&self, cfg: QueueConfig) -> RawResult<SimQueue> {
821 let mut w = lock(&self.world);
822 w.queue_made();
823 Ok(SimQueue {
824 _live: w.live(),
825 world: Arc::clone(&self.world),
826 in_flight: Vec::new(),
827 depth: cfg.depth.max(1) as usize,
828 woken: Arc::new(AtomicBool::new(false)),
829 })
830 }
831}