1use std::fmt;
19use std::fs::{File, OpenOptions};
20use std::io;
21use std::ops::Range;
22use std::os::fd::AsRawFd;
23use std::os::unix::fs::MetadataExt;
24use std::path::PathBuf;
25use std::sync::Arc;
26use std::time::UNIX_EPOCH;
27
28use async_trait::async_trait;
29use bytes::Bytes;
30use futures::stream::{self, BoxStream, StreamExt};
31use io_uring::{IoUring, opcode, types};
32use object_store::local::LocalFileSystem;
33use object_store::path::Path;
34use object_store::{
35 Attributes, CopyOptions, Error, GetOptions, GetResult, GetResultPayload, ListResult,
36 MultipartUpload, ObjectMeta, ObjectStore, PutMode, PutMultipartOptions, PutOptions, PutPayload,
37 PutResult, RenameOptions, Result,
38};
39use oxidelake_core::EngineError;
40use tokio::sync::{mpsc, oneshot};
41
42const STORE: &str = "UringLocalFileSystem";
43
44const JOB_QUEUE: usize = 256;
49
50const STREAM_CHUNK: u64 = 8 << 20;
58
59enum Job {
60 Read {
61 file: Arc<File>,
65 range: Range<u64>,
66 reply: oneshot::Sender<io::Result<Bytes>>,
67 },
68 Write {
69 path: PathBuf,
70 payload: PutPayload,
71 reply: oneshot::Sender<io::Result<String>>,
73 },
74}
75
76const REAP_ATTEMPTS: u32 = 1024;
82
83fn submit_one(
97 ring: &mut IoUring,
98 entry: &io_uring::squeue::Entry,
99 seq: &mut u64,
100) -> io::Result<i32> {
101 *seq = seq.wrapping_add(1);
102 let tag = *seq;
103 let entry = entry.clone().user_data(tag);
104
105 unsafe { ring.submission().push(&entry) }
109 .map_err(|e| io::Error::other(format!("io_uring submission queue is full: {e:?}")))?;
110
111 let mut failure: Option<io::Error> = None;
113 let mut attempts = 0u32;
114 let result = loop {
115 if let Err(e) = ring.submit_and_wait(1) {
116 if e.kind() == io::ErrorKind::Interrupted {
117 continue;
122 }
123 if failure.is_none() {
124 failure = Some(e);
125 }
126 }
127 match ring.completion().next() {
128 Some(cqe) if cqe.user_data() == tag => break cqe.result(),
129 Some(cqe) => {
130 return Err(io::Error::other(format!(
134 "io_uring completion carries submission {} where {tag} was expected; \
135 the ring is out of step with the worker",
136 cqe.user_data()
137 )));
138 }
139 None => {
140 attempts += 1;
141 assert!(
142 failure.is_none() || attempts < REAP_ATTEMPTS,
143 "io_uring: submission {tag} was accepted by the kernel but its completion \
144 never arrived after {REAP_ATTEMPTS} attempts ({}); returning would free a \
145 buffer the kernel may still be writing into (#27)",
146 failure
147 .as_ref()
148 .map_or_else(|| "no error".to_owned(), |e| e.to_string()),
149 );
150 }
151 }
152 };
153 if let Some(e) = failure {
154 return Err(e);
155 }
156 if result < 0 {
157 return Err(io::Error::from_raw_os_error(-result));
158 }
159 Ok(result)
160}
161
162fn read_range(
163 ring: &mut IoUring,
164 seq: &mut u64,
165 file: &File,
166 range: Range<u64>,
167) -> io::Result<Bytes> {
168 let len = usize::try_from(range.end.saturating_sub(range.start))
169 .map_err(|_| io::Error::other("range too large"))?;
170 let mut buf = vec![0u8; len];
171 let mut done = 0usize;
172 while done < len {
173 let chunk = u32::try_from(len - done).unwrap_or(u32::MAX);
174 let entry = opcode::Read::new(types::Fd(file.as_raw_fd()), buf[done..].as_mut_ptr(), chunk)
175 .offset(range.start + done as u64)
176 .build();
177 let n = submit_one(ring, &entry, seq)?;
178 if n == 0 {
179 return Err(io::Error::new(
180 io::ErrorKind::UnexpectedEof,
181 format!(
182 "short read: {done} of {len} bytes at offset {}",
183 range.start
184 ),
185 ));
186 }
187 done += usize::try_from(n).unwrap_or(0);
188 }
189 Ok(Bytes::from(buf))
190}
191
192fn write_at(
193 ring: &mut IoUring,
194 seq: &mut u64,
195 file: &File,
196 data: &[u8],
197 offset: u64,
198) -> io::Result<()> {
199 let mut done = 0usize;
200 while done < data.len() {
201 let chunk = u32::try_from(data.len() - done).unwrap_or(u32::MAX);
202 let entry = opcode::Write::new(types::Fd(file.as_raw_fd()), data[done..].as_ptr(), chunk)
203 .offset(offset + done as u64)
204 .build();
205 let n = submit_one(ring, &entry, seq)?;
206 if n == 0 {
207 return Err(io::Error::new(
208 io::ErrorKind::WriteZero,
209 "io_uring wrote zero bytes",
210 ));
211 }
212 done += usize::try_from(n).unwrap_or(0);
213 }
214 Ok(())
215}
216
217fn e_tag(metadata: &std::fs::Metadata) -> String {
224 let mtime = metadata
225 .modified()
226 .ok()
227 .and_then(|mtime| mtime.duration_since(UNIX_EPOCH).ok())
228 .unwrap_or_default()
229 .as_micros();
230 format!("{:x}-{mtime:x}-{:x}", metadata.ino(), metadata.len())
231}
232
233fn new_staged_upload(path: &std::path::Path) -> io::Result<(File, PathBuf)> {
237 if let Some(parent) = path.parent() {
238 std::fs::create_dir_all(parent)?;
239 }
240 let mut n = 1u64;
241 loop {
242 let mut staged = path.as_os_str().to_owned();
243 staged.push(format!("#{n}"));
244 let staged = PathBuf::from(staged);
245 match OpenOptions::new()
248 .read(true)
249 .write(true)
250 .create_new(true)
251 .open(&staged)
252 {
253 Ok(file) => return Ok((file, staged)),
254 Err(e) if e.kind() == io::ErrorKind::AlreadyExists => n += 1,
255 Err(e) => return Err(e),
256 }
257 }
258}
259
260fn write_staged(
272 ring: &mut IoUring,
273 seq: &mut u64,
274 path: &std::path::Path,
275 payload: &PutPayload,
276) -> io::Result<String> {
277 let (file, staged) = new_staged_upload(path)?;
278 let written = publish_staged(ring, seq, file, &staged, path, payload);
279 if written.is_err() {
280 let _ = std::fs::remove_file(&staged);
283 }
284 written
285}
286
287fn publish_staged(
288 ring: &mut IoUring,
289 seq: &mut u64,
290 file: File,
291 staged: &std::path::Path,
292 path: &std::path::Path,
293 payload: &PutPayload,
294) -> io::Result<String> {
295 let mut offset = 0u64;
296 for part in payload.iter() {
297 write_at(ring, seq, &file, part, offset)?;
298 offset += part.len() as u64;
299 }
300 let tag = e_tag(&file.metadata()?);
304 drop(file);
307 std::fs::rename(staged, path)?;
308 Ok(tag)
309}
310
311fn worker(mut rx: mpsc::Receiver<Job>, mut ring: IoUring) {
312 let mut seq = 0u64;
316 while let Some(job) = rx.blocking_recv() {
317 match job {
318 Job::Read { file, range, reply } => {
319 let _ = reply.send(read_range(&mut ring, &mut seq, &file, range));
320 }
321 Job::Write {
322 path,
323 payload,
324 reply,
325 } => {
326 let _ = reply.send(write_staged(&mut ring, &mut seq, &path, &payload));
327 }
328 }
329 }
330}
331
332async fn ring_read(
334 tx: &mpsc::Sender<Job>,
335 location: &Path,
336 file: Arc<File>,
337 range: Range<u64>,
338) -> Result<Bytes> {
339 let (reply, rx) = oneshot::channel();
340 tx.send(Job::Read { file, range, reply })
341 .await
342 .map_err(|_| Error::Generic {
343 store: STORE,
344 source: "io_uring worker thread has exited".into(),
345 })?;
346 let result = rx.await.map_err(|_| Error::Generic {
347 store: STORE,
348 source: "io_uring worker dropped the reply".into(),
349 })?;
350 result.map_err(|e| UringLocalFileSystem::io_error(location, e))
351}
352
353struct Opened {
356 file: Arc<File>,
357 meta: ObjectMeta,
358 range: Range<u64>,
359 attributes: Attributes,
360}
361
362pub struct UringLocalFileSystem {
364 inner: Arc<LocalFileSystem>,
365 tx: mpsc::Sender<Job>,
366}
367
368impl UringLocalFileSystem {
369 pub fn try_new(inner: LocalFileSystem, entries: u32) -> std::result::Result<Self, EngineError> {
372 let ring = IoUring::new(entries.max(2)).map_err(|e| {
373 EngineError::Io(io::Error::new(
374 e.kind(),
375 format!("io_uring is unavailable on this machine ({e}); use LocalFileSystem"),
376 ))
377 })?;
378 let (tx, rx) = mpsc::channel(JOB_QUEUE);
379 std::thread::Builder::new()
380 .name("oxide-io-uring".into())
381 .spawn(move || worker(rx, ring))?;
382 Ok(Self {
383 inner: Arc::new(inner),
384 tx,
385 })
386 }
387
388 pub fn probe() -> io::Result<()> {
390 IoUring::new(2).map(|_| ())
391 }
392
393 fn filesystem_path(&self, location: &Path) -> Result<PathBuf> {
394 self.inner.path_to_filesystem(location)
395 }
396
397 fn io_error(location: &Path, err: io::Error) -> Error {
398 if err.kind() == io::ErrorKind::NotFound {
399 Error::NotFound {
400 path: location.to_string(),
401 source: Box::new(err),
402 }
403 } else {
404 Error::Generic {
405 store: STORE,
406 source: Box::new(err),
407 }
408 }
409 }
410
411 async fn open(&self, location: &Path, options: GetOptions) -> Result<Opened> {
423 let GetResult {
424 payload,
425 meta,
426 range,
427 attributes,
428 } = self.inner.get_opts(location, options).await?;
429 match payload {
430 GetResultPayload::File(file, _) => Ok(Opened {
431 file: Arc::new(file),
432 meta,
433 range,
434 attributes,
435 }),
436 GetResultPayload::Stream(_) => Err(Error::Generic {
437 store: STORE,
438 source: "LocalFileSystem answered a get with a stream rather than a file".into(),
439 }),
440 }
441 }
442
443 async fn read(&self, location: &Path, file: Arc<File>, range: Range<u64>) -> Result<Bytes> {
444 ring_read(&self.tx, location, file, range).await
445 }
446
447 fn chunks(
451 &self,
452 location: &Path,
453 file: Arc<File>,
454 range: Range<u64>,
455 ) -> BoxStream<'static, Result<Bytes>> {
456 let tx = self.tx.clone();
457 let location = location.clone();
458 let end = range.end;
459 stream::try_unfold(range.start, move |offset| {
460 let (tx, location, file) = (tx.clone(), location.clone(), Arc::clone(&file));
461 async move {
462 if offset >= end {
463 return Ok(None);
464 }
465 let next = end.min(offset.saturating_add(STREAM_CHUNK));
466 let chunk = ring_read(&tx, &location, file, offset..next).await?;
467 Ok(Some((chunk, next)))
468 }
469 })
470 .boxed()
471 }
472
473 async fn write(&self, location: &Path, payload: PutPayload) -> Result<String> {
475 let path = self.filesystem_path(location)?;
476 let (reply, rx) = oneshot::channel();
477 self.tx
478 .send(Job::Write {
479 path,
480 payload,
481 reply,
482 })
483 .await
484 .map_err(|_| Error::Generic {
485 store: STORE,
486 source: "io_uring worker thread has exited".into(),
487 })?;
488 let result = rx.await.map_err(|_| Error::Generic {
489 store: STORE,
490 source: "io_uring worker dropped the reply".into(),
491 })?;
492 result.map_err(|e| Self::io_error(location, e))
493 }
494}
495
496impl fmt::Debug for UringLocalFileSystem {
497 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
498 f.debug_struct(STORE).field("inner", &self.inner).finish()
499 }
500}
501
502impl fmt::Display for UringLocalFileSystem {
503 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
504 write!(f, "{STORE}({})", self.inner)
505 }
506}
507
508#[async_trait]
509impl ObjectStore for UringLocalFileSystem {
510 async fn put_opts(
511 &self,
512 location: &Path,
513 payload: PutPayload,
514 opts: PutOptions,
515 ) -> Result<PutResult> {
516 match opts.mode {
517 PutMode::Overwrite => {
518 if !opts.attributes.is_empty() {
522 return Err(Error::NotImplemented {
523 operation: "`put_opts` with `opts.attributes` specified".into(),
524 implementer: self.to_string(),
525 });
526 }
527 let e_tag = self.write(location, payload).await?;
528 Ok(PutResult {
529 e_tag: Some(e_tag),
530 version: None,
531 })
532 }
533 _ => self.inner.put_opts(location, payload, opts).await,
536 }
537 }
538
539 async fn put_multipart_opts(
540 &self,
541 location: &Path,
542 opts: PutMultipartOptions,
543 ) -> Result<Box<dyn MultipartUpload>> {
544 self.inner.put_multipart_opts(location, opts).await
545 }
546
547 async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
548 let head = options.head;
552 let Opened {
553 file,
554 meta,
555 range,
556 attributes,
557 } = self.open(location, options).await?;
558 let payload = if head {
559 stream::empty().boxed()
560 } else {
561 self.chunks(location, file, range.clone())
562 };
563 Ok(GetResult {
564 payload: GetResultPayload::Stream(payload),
565 meta,
566 range,
567 attributes,
568 })
569 }
570
571 async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
572 let Opened { file, .. } = self.open(location, GetOptions::default()).await?;
576 let mut out = Vec::with_capacity(ranges.len());
577 for range in ranges {
578 out.push(
579 self.read(location, Arc::clone(&file), range.clone())
580 .await?,
581 );
582 }
583 Ok(out)
584 }
585
586 fn delete_stream(
587 &self,
588 locations: BoxStream<'static, Result<Path>>,
589 ) -> BoxStream<'static, Result<Path>> {
590 self.inner.delete_stream(locations)
591 }
592
593 fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
594 self.inner.list(prefix)
595 }
596
597 fn list_with_offset(
598 &self,
599 prefix: Option<&Path>,
600 offset: &Path,
601 ) -> BoxStream<'static, Result<ObjectMeta>> {
602 self.inner.list_with_offset(prefix, offset)
603 }
604
605 async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
606 self.inner.list_with_delimiter(prefix).await
607 }
608
609 async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
610 self.inner.copy_opts(from, to, options).await
611 }
612
613 async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> {
614 self.inner.rename_opts(from, to, options).await
615 }
616}
617
618#[cfg(test)]
619#[allow(clippy::unwrap_used, clippy::expect_used)]
620mod tests {
621 use super::*;
622
623 extern "C" fn noop(_: libc::c_int) {}
629
630 fn install_interrupting_handler() {
631 unsafe {
634 let mut action: libc::sigaction = std::mem::zeroed();
635 action.sa_sigaction = noop as *const () as usize;
636 action.sa_flags = 0;
637 libc::sigemptyset(&mut action.sa_mask);
638 assert_eq!(
639 libc::sigaction(libc::SIGUSR1, &action, std::ptr::null_mut()),
640 0,
641 "could not install the SIGUSR1 handler: {}",
642 io::Error::last_os_error()
643 );
644 }
645 }
646
647 #[test]
661 fn a_signal_during_the_wait_does_not_abandon_the_completion() {
662 let Ok(mut ring) = IoUring::new(4) else {
663 eprintln!("SKIPPED: io_uring unavailable in this environment");
664 return;
665 };
666 install_interrupting_handler();
667
668 let mut fds = [0 as libc::c_int; 2];
670 assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0);
671 let (read_fd, write_fd) = (fds[0], fds[1]);
672
673 let tid = unsafe { libc::syscall(libc::SYS_gettid) } as libc::pid_t;
675 let pid = unsafe { libc::getpid() };
676
677 let signaller = std::thread::spawn(move || {
678 std::thread::sleep(std::time::Duration::from_millis(50));
680 for _ in 0..5 {
681 unsafe { libc::syscall(libc::SYS_tgkill, pid, tid, libc::SIGUSR1) };
683 std::thread::sleep(std::time::Duration::from_millis(10));
684 }
685 let payload = b"uring-ok";
688 unsafe { libc::write(write_fd, payload.as_ptr().cast(), payload.len()) };
690 });
691
692 let mut buf = [0u8; 8];
693 let entry = opcode::Read::new(types::Fd(read_fd), buf.as_mut_ptr(), 8).build();
694 let mut seq = 0u64;
695 let n = submit_one(&mut ring, &entry, &mut seq)
696 .expect("a signal during the wait is not a failure: the SQE is still in flight");
697
698 signaller.join().unwrap();
699 assert_eq!(n, 8, "the whole payload was read");
700 assert_eq!(&buf, b"uring-ok", "the bytes are the ones written");
701 assert_eq!(seq, 1, "the submission was tagged");
702
703 unsafe {
705 libc::close(read_fd);
706 libc::close(write_fd);
707 }
708 }
709
710 #[test]
714 fn each_submission_is_tagged_with_a_fresh_value() {
715 let Ok(mut ring) = IoUring::new(4) else {
716 eprintln!("SKIPPED: io_uring unavailable in this environment");
717 return;
718 };
719 let dir = tempfile::tempdir().unwrap();
720 let path = dir.path().join("tagged.bin");
721 std::fs::write(&path, b"0123456789").unwrap();
722 let file = File::open(&path).unwrap();
723
724 let mut seq = 0u64;
725 for expected in 1..=3u64 {
726 let mut buf = [0u8; 10];
727 let entry =
728 opcode::Read::new(types::Fd(file.as_raw_fd()), buf.as_mut_ptr(), 10).build();
729 let n = submit_one(&mut ring, &entry, &mut seq).unwrap();
730 assert_eq!(n, 10);
731 assert_eq!(&buf, b"0123456789");
732 assert_eq!(seq, expected, "each submission takes the next tag");
733 }
734 }
735}