virtiofsd 1.13.3

A virtio-fs vhost-user device daemon
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE-BSD-3-Clause file.

use crate::fuse;
use crate::passthrough::device_state::preserialization::InodeMigrationInfo;
use crate::passthrough::file_handle::{FileHandle, FileOrHandle};
use crate::passthrough::stat::MountId;
use crate::passthrough::util::{
    ebadf, get_path_by_fd, is_safe_inode, reopen_fd_through_proc, FdPathError,
};
use crate::util::other_io_error;
use std::collections::BTreeMap;
use std::ffi::CString;
use std::fs::File;
use std::ops::Deref;
use std::os::unix::io::{AsRawFd, RawFd};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use std::{fmt, io};

pub type Inode = u64;

#[derive(Clone, Copy, Default, Eq, Ord, PartialEq, PartialOrd)]
pub(crate) struct InodeIds {
    pub ino: libc::ino64_t,
    pub dev: libc::dev_t,
    pub mnt_id: MountId,
}

/// Strong reference to some inode in our inode store, which is counted against the
/// `InodeData.refcount` field.  Dropping this object will thus decrement that refcount, and
/// potentially remove the inode from the store (when the refcount reaches 0).
/// Note that dropping this object locks its inode store, so care must be taken not to drop strong
/// references while the inode store is locked, or to use `StrongInodeReference::drop_unlocked()`.
pub(crate) struct StrongInodeReference {
    /// Referenced inode's data.
    /// Is only `None` after the inode has been leaked, which cannot occur outside of `leak()` and
    /// `drop()`, because `leak()` consumes the object.
    inode_data: Option<Arc<InodeData>>,

    /// Inode store that holds the referenced inode.
    inode_store: Arc<RwLock<InodeStoreInner>>,
}

pub(crate) struct InodeData {
    pub inode: Inode,
    // Most of these aren't actually files but ¯\_(ツ)_/¯.
    pub file_or_handle: FileOrHandle,
    pub refcount: AtomicU64,

    // Used as key in the `InodeStoreInner::by_ids` map.
    pub ids: InodeIds,

    // File type and mode
    pub mode: u32,

    // Constructed in the `prepare_serialization` phase of migration, and must be set on all inodes
    // when we are actually going to serialize our internal state to send it to the migration
    // destination.
    // Because this may contain a strong inode reference, which must not be dropped while the inode
    // store is locked, this info must in turn not be dropped while the store is locked.
    // To ensure this, locking of the store is only done here in this file, and here we ensure that
    // while the store is locked, `InodeMigrationInfo` (e.g. as part of an `InodeData`) is dropped
    // only by using `drop_unlocked()` for a potentially contained strong reference.
    pub(super) migration_info: Mutex<Option<InodeMigrationInfo>>,
}

/**
 * Represents the file associated with an inode (`InodeData`).
 *
 * When obtaining such a file, it may either be a new file (the `Owned` variant), in which case the
 * object's lifetime is static, or it may reference `InodeData.file` (the `Ref` variant), in which
 * case the object's lifetime is that of the respective `InodeData` object.
 */
pub(crate) enum InodeFile<'inode_lifetime> {
    Owned(File),
    Ref(&'inode_lifetime File),
}

#[derive(Default)]
struct InodeStoreInner {
    data: BTreeMap<Inode, Arc<InodeData>>,
    by_ids: BTreeMap<InodeIds, Inode>,
    by_handle: BTreeMap<FileHandle, Inode>,
}

#[derive(Default)]
pub(crate) struct InodeStore {
    inner: Arc<RwLock<InodeStoreInner>>,
}

/**
 * Iterates over the inode store.
 *
 * Does not keep the store locked between `next()` calls, and will return inodes added while
 * iterating.
 */
pub(crate) struct InodeIterator<'a> {
    /// Inode store.
    store: &'a InodeStore,

    /**
     * Last inode ID returned through `next()`.
     *
     * We visit inodes in numerical order of their ID, and because new IDs added to the store are
     * always greater than all previous IDs, all remaining IDs to visit must be greater than this
     * one.
     */
    last_inode: Option<Inode>,
}

/**
 * Errors that `InodeData::get_path()` can encounter.
 *
 * This specialized error type exists so that
 * [`crate::passthrough::device_state::preserialization::proc_paths`] can decide which errors it
 * considers recoverable.
 */
#[derive(Debug)]
pub(crate) enum InodePathError {
    /// Failed to get an FD for this inode.
    NoFd(io::Error),

    /// `util::get_path_by_fd()` failed.
    FdPathError(FdPathError),

    /// Path reported by `util::get_path_by_fd()` is outside of the shared directory.
    OutsideRoot,
}

impl<'a> InodeData {
    /// Get an `O_PATH` file for this inode
    pub fn get_file(&'a self) -> io::Result<InodeFile<'a>> {
        match &self.file_or_handle {
            FileOrHandle::File(f) => Ok(InodeFile::Ref(f.get_file())),
            FileOrHandle::Handle(h) => {
                let file = h.open(libc::O_PATH)?;
                Ok(InodeFile::Owned(file))
            }
            FileOrHandle::Invalid(err) => Err(io::Error::new(
                err.kind(),
                format!("Inode is invalid because of an error during the preceding migration, which was: {err}"),
            )),
        }
    }

    /// Try to obtain this inode's path through /proc/self/fd
    pub fn get_path(&self, proc_self_fd: &File) -> Result<CString, InodePathError> {
        let fd = self.get_file().map_err(InodePathError::NoFd)?;
        let path = get_path_by_fd(&fd, proc_self_fd).map_err(InodePathError::FdPathError)?;

        // Kernel will report nodes beyond our root as having path / -- but only the root node (the
        // shared directory) can actually have that path, so for others, it must be inaccurate
        if path.as_bytes() == b"/" && self.inode != fuse::ROOT_ID {
            return Err(InodePathError::OutsideRoot);
        }

        Ok(path)
    }

    /// Open this inode with the given flags
    /// (always returns a new (i.e. `Owned`) file, hence the static lifetime)
    pub fn open_file(
        &self,
        flags: libc::c_int,
        proc_self_fd: &File,
    ) -> io::Result<InodeFile<'static>> {
        // Do not move the `is_safe_inode()` check up: It is always false for invalid inodes, so
        // would hide their perfectly good error message
        match &self.file_or_handle {
            FileOrHandle::File(f) => {
                if !is_safe_inode(self.mode) {
                    return Err(ebadf());
                }
                let new_file = reopen_fd_through_proc(f, flags, proc_self_fd)?;
                Ok(InodeFile::Owned(new_file))
            }
            FileOrHandle::Handle(h) => {
                if !is_safe_inode(self.mode) {
                    return Err(ebadf());
                }
                let new_file = h.open(flags)?;
                Ok(InodeFile::Owned(new_file))
            }
            FileOrHandle::Invalid(err) => Err(io::Error::new(
                err.kind(),
                format!("Inode is invalid because of an error during the preceding migration, which was: {err}"),
            )),
        }
    }

    /// Return some human-readable identification of this inode, ideally the path.  Will perform
    /// I/O, so is not extremely cheap to call.
    pub fn identify(&self, proc_self_fd: &File) -> String {
        if let Ok(path) = self.get_path(proc_self_fd) {
            path.to_string_lossy().to_string()
        } else {
            let mode = match self.mode & libc::S_IFMT {
                libc::S_IFREG => "file",
                libc::S_IFDIR => "directory",
                libc::S_IFLNK => "symbolic link",
                libc::S_IFIFO => "FIFO",
                libc::S_IFSOCK => "socket",
                libc::S_IFCHR => "character device",
                libc::S_IFBLK => "block device",
                _ => "unknown inode type",
            };
            format!(
                "[{mode}; mount_id={} device_id={} inode_id={}]",
                self.ids.mnt_id, self.ids.dev, self.ids.ino,
            )
        }
    }
}

impl InodeFile<'_> {
    /// Create a standalone `File` object
    pub fn into_file(self) -> io::Result<File> {
        match self {
            Self::Owned(file) => Ok(file),
            Self::Ref(file_ref) => file_ref.try_clone(),
        }
    }
}

impl AsRawFd for InodeFile<'_> {
    /// Return a file descriptor for this file
    /// Note: This fd is only valid as long as the `InodeFile` exists.
    fn as_raw_fd(&self) -> RawFd {
        match self {
            Self::Owned(file) => file.as_raw_fd(),
            Self::Ref(file_ref) => file_ref.as_raw_fd(),
        }
    }
}

impl InodeStoreInner {
    /// Insert a new entry into the inode store.  Panics if the entry already existed.
    /// (This guarantees that inserting a value will not drop an existing `InodeMigrationInfo`
    /// object.)
    fn insert_new(&mut self, data: Arc<InodeData>) {
        // Overwriting something in `by_ids` or `by_handle` is not exactly what we want, but having
        // the same physical inode under several different FUSE IDs is not catastrophic, so do not
        // panic about that.
        self.by_ids.insert(data.ids, data.inode);
        if let FileOrHandle::Handle(handle) = &data.file_or_handle {
            self.by_handle.insert(handle.inner().clone(), data.inode);
        }
        let existing = self.data.insert(data.inode, data);
        assert!(existing.is_none());
    }

    /// Remove the given inode, and, if found, take care to drop any associated strong reference in
    /// the migration info via `drop_unlocked()`.
    fn remove(&mut self, inode: Inode) {
        let data = self.data.remove(&inode);
        if let Some(data) = data {
            if let FileOrHandle::Handle(handle) = &data.file_or_handle {
                self.by_handle.remove(handle.inner());
            }
            self.by_ids.remove(&data.ids);
            if let Some(mig_info) = data.migration_info.lock().unwrap().take() {
                mig_info.for_each_strong_reference(|strong_ref| strong_ref.drop_unlocked(self));
            }
        }
    }

    fn clear(&mut self) {
        self.clear_migration_info();
        self.data.clear();
        self.by_handle.clear();
        self.by_ids.clear();
    }

    /// Clears all migration info, using `drop_unlocked()` to drop any strong references within.
    fn clear_migration_info(&mut self) {
        let mut strong_references = Vec::<StrongInodeReference>::new();
        for inode in self.data.values() {
            if inode.inode == fuse::ROOT_ID {
                // Ignore root inode, we always want to keep its migration info around
                continue;
            }

            if let Some(mig_info) = inode.migration_info.lock().unwrap().take() {
                mig_info.for_each_strong_reference(|strong_ref| strong_references.push(strong_ref));
            }
        }
        for strong_reference in strong_references {
            strong_reference.drop_unlocked(self);
        }
    }

    fn get(&self, inode: Inode) -> Option<&Arc<InodeData>> {
        self.data.get(&inode)
    }

    fn get_by_ids(&self, ids: &InodeIds) -> Option<&Arc<InodeData>> {
        self.inode_by_ids(ids).map(|inode| self.get(inode).unwrap())
    }

    fn get_by_handle(&self, handle: &FileHandle) -> Option<&Arc<InodeData>> {
        self.inode_by_handle(handle)
            .map(|inode| self.get(inode).unwrap())
    }

    fn contains(&self, inode: Inode) -> bool {
        self.data.contains_key(&inode)
    }

    fn inode_by_ids(&self, ids: &InodeIds) -> Option<Inode> {
        self.by_ids.get(ids).copied()
    }

    fn inode_by_handle(&self, handle: &FileHandle) -> Option<Inode> {
        self.by_handle.get(handle).copied()
    }

    fn is_empty(&self) -> bool {
        self.data.is_empty()
    }

    /// Decrement the refcount of the given `inode` ID, and remove it from the store when it
    /// reaches 0
    fn forget_one(&mut self, inode: Inode, count: u64) {
        if let Some(data) = self.get(inode) {
            // Having a mutable reference on `self` prevents concurrent lookups from incrementing
            // the refcount but there is the possibility that a previous lookup already acquired a
            // reference to the inode data and is in the process of updating the refcount so we
            // need to loop here until we can decrement successfully.
            loop {
                let refcount = data.refcount.load(Ordering::Relaxed);

                // Saturating sub because it doesn't make sense for a refcount to go below zero and
                // we don't want misbehaving clients to cause integer overflow.
                let new_count = refcount.saturating_sub(count);

                // We don't need any stronger ordering, because the refcount itself doesn't protect
                // any data.
                if data.refcount.compare_exchange(
                    refcount,
                    new_count,
                    Ordering::Relaxed,
                    Ordering::Relaxed,
                ) == Ok(refcount)
                {
                    if new_count == 0 {
                        // We just removed the last refcount for this inode. There's no need for an
                        // acquire fence here because we have a mutable reference on `self`. So
                        // there's is no other release store for us to synchronize with before
                        // deleting the entry.
                        self.remove(inode);
                    }
                    break;
                }
            }
        }
    }
}

impl InodeStore {
    pub fn get(&self, inode: Inode) -> Option<Arc<InodeData>> {
        self.inner.read().unwrap().get(inode).cloned()
    }

    /**
     * Iterate over every inode that we have in the store.
     *
     * Does not keep the store locked between `next()` calls, and will return inodes added while
     * iterating.
     */
    pub fn iter(&self) -> InodeIterator<'_> {
        InodeIterator {
            store: self,
            last_inode: None,
        }
    }

    /// Turn the weak reference `inode` into a strong one (increments its refcount)
    pub fn get_strong(&self, inode: Inode) -> io::Result<StrongInodeReference> {
        StrongInodeReference::new(inode, self)
    }

    /// Attempt to get an inode from `inodes` and create a strong reference to it, i.e. increment
    /// its refcount.  Return that reference on success, and an error on failure.
    /// Reasons for failure can be that the inode isn't in the map or that the refcount is zero.
    /// This function will never increment a refcount that's already zero.
    /// Note that dropping the returned strong reference will automatically decrement the refcount
    /// again.
    pub fn claim_inode(
        &self,
        handle: Option<&FileHandle>,
        ids: &InodeIds,
    ) -> io::Result<StrongInodeReference> {
        self.do_claim_inode(&self.inner.read().unwrap(), handle, ids)
    }

    fn do_claim_inode<I: Deref<Target = InodeStoreInner>>(
        &self,
        inner: &I,
        handle: Option<&FileHandle>,
        ids: &InodeIds,
    ) -> io::Result<StrongInodeReference> {
        let data = handle
            .and_then(|h| inner.get_by_handle(h))
            .or_else(|| {
                inner.get_by_ids(ids).filter(|data| {
                    // When we have to fall back to looking up an inode by its inode ID, ensure
                    // that we hit an entry that has a valid file descriptor.  Having an FD open
                    // means that the inode cannot really be deleted until the FD is closed, so
                    // that the inode ID remains valid until we evict the `InodeData`.  With no FD
                    // open (and just a file handle), the inode can be deleted while we still have
                    // our `InodeData`, and so the inode ID may be reused by a completely different
                    // new inode.  Such inodes must be looked up by file handle, because this
                    // handle contains a generation ID to differentiate between the old and the new
                    // inode.
                    matches!(data.file_or_handle, FileOrHandle::File(_))
                })
            })
            .ok_or_else(|| {
                io::Error::new(
                    io::ErrorKind::NotFound,
                    "Cannot take strong reference to inode by handle or IDs, not found".to_string(),
                )
            })?;

        StrongInodeReference::new_with_data(Arc::clone(data), self)
    }

    /// Check whether a matching inode is already present (see `claim_inode`), and if so, return
    /// that inode and drop `inode_data`.
    /// Otherwise, insert `inode_data`, and return a strong reference to it.  `inode_data.refcount`
    /// is ignored; the returned strong reference is the only one that can exist, so the refcount
    /// is hard-set to 1.
    pub fn get_or_insert(&self, mut inode_data: InodeData) -> io::Result<StrongInodeReference> {
        let mut inner = self.inner.write().unwrap();
        let handle = match &inode_data.file_or_handle {
            FileOrHandle::File(_) => None,
            FileOrHandle::Handle(handle) => Some(handle.inner()),
            FileOrHandle::Invalid(_) => None,
        };
        if let Ok(inode) = self.do_claim_inode(&inner, handle, &inode_data.ids) {
            // `InodeData`s should not be dropped while the inode store is locked, so drop the lock
            // before `inode_data`
            drop(inner);
            return Ok(inode);
        }
        if inner.contains(inode_data.inode) {
            // `InodeData`s should not be dropped while the inode store is locked, so drop the lock
            // before `inode_data`
            drop(inner);
            return Err(other_io_error(format!(
                "Double-use of FUSE inode ID {}",
                inode_data.inode
            )));
        }

        // Safe because we have the only reference
        inode_data.refcount = AtomicU64::new(1);
        let inode_data = Arc::new(inode_data);
        inner.insert_new(Arc::clone(&inode_data));

        // We just set the reference to 1 to account for this
        Ok(unsafe { StrongInodeReference::new_no_increment(inode_data, self) })
    }

    /// Insert `inode_data` into the inode store regardless of whether a matching inode already
    /// exists.  However, if the given inode ID already exists, return an error and drop
    /// `inode_data.`
    pub fn new_inode(&self, inode_data: InodeData) -> io::Result<()> {
        let mut inner = self.inner.write().unwrap();
        if inner.contains(inode_data.inode) {
            // `InodeData`s should not be dropped while the inode store is locked, so drop the lock
            // before `inode_data`
            drop(inner);
            return Err(other_io_error(format!(
                "Double-use of FUSE inode ID {}",
                inode_data.inode
            )));
        }
        inner.insert_new(Arc::new(inode_data));
        Ok(())
    }

    pub fn forget_one(&self, inode: Inode, count: u64) {
        self.inner.write().unwrap().forget_one(inode, count);
    }

    pub fn forget_many<I: IntoIterator<Item = (Inode, u64)>>(&self, inodes: I) {
        let mut inner = self.inner.write().unwrap();
        for (inode, count) in inodes {
            inner.forget_one(inode, count);
        }
    }

    pub fn clear(&self) {
        self.inner.write().unwrap().clear();
    }

    pub fn clear_migration_info(&self) {
        self.inner.write().unwrap().clear_migration_info();
    }

    pub fn is_empty(&self) -> bool {
        self.inner.read().unwrap().is_empty()
    }
}

impl StrongInodeReference {
    /// Create a new strong reference to the given inode in the given inode store, incrementing the
    /// refcount appropriately.
    pub fn new(inode: Inode, inode_store: &InodeStore) -> io::Result<Self> {
        let inode_data = inode_store.get(inode).ok_or_else(|| {
            io::Error::new(
                io::ErrorKind::NotFound,
                format!("Cannot take strong reference to inode {inode}: Not found"),
            )
        })?;

        Self::new_with_data(inode_data, inode_store)
    }

    /// Create a new strong reference to an inode with the given data from the given inode store,
    /// incrementing the refcount appropriately.
    pub fn new_with_data(inode_data: Arc<InodeData>, inode_store: &InodeStore) -> io::Result<Self> {
        Self::increment_refcount_for(&inode_data)?;

        // Safe because we have just incremented the refcount
        Ok(unsafe { StrongInodeReference::new_no_increment(inode_data, inode_store) })
    }

    /// Create a new strong reference to an inode with the given data from the given inode store,
    /// but do not increment the inode's refcount, and instead assume that the caller has already
    /// done it.
    ///
    /// # Safety
    /// Caller ensures the inode's refcount is incremented by 1 to account for this strong
    /// reference.
    pub unsafe fn new_no_increment(inode_data: Arc<InodeData>, inode_store: &InodeStore) -> Self {
        StrongInodeReference {
            inode_data: Some(inode_data),
            inode_store: Arc::clone(&inode_store.inner),
        }
    }

    /// Tries to increment the refcount in the given `inode_data`, but will refuse to increment a
    /// refcount that is 0 (because in this case, the inode is already in the process of being
    /// removed from the store, so continuing to use it would not be safe).
    fn increment_refcount_for(inode_data: &InodeData) -> io::Result<()> {
        // Use `.fetch_update()` instead of `.fetch_add()` to ensure we never increment the
        // refcount from zero to one.
        match inode_data
            .refcount
            .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |rc| {
                (rc > 0).then_some(rc + 1)
            }) {
            Ok(_old_rc) => Ok(()),
            Err(_old_rc) => Err(io::Error::new(
                io::ErrorKind::NotFound,
                format!(
                    "Cannot take strong reference to inode {}: Is already deleted",
                    inode_data.inode
                ),
            )),
        }
    }

    /// Consume this strong reference, yield the underlying inode ID, without decrementing the
    /// inode's refcount.
    ///
    /// # Safety
    /// Caller must guarantee that the refcount is tracked somehow still, i.e. that forget_one()
    /// will eventually be called.  Otherwise, this inode will be truly leaked, which generally is
    /// not good.
    pub unsafe fn leak(mut self) -> Inode {
        // Unwrapping is safe: Every initializer sets this to `Some(_)`, and every function that
        // `take()`s the value (`leak()`, `drop_unlocked()`, `drop()`) also consumes `self`, so
        // outside of them, this must always be `None`.
        self.inode_data.take().unwrap().inode
    }

    /// Yield the underlying inode ID.
    ///
    /// # Safety
    /// The inode ID is technically a form of a weak reference.  To ensure safety, the caller may
    /// not assume that it is valid beyond the lifetime of the corresponding strong reference.
    pub unsafe fn get_raw(&self) -> Inode {
        // Unwrapping is safe: Every initializer sets this to `Some(_)`, and every function that
        // `take()`s the value (`leak()`, `drop_unlocked()`, `drop()`) also consumes `self`, so
        // outside of them, this must always be `None`.
        self.inode_data.as_ref().unwrap().inode
    }

    /// Get the associated inode data.
    pub fn get(&self) -> &InodeData {
        // Unwrapping is safe: Every initializer sets this to `Some(_)`, and every function that
        // `take()`s the value (`leak()`, `drop_unlocked()`, `drop()`) also consumes `self`, so
        // outside of them, this must always be `None`.
        self.inode_data.as_ref().unwrap()
    }

    /// This function allows dropping a `StrongInodeReference` while the inode store is locked, but
    /// the caller must have mutable access to the inode store.
    fn drop_unlocked(mut self, inodes: &mut InodeStoreInner) {
        if let Some(inode_data) = self.inode_data.take() {
            inodes.forget_one(inode_data.inode, 1);
        }
    }
}

impl Clone for StrongInodeReference {
    /// Create an additional strong reference.
    fn clone(&self) -> Self {
        // Unwrapping is safe: Every initializer sets this to `Some(_)`, and every function that
        // `take()`s the value (`leak()`, `drop_unlocked()`, `drop()`) also consumes `self`, so
        // outside of them, this must always be `None`.
        let cloned_data = Arc::clone(self.inode_data.as_ref().unwrap());
        let cloned_store = Arc::clone(&self.inode_store);

        // Unwrapping is safe, because this can only fail if the refcount became 0, which is
        // impossible because `self` is a strong reference
        Self::increment_refcount_for(&cloned_data).unwrap();

        StrongInodeReference {
            inode_data: Some(cloned_data),
            inode_store: cloned_store,
        }
    }
}

impl Drop for StrongInodeReference {
    /// Decrement the refcount on the referenced inode, removing it from the store when the
    /// refcount reaches 0.
    /// Note that this function locks `self.inode_store`, so a `StrongInodeReference` must not be
    /// dropped while that inode store is locked.  In such a case,
    /// `StrongInodeReference::drop_unlocked()` must be used.
    fn drop(&mut self) {
        if let Some(inode_data) = self.inode_data.take() {
            self.inode_store
                .write()
                .unwrap()
                .forget_one(inode_data.inode, 1);
        }
    }
}

impl Drop for InodeStore {
    /// Explicitly clear the inner inode store on drop, because there may be circular references
    /// within (in the migration info's strong references) that may otherwise prevent the
    /// `InodeStoreInner` from being dropped.
    fn drop(&mut self) {
        self.inner.write().unwrap().clear();
    }
}

impl Iterator for InodeIterator<'_> {
    type Item = Arc<InodeData>;

    fn next(&mut self) -> Option<Arc<InodeData>> {
        let store = self.store.inner.read().unwrap();

        // Find the inode with the lowest ID after `last_inode`.
        // Note that iterators over `BTreeMap` return keys in numerical order, so
        // `range(x..).next()` will always return the inode with the lowest ID greater than or
        // equal to `x` (if any).
        let lower_bound = self.last_inode.map(|last_id| last_id + 1).unwrap_or(0);
        let (inode_id, inode_data) = store.data.range(lower_bound..).next()?;

        self.last_inode = Some(*inode_id);
        Some(Arc::clone(inode_data))
    }
}

impl From<InodePathError> for io::Error {
    fn from(err: InodePathError) -> Self {
        match err {
            InodePathError::NoFd(err) => err,
            InodePathError::FdPathError(err) => err.into(),
            InodePathError::OutsideRoot => other_io_error(
                "Got empty path for non-root node, so it is outside the shared directory",
            ),
        }
    }
}

impl fmt::Display for InodePathError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            InodePathError::NoFd(err) => write!(f, "{err}"),
            InodePathError::FdPathError(err) => write!(f, "{err}"),
            InodePathError::OutsideRoot => write!(
                f,
                "Got empty path for non-root node, so it is outside the shared directory",
            ),
        }
    }
}

impl std::error::Error for InodePathError {}