squashfs_async/
lib.rs

1#![doc = include_str!("../README.md")]
2
3mod data;
4mod deser;
5pub mod directory_table;
6pub mod error;
7pub mod fragments;
8pub mod inodes;
9mod metadata;
10pub mod pools;
11mod squashfuse;
12mod superblock;
13#[doc(hidden)]
14pub mod utils;
15use error::CacheError;
16pub use error::Error;
17use fragments::FragmentsTable;
18pub use superblock::{Compression, SuperBlock};
19
20use std::collections::BTreeMap;
21use std::fmt::Write;
22use std::ops::DerefMut;
23use std::path::Path;
24
25use clap::Parser;
26use deadpool::managed::Pool;
27use fuser_async::cache::{DataBlockCache, IndexCache, LRUCache};
28use tokio::sync::RwLock;
29use tracing::*;
30
31trait_set::trait_set! {
32    /// Convenience trait alias
33    pub trait AsyncRead = tokio::io::AsyncRead + Send + Sync + std::marker::Unpin;
34    /// Convenience trait alias
35    pub trait AsyncSeekBufRead = tokio::io::AsyncSeek + tokio::io::AsyncBufRead + Send + Sync + std::marker::Unpin;
36
37    pub trait ManagerFactory<R> = Fn(pools::ReadFlags) -> Result<R, Error> + Send + Sync + 'static;
38}
39
40const TABLES_DIRECT_THRESHOLD: u64 = 50_000;
41
42/// Squashfs reading options.
43#[derive(Parser)]
44pub struct Options {
45    /// Cache size (MB) for decoded blocks.
46    #[clap(long, default_value_t = 100)]
47    pub cache_mb: u64,
48    /// Number of readers
49    #[clap(long, default_value_t = 4)]
50    pub readers: usize,
51    /// Limit (B) for reading small files with direct access.
52    ///
53    /// This is useful for example when the underlying storage is networked and buffered: for
54    /// fast access to small files, one may want to request exactly the data needed, rather than
55    /// fetching a whole chunk for buffering.
56    ///
57    /// This will use another `cache_mb` amount of cache.
58    #[clap(long, default_value_t = 0)]
59    pub direct_limit: usize,
60}
61
62/// Base structure representing a loaded SquashFS image.
63///
64/// Note that the tables (inode, directory...) are fully parsed on creation and kept in memory,
65/// rather than being accessed lazily.
66///
67/// This implements the [`fuser_async::Filesystem`] trait.
68///
69/// The type `R` is a [`deadpool`] pool manager for the underlying filesystem readers.
70/// See [`crate::pools`].
71pub struct SquashFs<R: deadpool::managed::Manager> {
72    pub superblock: superblock::SuperBlock,
73    pub inode_table: inodes::InodeTable,
74    pub fragments_table: FragmentsTable,
75    /// Table for each directory inode
76    pub directory_tables: BTreeMap<u32 /* inode */, directory_table::DirectoryTable>,
77    root_inode: u32,
78    pub handles: RwLock<BTreeMap<u64, pools::ReadFlags>>,
79    manager_factory: Box<dyn ManagerFactory<R>>,
80    readers: RwLock<BTreeMap<pools::ReadFlags, Pool<R>>>,
81    n_readers: usize,
82    inode_extra: u32,
83    /// Files smaller than this size will be accessed with the O_NONBLOCK, which allows triggering
84    /// optimizations on the storage backend (e.g. do not pre-fetch a large block for a small file).
85    /// See the documentation in [`Options`].
86    direct_limit: usize,
87    /// Cache for decoded blocks in the image
88    cache: Option<IndexCache>,
89    /// Cache for small files (< direct_limit), that are read at once.
90    small_files_cache: Option<LRUCache>,
91}
92impl<R: deadpool::managed::Manager> std::fmt::Debug for SquashFs<R> {
93    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
94        writeln!(f, "{:?}", self.superblock)?;
95        writeln!(f, "{}", self.fragments_table)?;
96        writeln!(f, "{}, root inode {}", self.inode_table, self.root_inode)?;
97        self.tree(0, self.root_inode, f)?;
98        if let Some(cache) = &self.cache {
99            writeln!(f, "{}", cache)?;
100        }
101        if let Some(cache) = &self.small_files_cache {
102            writeln!(f, "{}", cache)?;
103        }
104        Ok(())
105    }
106}
107
108impl<T, P> SquashFs<P>
109where
110    T: AsyncSeekBufRead,
111    P: pools::LocalReadersPool
112        + deadpool::managed::Manager<Type = T, Error = tokio::io::Error>
113        + Send
114        + Sync,
115{
116    /// Open squashfs image from a local file
117    pub async fn open(file: &Path, options: &Options) -> Result<Self, Error> {
118        let file = file.to_owned();
119        Self::from_reader(options, move |_| P::new(&file)).await
120    }
121}
122
123impl<R: deadpool::managed::Manager> SquashFs<R> {
124    fn tree<W: Write>(&self, level: usize, root_inode: u32, f: &mut W) -> std::fmt::Result {
125        for e in &self.directory_tables.get(&root_inode).unwrap().entries {
126            writeln!(f, "{:level$}{}", "", e, level = 4 * level)?;
127            if e.is_dir() {
128                self.tree(level + 1, e.inode, f)?;
129            }
130        }
131        Ok(())
132    }
133    pub fn inodes(&self) -> impl Iterator<Item = u32> + '_ {
134        self.inode_table
135            .files
136            .keys()
137            .chain(self.inode_table.directories.keys())
138            .copied()
139    }
140}
141
142impl<T, R> SquashFs<R>
143where
144    T: AsyncSeekBufRead,
145    R: deadpool::managed::Manager<Type = T, Error = tokio::io::Error> + Send + Sync,
146{
147    async fn get_reader(
148        &self,
149        flags: pools::ReadFlags,
150    ) -> Result<deadpool::managed::Object<R>, Error> {
151        let readers = self.readers.read().await;
152        if !readers.contains_key(&flags) {
153            drop(readers);
154            let mut readers = self.readers.write().await;
155            readers.insert(
156                flags,
157                Pool::builder((self.manager_factory)(flags)?)
158                    .max_size(self.n_readers)
159                    .build()?,
160            );
161            Ok(readers.get(&flags).unwrap().get().await?)
162        } else {
163            Ok(readers.get(&flags).unwrap().get().await?)
164        }
165    }
166    pub async fn has_handles(&self) -> bool {
167        let handles = self.handles.read().await;
168        !handles.is_empty()
169    }
170    /// Open squashfs image from a reader factory, responsible for creating readers with the
171    /// requested open flags.
172    pub async fn from_reader(
173        options: &Options,
174        manager_factory: impl ManagerFactory<R>,
175    ) -> Result<Self, Error> {
176        if options.readers == 0 {
177            return Err(Error::InvalidOptions("The number of readers must be >=1"));
178        }
179        if options.direct_limit as u64 * 10 > options.cache_mb * (1e6 as u64) {
180            return Err(Error::InvalidOptions(
181                "The cache size must be at least 10x as large as --direct-limit.",
182            ));
183        }
184        let manager_factory = Box::new(manager_factory);
185
186        let mut readers = BTreeMap::<pools::ReadFlags, Pool<R>>::default();
187        for flags in [0, libc::O_NONBLOCK] {
188            readers.insert(
189                flags,
190                Pool::builder(manager_factory(flags)?)
191                    .max_size(options.readers)
192                    .build()?,
193            );
194        }
195
196        let mut r = readers.get(&libc::O_NONBLOCK).unwrap().get().await?;
197
198        let superblock = superblock::SuperBlock::from_reader(&mut r.deref_mut()).await?;
199        debug!(
200            "{:?} Tables take {} bytes",
201            superblock,
202            superblock.tables_length()
203        );
204
205        let mut r = if superblock.tables_length() < TABLES_DIRECT_THRESHOLD {
206            r
207        } else {
208            // Don't use direct access if the tables are quite large
209            readers.get(&0).unwrap().get().await?
210        };
211        let mut r = r.deref_mut();
212        let root_inode =
213            inodes::InodeTable::read_root_inode(superblock.root_inode, &superblock, &mut r).await?;
214        let inode_table = inodes::InodeTable::from_reader(&superblock, &mut r).await?;
215        let fragments_table = fragments::FragmentsTable::from_reader(&superblock, &mut r).await?;
216        let mut directory_table: BTreeMap<u32, directory_table::DirectoryTable> =
217            Default::default();
218
219        debug!("Caching directory table");
220        for (inode, dir) in &inode_table.directories {
221            directory_table.insert(
222                *inode,
223                directory_table::DirectoryTable::from_reader_directory(
224                    dir,
225                    &superblock,
226                    r.deref_mut(),
227                )
228                .await?,
229            );
230        }
231
232        let cache: Option<IndexCache> = if options.cache_mb > 0 {
233            let cache: Result<IndexCache, CacheError> = IndexCache::new(
234                options.cache_mb,
235                superblock.block_size as u64,
236                superblock.bytes_used,
237            );
238            Some(cache?)
239        } else {
240            None
241        };
242
243        let small_files_cache: Option<LRUCache> = if options.direct_limit > 0 {
244            let direct_cache: Result<LRUCache, CacheError> = LRUCache::new(
245                options.cache_mb,
246                options.direct_limit as u64,
247                (superblock.inode_count as u64) * (options.direct_limit as u64),
248            );
249            Some(direct_cache?)
250        } else {
251            None
252        };
253        Ok(Self {
254            cache,
255            small_files_cache,
256            inode_extra: inode_table.ids().max().unwrap() + 1,
257            superblock,
258            n_readers: options.readers,
259            directory_tables: directory_table,
260            fragments_table,
261            inode_table,
262            manager_factory,
263            root_inode,
264            handles: Default::default(),
265            readers: RwLock::new(readers),
266            direct_limit: options.direct_limit,
267        })
268    }
269}
squashfs_async/lib.rs

squashfs_async/
lib.rs
