use std::{
collections::HashMap,
io,
path::Path,
sync::Arc,
time::{Duration, Instant, SystemTime},
};
use serde_json::Value;
use tokio::sync::Mutex;
use unicode_normalization::UnicodeNormalization;
use super::accessor::{DOWNLOAD_URL_KEY, OnedriveAccessor, OnedriveConfig};
use crate::{
BoxFuture,
fs::filesystem::{Dirent, DirentKind, FileSystem, Stat},
};
const DIR_TTL: Duration = Duration::from_secs(300);
const READ_SPAN: u64 = 64 * 1024 * 1024;
const FIRST_SPAN: u64 = 8 * 1024 * 1024;
const MAX_FOLDER_ITEMS: usize = 10_000;
const MAX_CACHED_DIRS: usize = 4_096;
type CachedListing = (Instant, Arc<Vec<Child>>);
const HELD_BUDGET: u64 = READ_SPAN;
const MIN_SPAN: u64 = 4 * 1024 * 1024;
const ACTIVE: Duration = Duration::from_secs(3);
struct HeldSpan {
at: u64,
to_eof: bool,
when: Instant,
used: Instant,
bytes: Arc<Vec<u8>>,
}
#[derive(Clone)]
struct Child {
id: String,
name: String,
is_dir: bool,
size: u64,
mtime: Option<SystemTime>,
created: Option<SystemTime>,
etag: Option<String>,
download_url: Option<String>,
}
pub struct OnedriveFs {
accessor: OnedriveAccessor,
dir_cache: Mutex<HashMap<String, CachedListing>>,
held: Mutex<HashMap<String, HeldSpan>>,
}
impl OnedriveFs {
pub fn new(config: &OnedriveConfig) -> anyhow::Result<Self> {
Ok(Self {
accessor: OnedriveAccessor::new(config)?,
dir_cache: Mutex::new(HashMap::new()),
held: Mutex::new(HashMap::new()),
})
}
async fn list_dir(&self, folder: &str) -> io::Result<Arc<Vec<Child>>> {
{
let cache = self.dir_cache.lock().await;
if let Some((at, children)) = cache.get(folder)
&& at.elapsed() < DIR_TTL
{
return Ok(children.clone());
}
}
let rows = match self.accessor.list_children(folder, MAX_FOLDER_ITEMS).await {
Ok(rows) => rows,
Err(e) if is_not_found(&e) => self.list_unspellable(folder, e).await?,
Err(e) => return Err(not_found_or_backend(e)),
};
let children: Vec<Child> = rows.iter().filter_map(child_from_item).collect();
let children = Arc::new(children);
let mut cache = self.dir_cache.lock().await;
cache.retain(|_, (at, _)| at.elapsed() < DIR_TTL);
if cache.len() >= MAX_CACHED_DIRS {
cache.clear();
}
cache.insert(folder.to_string(), (Instant::now(), children.clone()));
Ok(children)
}
async fn list_unspellable(
&self,
folder: &str,
as_given: anyhow::Error,
) -> io::Result<Vec<Value>> {
let composed: String = folder.nfc().collect();
let decomposed: String = folder.nfd().collect();
if composed == folder && decomposed == folder {
return Err(not_found_or_backend(as_given));
}
if composed != folder {
match self
.accessor
.list_children(&composed, MAX_FOLDER_ITEMS)
.await
{
Ok(rows) => return Ok(rows),
Err(e) if !is_not_found(&e) => return Err(not_found_or_backend(e)),
Err(_) => {}
}
}
let entry = Box::pin(self.resolve(folder)).await?;
if !entry.is_dir {
return Err(io::Error::from(io::ErrorKind::NotFound));
}
self.accessor
.list_children_of_id(&entry.id, MAX_FOLDER_ITEMS)
.await
.map_err(not_found_or_backend)
}
async fn resolve(&self, path: &str) -> io::Result<Child> {
let (parent, name) = split_last(path);
let children = self.list_dir(&parent).await?;
children
.iter()
.find(|c| same_name(&c.name, &name))
.cloned()
.ok_or_else(|| io::Error::from(io::ErrorKind::NotFound))
}
async fn span(
&self,
child: &Child,
path: &str,
start: u64,
want: u64,
) -> io::Result<(u64, Arc<Vec<u8>>)> {
let (walking, sharers) = {
let mut held = self.held.lock().await;
let sharers = held
.iter()
.filter(|(k, h)| k.as_str() != path && h.used.elapsed() < ACTIVE)
.count() as u64
+ 1;
let walking = match held.get_mut(path) {
Some(h) if h.when.elapsed() < DIR_TTL => {
if h.at <= start
&& (start.saturating_add(want) <= h.at.saturating_add(h.bytes.len() as u64)
|| h.to_eof)
{
h.used = Instant::now();
return Ok((h.at, h.bytes.clone()));
}
h.at <= start && start <= h.at.saturating_add(h.bytes.len() as u64)
}
_ => false,
};
(walking, sharers)
};
let share = (HELD_BUDGET / sharers).max(MIN_SPAN);
let len = want.max(if walking {
share
} else {
share.min(FIRST_SPAN)
});
let range = start..start.saturating_add(len);
let url = match child.download_url.as_deref() {
Some(u) => u.to_string(),
None => self.fresh_download_url(child).await?,
};
let fetched = self.accessor.download(&url, Some(range.clone())).await;
let (at, bytes) = match fetched {
Ok(v) => v,
Err(_) => {
let url = self.fresh_download_url(child).await?;
self.accessor
.download(&url, Some(range))
.await
.map_err(not_found_or_backend)?
}
};
let to_eof = (bytes.len() as u64) < len;
let bytes = Arc::new(bytes);
let now = Instant::now();
self.hold(
path,
HeldSpan {
at,
to_eof,
when: now,
used: now,
bytes: bytes.clone(),
},
)
.await;
Ok((at, bytes))
}
async fn hold(&self, path: &str, span: HeldSpan) {
let len = span.bytes.len() as u64;
let mut held = self.held.lock().await;
held.remove(path);
held.retain(|_, h| h.when.elapsed() < DIR_TTL);
while !held.is_empty()
&& held.values().map(|h| h.bytes.len() as u64).sum::<u64>() + len > HELD_BUDGET
{
let coldest = held
.iter()
.min_by_key(|(_, h)| h.used)
.map(|(k, _)| k.clone());
match coldest {
Some(k) => {
held.remove(&k);
}
None => break,
}
}
held.insert(path.to_string(), span);
}
async fn fresh_download_url(&self, child: &Child) -> io::Result<String> {
let item = self
.accessor
.get_item_by_id(&child.id)
.await
.map_err(not_found_or_backend)?;
download_url_of(&item).ok_or_else(|| {
io::Error::other(format!("onedrive: {} has no download url", child.name))
})
}
async fn read_window(
&self,
path: &Path,
range: Option<std::ops::Range<u64>>,
) -> io::Result<Vec<u8>> {
let path = vpath(path)?;
let path = path.as_str();
if path == "/" {
return Err(io::Error::other("is a directory: /"));
}
let child = self.resolve(path).await?;
if child.is_dir {
return Err(io::Error::other(format!("is a directory: {path}")));
}
let Some(r) = range else {
let url = match child.download_url.as_deref() {
Some(u) => u.to_string(),
None => self.fresh_download_url(&child).await?,
};
let whole = match self.accessor.download(&url, None).await {
Ok((_, b)) => b,
Err(_) => {
let url = self.fresh_download_url(&child).await?;
self.accessor
.download(&url, None)
.await
.map_err(not_found_or_backend)?
.1
}
};
return Ok(whole);
};
let want = r.end.saturating_sub(r.start);
if want == 0 {
return Ok(Vec::new());
}
let (at, bytes) = self.span(&child, path, r.start, want).await?;
let from = r.start.saturating_sub(at);
Ok(slice(&bytes, Some(from..r.end.saturating_sub(at))))
}
#[cfg(test)]
pub(crate) async fn listings_retained(&self) -> usize {
self.dir_cache.lock().await.len()
}
#[cfg(test)]
pub(crate) async fn held_bytes(&self, path: &str) -> Option<u64> {
self.held
.lock()
.await
.get(path)
.map(|h| h.bytes.len() as u64)
}
#[cfg(test)]
pub(crate) async fn age_spans_for_test(&self, by: Duration) {
let mut held = self.held.lock().await;
for h in held.values_mut() {
if let (Some(w), Some(u)) = (h.when.checked_sub(by), h.used.checked_sub(by)) {
h.when = w;
h.used = u;
}
}
}
#[cfg(test)]
pub(crate) async fn age_listings_for_test(&self) {
let mut cache = self.dir_cache.lock().await;
let stale = Instant::now() - DIR_TTL - Duration::from_secs(1);
for (at, _) in cache.values_mut() {
*at = stale;
}
}
}
impl FileSystem for OnedriveFs {
fn stat<'a>(&'a self, path: &'a Path) -> BoxFuture<'a, io::Result<Stat>> {
Box::pin(async move {
let path = vpath(path)?;
if path == "/" {
return Ok(Stat::new(DirentKind::Dir, 0));
}
let child = self.resolve(&path).await?;
Ok(stat_of(&child))
})
}
fn list<'a>(&'a self, path: &'a Path) -> BoxFuture<'a, io::Result<Vec<Dirent>>> {
Box::pin(async move {
let path = vpath(path)?;
let children = self.list_dir(&path).await?;
Ok(children
.iter()
.map(|c| Dirent::with_stat(c.name.clone(), stat_of(c)))
.collect())
})
}
fn read_at<'a>(
&'a self,
path: &'a Path,
buf: &'a mut [u8],
offset: u64,
) -> BoxFuture<'a, io::Result<usize>> {
Box::pin(async move {
if buf.is_empty() {
return Ok(0);
}
let want = buf.len() as u64;
let bytes = self
.read_window(path, Some(offset..offset.saturating_add(want)))
.await?;
let n = bytes.len().min(buf.len());
buf[..n].copy_from_slice(&bytes[..n]);
Ok(n)
})
}
}
fn stat_of(c: &Child) -> Stat {
Stat {
kind: if c.is_dir {
DirentKind::Dir
} else {
DirentKind::File
},
size: c.size,
mtime: c.mtime,
created: c.created,
etag: c.etag.clone(),
..Stat::new(
if c.is_dir {
DirentKind::Dir
} else {
DirentKind::File
},
c.size,
)
}
}
fn child_from_item(v: &Value) -> Option<Child> {
if v.get("package").is_some() {
return None;
}
let id = v.get("id")?.as_str()?.to_string();
let name = sanitize_name(v.get("name")?.as_str()?);
let is_dir = v.get("folder").is_some();
if !is_dir && v.get("file").is_none() {
return None;
}
Some(Child {
id,
name,
is_dir,
size: if is_dir {
0
} else {
v.get("size").and_then(|s| s.as_u64()).unwrap_or(0)
},
mtime: time_field(v, "lastModifiedDateTime"),
created: time_field(v, "createdDateTime"),
etag: v
.get("cTag")
.or_else(|| v.get("eTag"))
.and_then(|t| t.as_str())
.map(str::to_string),
download_url: download_url_of(v),
})
}
fn download_url_of(v: &Value) -> Option<String> {
v.get(DOWNLOAD_URL_KEY)
.and_then(|u| u.as_str())
.map(str::to_string)
}
fn time_field(v: &Value, key: &str) -> Option<SystemTime> {
let s = v.get(key)?.as_str()?;
let secs = chrono::DateTime::parse_from_rfc3339(s).ok()?.timestamp();
(secs >= 0).then(|| std::time::UNIX_EPOCH + Duration::from_secs(secs as u64))
}
fn not_found_or_backend(e: anyhow::Error) -> io::Error {
if is_not_found(&e) {
io::Error::from(io::ErrorKind::NotFound)
} else {
io::Error::other(format!("{e:#}"))
}
}
fn is_not_found(e: &anyhow::Error) -> bool {
e.downcast_ref::<reqwest::Error>()
.and_then(reqwest::Error::status)
== Some(reqwest::StatusCode::NOT_FOUND)
}
fn sanitize_name(name: &str) -> String {
let cleaned: String = name
.trim()
.chars()
.map(|c| {
if c == '/' || c == '\\' || c.is_control() {
'_'
} else {
c
}
})
.collect();
match cleaned.as_str() {
"" | "." | ".." => "untitled".to_string(),
_ => cleaned,
}
}
fn same_name(a: &str, b: &str) -> bool {
a == b || a.nfc().eq(b.nfc())
}
fn vpath(path: &Path) -> io::Result<String> {
let mut parts: Vec<&str> = Vec::new();
for comp in path.components() {
match comp {
std::path::Component::RootDir | std::path::Component::CurDir => {}
std::path::Component::Normal(name) => parts.push(
name.to_str()
.ok_or(io::Error::from(io::ErrorKind::InvalidFilename))?,
),
std::path::Component::ParentDir | std::path::Component::Prefix(_) => {
return Err(io::ErrorKind::InvalidFilename.into());
}
}
}
Ok(format!("/{}", parts.join("/")))
}
fn split_last(path: &str) -> (String, String) {
let p = path.trim_end_matches('/');
match p.rsplit_once('/') {
Some((parent, name)) => {
let parent = if parent.is_empty() {
"/".to_string()
} else {
parent.to_string()
};
(parent, name.to_string())
}
None => ("/".to_string(), p.to_string()),
}
}
fn slice(data: &[u8], range: Option<std::ops::Range<u64>>) -> Vec<u8> {
match range {
Some(r) => {
debug_assert!(r.end >= r.start, "callers hand this a forward range");
let start = (r.start as usize).min(data.len());
let end = (r.end as usize).min(data.len());
data[start..end].to_vec()
}
None => data.to_vec(),
}
}
#[cfg(test)]
#[path = "onedrive_tests.rs"]
mod tests;