use std::{
collections::{HashMap, VecDeque},
ffi::OsStr,
fs::Metadata,
os::unix::prelude::MetadataExt,
path::Path,
time::Duration,
};
use async_trait::async_trait;
use camino::{Utf8Component, Utf8Path, Utf8PathBuf};
use futures_util::{StreamExt as _, TryStreamExt as _};
use libdav::xmlutils::normalise_newlines;
use tokio::{
fs::{File, create_dir, metadata, read_dir, read_to_string, remove_dir, remove_file},
io::{AsyncReadExt as _, AsyncWriteExt},
sync::{
RwLock,
oneshot::{self, Sender},
},
};
use crate::{
CollectionId, Error, ErrorKind, Etag, Href, ItemKind, Result,
atomic::AtomicFile,
base::{Collection, CreateItemOptions, FetchedItem, Item, ItemVersion, Storage},
disco::{DiscoveredCollection, Discovery},
property::Property,
watch::StorageMonitor,
};
#[cfg_attr(target_os = "linux", path = "vdir/linux.rs")]
#[cfg_attr(not(target_os = "linux"), path = "vdir/non_linux.rs")]
mod monitor;
pub use monitor::VdirMonitor;
pub struct VdirStorage {
pub path: Utf8PathBuf,
pub extension: String,
kind: ItemKind,
file_locks: FileLocker,
}
const SAFE_FILENAME_CHARS: &str =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+";
pub struct VdirStorageBuilder {
path: Utf8PathBuf,
extension: Option<String>,
}
impl VdirStorageBuilder {
#[must_use]
pub fn with_extension(mut self, extension: String) -> Self {
self.extension = Some(extension);
self
}
#[must_use]
pub fn build(self, kind: ItemKind) -> VdirStorage {
let extension = self.extension.unwrap_or_else(|| match kind {
ItemKind::Calendar => "ics".to_string(),
ItemKind::AddressBook => "vcf".to_string(),
});
VdirStorage {
path: self.path,
extension,
kind,
file_locks: FileLocker::default(),
}
}
}
#[async_trait]
impl Storage for VdirStorage {
fn item_kind(&self) -> ItemKind {
self.kind
}
async fn check(&self) -> Result<()> {
let meta = metadata(&self.path)
.await
.map_err(|e| ErrorKind::DoesNotExist.error(e))?;
if meta.is_dir() {
Ok(())
} else {
Err(ErrorKind::NotAStorage.error("path is not a directory"))
}
}
async fn discover_collections(&self) -> Result<Discovery> {
let mut entries = read_dir(&self.path).await?;
let mut collections = Vec::<_>::new();
while let Some(entry) = entries.next_entry().await? {
if !metadata(entry.path()).await?.is_dir() {
continue;
}
let href = entry
.file_name()
.into_string()
.map_err(|_| ErrorKind::InvalidData.error("collection id is not utf8"))?;
if href.starts_with('.') {
continue;
}
let id = href.parse().map_err(|e| ErrorKind::InvalidData.error(e))?;
collections.push(DiscoveredCollection::new(href, id));
}
collections
.try_into()
.map_err(|e| ErrorKind::InvalidData.error(e))
}
async fn create_collection(&self, href: &str) -> Result<Collection> {
let path = build_collection_path(&self.path, href)?;
create_dir(&path).await?;
Ok(Collection::new(href.to_string()))
}
async fn delete_collection(&self, href: &str) -> Result<()> {
let path = build_collection_path(&self.path, href)?;
for prop in Property::known_properties(self.kind) {
if let Err(err) = self.unset_property(href, *prop).await
&& err.kind != ErrorKind::DoesNotExist
{
return Err(err);
}
}
remove_dir(path).await.map_err(Error::from)
}
async fn list_items(&self, collection_href: &str) -> Result<Vec<ItemVersion>> {
let mut read_dir = read_dir(build_collection_path(&self.path, collection_href)?).await?;
let mut items = Vec::new();
let extension = OsStr::new(self.extension.as_str());
while let Some(entry) = read_dir.next_entry().await? {
let path = entry.path();
if path.extension() != Some(extension) {
continue;
}
let href = href_for_path(&self.path, &path)?;
let etag = etag_for_path(path).await?;
items.push(ItemVersion::new(href, etag));
}
Ok(items)
}
async fn get_item(&self, href: &str) -> Result<(Item, Etag)> {
let path = build_item_path(&self.path, &self.extension, href)?;
let mut file = File::open(&path).await?;
let mut buf = String::new();
file.read_to_string(&mut buf).await?;
let item = Item::from(normalise_newlines(&buf));
let etag = etag_for_metadata(&file.metadata().await?);
Ok((item, etag))
}
async fn get_many_items(&self, hrefs: &[&str]) -> Result<Vec<FetchedItem>> {
futures_util::stream::iter(hrefs)
.then(|href| async move {
self.get_item(href).await.map(|(item, etag)| FetchedItem {
href: String::from(*href),
item,
etag,
})
})
.try_collect()
.await
}
async fn get_all_items(&self, collection_href: &str) -> Result<Vec<FetchedItem>> {
let mut read_dir = read_dir(build_collection_path(&self.path, collection_href)?).await?;
let mut items = Vec::new();
let extension = OsStr::new(self.extension.as_str());
while let Some(entry) = read_dir.next_entry().await? {
let path = entry.path();
if path.extension() != Some(extension) {
continue;
}
let mut file = File::open(&path).await?;
let mut buf = String::new();
file.read_to_string(&mut buf).await?;
let item = Item::from(normalise_newlines(&buf));
let etag = etag_for_metadata(&file.metadata().await?);
items.push(FetchedItem {
href: href_for_path(&self.path, &path)?,
item,
etag,
});
}
Ok(items)
}
async fn set_property(&self, href: &str, meta: Property, value: &str) -> Result<()> {
let filename = self.property_filename(meta)?;
let path = build_collection_path(&self.path, href)?.join(filename);
let file_lock = self.file_locks.lock_file(path.as_str()).await;
let mut file = AtomicFile::new(&path)?;
file.write_all(value.as_bytes()).await?;
file.commit().await?;
self.file_locks.release_file(file_lock).await;
Ok(())
}
async fn unset_property(&self, href: &str, meta: Property) -> Result<()> {
let filename = self.property_filename(meta)?;
let path = build_collection_path(&self.path, href)?.join(filename);
let file_lock = self.file_locks.lock_file(path.as_str()).await;
remove_file(filename).await?;
self.file_locks.release_file(file_lock).await;
Ok(())
}
async fn get_property(&self, href: &str, meta: Property) -> Result<Option<String>> {
let filename = self.property_filename(meta)?;
let path = build_collection_path(&self.path, href)?.join(filename);
match read_to_string(path).await {
Ok(value) => Ok(Some(value.trim().to_string())),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(Error::from(e)),
}
}
async fn create_item(
&self,
collection_href: &str,
item: &Item,
opts: CreateItemOptions,
) -> Result<ItemVersion> {
let basename = valid_creation_filename(item, opts, &self.extension);
let filename = format!("{}.{}", basename, self.extension);
let relpath = Utf8PathBuf::from(collection_href).join(filename);
let absolute_path = build_item_path(&self.path, &self.extension, relpath.as_str())?;
let mut file = AtomicFile::new(&absolute_path)?;
file.write_all(item.as_str().as_bytes()).await?;
let meta = file.commit_new().await?;
let etag = etag_for_metadata(&meta);
let item_ver = ItemVersion::new(relpath.into_string(), etag);
Ok(item_ver)
}
async fn update_item(&self, href: &str, etag: &Etag, item: &Item) -> Result<Etag> {
let filename = build_item_path(&self.path, &self.extension, href)?;
let actual_etag = etag_for_path(&filename).await?;
if *etag != actual_etag {
return Err(ErrorKind::InvalidData.error("etag mismatch when updating item"));
}
let file_lock = self.file_locks.lock_file(filename.as_str()).await;
let mut file = AtomicFile::new(&filename)?;
file.write_all(item.as_str().as_bytes()).await?;
let meta = file.commit().await?;
let etag = etag_for_metadata(&meta);
self.file_locks.release_file(file_lock).await;
Ok(etag)
}
async fn delete_item(&self, href: &str, etag: &Etag) -> Result<()> {
let filename = build_item_path(&self.path, &self.extension, href)?;
let actual_etag = etag_for_path(&filename).await?;
if *etag != actual_etag {
return Err(ErrorKind::InvalidData.error("wrong etag"));
}
let file_lock = self.file_locks.lock_file(filename.as_str()).await;
remove_file(&filename).await?;
self.file_locks.release_file(file_lock).await;
Ok(())
}
fn href_for_collection_id(&self, id: &CollectionId) -> Result<Href> {
Ok(id.to_string())
}
async fn monitor(&self, interval: Duration) -> Result<Box<dyn StorageMonitor>> {
VdirMonitor::new(self, interval).map(|m| Box::new(m) as Box<dyn StorageMonitor>)
}
}
impl VdirStorage {
pub fn builder(path: Utf8PathBuf) -> Result<VdirStorageBuilder> {
if !path.is_absolute() {
return Err(
ErrorKind::InvalidInput.error("VdirStorage must be created with absolute path")
);
}
Ok(VdirStorageBuilder {
path,
extension: None,
})
}
fn property_filename(&self, property: Property) -> Result<&'static str> {
if property.is_valid_for(self.kind) {
Ok(property.name())
} else {
Err(ErrorKind::InvalidInput.error(format!(
"property '{property}' is not valid for {:?}",
self.kind
)))
}
}
}
fn valid_creation_filename(item: &Item, opts: CreateItemOptions, extension: &str) -> String {
if let Some(name) = opts.resource_name
&& name != "."
&& name != ".."
&& !name.is_empty()
{
let suffix = format!(".{extension}");
return match name.strip_suffix(&suffix) {
Some(stripped) => stripped.to_string(),
None => name,
};
}
let name = item
.ident()
.chars()
.filter(|c| SAFE_FILENAME_CHARS.contains(*c))
.collect::<String>();
if name.is_empty() {
item.hash().to_string()
} else {
name
}
}
fn build_collection_path(root: &Utf8Path, collection_href: &str) -> Result<Utf8PathBuf> {
let href = Utf8Path::new(collection_href);
let mut components = href.components();
if !matches!(components.next(), Some(Utf8Component::Normal(_))) {
return Err(ErrorKind::InvalidInput.error("collection href must be a valid directory name"));
}
if components.next().is_some() {
return Err(
ErrorKind::InvalidInput.error("collection href must contain exactly one component")
);
}
Ok(root.join(href))
}
fn build_item_path(root: &Utf8Path, extension: &str, href: &str) -> Result<Utf8PathBuf> {
let href = Utf8Path::new(href);
let mut components = href.components();
if !matches!(components.next(), Some(Utf8Component::Normal(_))) {
return Err(ErrorKind::InvalidInput
.error("first component of item href must be a regular filename"));
}
if let Some(Utf8Component::Normal(name)) = components.next() {
let name = Utf8Path::new(name);
if name.extension() != Some(extension) {
Err(ErrorKind::InvalidInput
.error("item href does not have an extension matching this storage"))?;
}
} else {
return Err(ErrorKind::InvalidInput
.error("second component of item href must be a regular filename"));
}
if components.next().is_some() {
return Err(
ErrorKind::InvalidInput.error("item href cannot contain more than two components")
);
}
Ok(root.join(href))
}
fn href_for_path(root: &Utf8Path, path: &Path) -> Result<String> {
path.strip_prefix(root)
.expect("path of item must include storage path as prefix")
.to_str()
.ok_or_else(|| ErrorKind::InvalidData.error("Filename is not valid UTF-8"))
.map(str::to_string)
}
#[derive(Default)]
struct FileLocker(RwLock<HashMap<Box<str>, VecDeque<Sender<()>>>>);
impl FileLocker {
async fn lock_file<'a>(&self, filepath: &'a str) -> FileLock<'a> {
let mut locks = self.0.write().await;
if let Some(ref mut self_waiter) = locks.get_mut(&Box::from(filepath)) {
let (tx, rx) = oneshot::channel();
self_waiter.push_back(tx);
drop(locks);
rx.await
.expect("Previous locker of file must release successfully.");
} else {
locks.insert(Box::from(filepath), VecDeque::new());
}
FileLock(filepath)
}
async fn release_file(&self, lock: FileLock<'_>) {
let mut locks = self.0.write().await;
if let Some(ref mut self_waiter) = locks.get_mut(lock.0) {
if let Some(waiter) = self_waiter.pop_front() {
waiter
.send(())
.expect("Waiter for file lock must remain alive.");
} else {
locks.remove(lock.0);
}
}
drop(locks);
}
}
struct FileLock<'a>(&'a str);
async fn etag_for_path(path: impl AsRef<Path>) -> Result<Etag> {
let metadata = &metadata(path).await?;
Ok(etag_for_metadata(metadata))
}
fn etag_for_metadata(metadata: &Metadata) -> Etag {
format!("{};{}", metadata.mtime(), metadata.ino()).into()
}
#[cfg(test)]
mod tests {
use std::{
fs::{create_dir_all, write},
str::FromStr,
time::Duration,
};
use crate::{
CollectionId, ErrorKind,
base::{CreateItemOptions, Item, Storage},
property::Property,
vdir::{ItemKind, VdirStorage, build_collection_path, build_item_path},
watch::Event,
};
use tempfile::tempdir;
use tokio::fs::read_to_string;
#[tokio::test]
async fn missing_displayname() {
let dir = tempdir().unwrap();
let storage = VdirStorage::builder(dir.path().to_path_buf().try_into().unwrap())
.unwrap()
.build(ItemKind::Calendar);
let collection = storage.create_collection("test").await.unwrap();
let displayname = storage
.get_property(collection.href(), Property::DisplayName)
.await
.unwrap();
assert!(displayname.is_none());
}
#[tokio::test]
async fn path_concatenation() {
let dir = tempdir().unwrap();
let storage = VdirStorage::builder(dir.path().to_path_buf().try_into().unwrap())
.unwrap()
.build(ItemKind::Calendar);
let collection_name = "one";
let collection_path = dir.path().join(collection_name);
create_dir_all(&collection_path).unwrap();
let without_prodid = [
"BEGIN:VCALENDAR",
"BEGIN:VEVENT",
"DTSTART:19970714T170000Z",
"DTEND:19970715T035959Z",
"SUMMARY:Bastille Day Party",
"UID:11bb6bed-c29b-4999-a627-12dee35f8395",
"END:VEVENT",
"END:VCALENDAR",
]
.join("\r\n");
write(collection_path.join("item.ics"), &without_prodid).unwrap();
let listed_items = storage.list_items(collection_name).await.unwrap();
assert_eq!(listed_items.len(), 1);
assert_eq!(listed_items[0].href, "one/item.ics");
let all_items = storage.get_all_items(collection_name).await.unwrap();
assert_eq!(all_items.len(), 1);
assert_eq!(all_items[0].href, "one/item.ics");
let (_item, etag) = storage.get_item("one/item.ics").await.unwrap();
let many_items = storage.get_many_items(&["one/item.ics"]).await.unwrap();
assert_eq!(many_items.len(), 1);
assert_eq!(many_items[0].href, "one/item.ics");
storage.delete_item("one/item.ics", &etag).await.unwrap();
let item = Item::from(without_prodid);
let opts = CreateItemOptions::default();
storage.create_item("one", &item, opts).await.unwrap();
let all_items = storage.get_all_items(collection_name).await.unwrap();
assert_eq!(all_items.len(), 1);
}
#[tokio::test]
async fn missing_paths() {
let dir = tempdir().unwrap();
let storage = VdirStorage::builder(dir.path().to_path_buf().try_into().unwrap())
.unwrap()
.build(ItemKind::Calendar);
let missing_collection = "two";
let err = storage.list_items(missing_collection).await.unwrap_err();
assert_eq!(err.kind, ErrorKind::DoesNotExist);
let err = storage.get_all_items(missing_collection).await.unwrap_err();
assert_eq!(err.kind, ErrorKind::DoesNotExist);
}
#[tokio::test]
async fn write_read_colour() {
let dir = tempdir().unwrap();
let storage = VdirStorage::builder(dir.path().to_path_buf().try_into().unwrap())
.unwrap()
.build(ItemKind::Calendar);
let collection_name = "one";
storage.create_collection(collection_name).await.unwrap();
storage
.set_property(collection_name, Property::Colour, "#000000")
.await
.unwrap();
let colour = storage
.get_property(collection_name, Property::Colour)
.await
.unwrap();
assert_eq!(colour, Some(String::from("#000000")));
}
#[tokio::test]
async fn read_missing_description() {
let dir = tempdir().unwrap();
let storage = VdirStorage::builder(dir.path().to_path_buf().try_into().unwrap())
.unwrap()
.build(ItemKind::Calendar);
let collection_name = "one";
storage.create_collection(collection_name).await.unwrap();
let description = storage
.get_property(collection_name, Property::Description)
.await
.unwrap();
assert_eq!(description, None);
}
#[tokio::test]
async fn write_and_read_description() {
let dir = tempdir().unwrap();
let storage = VdirStorage::builder(dir.path().to_path_buf().try_into().unwrap())
.unwrap()
.build(ItemKind::Calendar);
let collection_name = "one";
storage.create_collection(collection_name).await.unwrap();
storage
.set_property(collection_name, Property::Description, "Just a test")
.await
.unwrap();
let description = storage
.get_property(collection_name, Property::Description)
.await
.unwrap();
assert_eq!(description, Some("Just a test".into()));
let expected_path = dir.path().join("one").join("description");
let value = read_to_string(expected_path).await.unwrap();
assert_eq!(value, "Just a test");
}
#[tokio::test]
async fn href_for_collection_id() {
let dir = tempdir().unwrap();
let storage = VdirStorage::builder(dir.path().to_path_buf().try_into().unwrap())
.unwrap()
.build(ItemKind::Calendar);
let collection_id = CollectionId::from_str("one").unwrap();
let href = storage.href_for_collection_id(&collection_id).unwrap();
assert_eq!(href, "one");
}
#[tokio::test]
async fn build_collection_path_is_safe() {
let dir = tempdir().unwrap();
let root = dir.path().try_into().unwrap();
assert!(build_collection_path(root, "penguins").is_ok());
assert!(build_collection_path(root, "penguins/").is_ok());
assert!(build_collection_path(root, "蛙类").is_ok());
assert!(build_collection_path(root, "/").is_err());
assert!(build_collection_path(root, "/usr/share/").is_err());
assert!(build_collection_path(root, "..").is_err());
assert!(build_collection_path(root, ".").is_err());
assert!(build_collection_path(root, "../d").is_err());
assert!(build_collection_path(root, "s/../../").is_err());
}
#[tokio::test]
async fn build_item_path_is_safe() {
let dir = tempdir().unwrap();
let root = dir.path().try_into().unwrap();
let extension = "ics";
assert!(build_item_path(root, extension, "penguins/someitem.ics").is_ok());
assert!(build_item_path(root, extension, "蛙类/item.ics").is_ok());
assert!(build_item_path(root, extension, "penguins/someitem.jpeg").is_err());
assert!(build_item_path(root, extension, "蛙类/item.jpeg").is_err());
assert!(build_item_path(root, extension, "penguins/someitem").is_err());
assert!(build_item_path(root, extension, "蛙类/item").is_err());
assert!(build_item_path(root, extension, "penguins").is_err());
assert!(build_item_path(root, extension, "蛙类").is_err());
assert!(build_item_path(root, extension, "penguins/../someitem.ics").is_err());
assert!(build_item_path(root, extension, "../penguins/someitem.ics").is_err());
assert!(build_item_path(root, extension, "/").is_err());
assert!(build_item_path(root, extension, "/usr/share/").is_err());
assert!(build_item_path(root, extension, "..").is_err());
assert!(build_item_path(root, extension, ".").is_err());
assert!(build_item_path(root, extension, "s/../../").is_err());
}
#[tokio::test]
async fn only_safe_chars_in_filenames() {
let dir = tempdir().unwrap();
let storage = VdirStorage::builder(dir.path().to_path_buf().try_into().unwrap())
.unwrap()
.build(ItemKind::Calendar);
let valid = [
"BEGIN:VCALENDAR",
"BEGIN:VEVENT",
"DTSTART:19970714T170000Z",
"DTEND:19970715T035959Z",
"SUMMARY:Bastille Day Party",
"UID:11bb6bed-c29b-4999-a627-12dee35f8395",
"END:VEVENT",
"END:VCALENDAR",
]
.join("\r\n");
let item = Item::from(valid);
storage.create_collection("one").await.unwrap();
let opts = CreateItemOptions::default();
let item_ver = storage.create_item("one", &item, opts).await.unwrap();
assert_eq!(
item_ver.href,
"one/11bb6bed-c29b-4999-a627-12dee35f8395.ics"
);
}
#[tokio::test]
async fn only_unsafe_chars_in_filenames() {
let dir = tempdir().unwrap();
let storage = VdirStorage::builder(dir.path().to_path_buf().try_into().unwrap())
.unwrap()
.build(ItemKind::Calendar);
let valid = [
"BEGIN:VCALENDAR",
"BEGIN:VEVENT",
"DTSTART:19970714T170000Z",
"DTEND:19970715T035959Z",
"SUMMARY:Bastille Day Party",
"UID:these/slashes/are/not/okay",
"END:VEVENT",
"END:VCALENDAR",
]
.join("\r\n");
let item = Item::from(valid);
storage.create_collection("one").await.unwrap();
let opts = CreateItemOptions::default();
let item_ver = storage.create_item("one", &item, opts).await.unwrap();
assert_eq!(item_ver.href, "one/theseslashesarenotokay.ics");
}
#[tokio::test]
async fn monitor_item_deletion() {
let path = tempdir().unwrap();
let storage = VdirStorage::builder(path.path().to_path_buf().try_into().unwrap())
.unwrap()
.build(ItemKind::Calendar);
let collection = storage.create_collection("test-calendar").await.unwrap();
let item_content = [
"BEGIN:VCALENDAR",
"VERSION:2.0",
"PRODID:-//test//test//EN",
"BEGIN:VEVENT",
"UID:test-event-123",
"DTSTAMP:20250101T120000Z",
"DTSTART:20250101T140000Z",
"SUMMARY:Test Event",
"END:VEVENT",
"END:VCALENDAR",
]
.join("\r\n");
let item = item_content.into();
let item_version = storage
.create_item(collection.href(), &item, CreateItemOptions::default())
.await
.unwrap();
let mut monitor = storage.monitor(Duration::from_secs(5)).await.unwrap();
assert_eq!(Event::General, monitor.next_event().await);
let item_path = path.path().join(&item_version.href);
std::fs::remove_file(&item_path).unwrap();
let event = tokio::time::timeout(Duration::from_secs(2), monitor.next_event())
.await
.unwrap();
match event {
Event::General => panic!("Expected deletion event, got general event"),
Event::Specific(e) => match e.kind {
crate::watch::EventKind::Change => {
panic!("Expected deletion event, got creation event.")
}
crate::watch::EventKind::Delete => {} },
}
}
}