use std::collections::HashMap;
use std::fs::OpenOptions;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use crate::error::NbError;
use super::virtual_tree::normalize_rel;
#[derive(Debug, Clone)]
pub(super) struct PendingIndexEdit {
pub(super) file_rel: String,
pub(super) kind: PendingIndexKind,
}
#[derive(Debug, Clone)]
pub(super) enum PendingIndexKind {
Append {
basename: String,
},
Blank {
basename: String,
},
Rename {
old_basename: String,
new_basename: String,
},
}
fn mangle_stem(title: &str) -> String {
let mut out = String::with_capacity(title.len());
let mut prev_underscore = false;
for c in title.chars() {
if c.is_ascii_alphanumeric() {
out.push(c.to_ascii_lowercase());
prev_underscore = false;
} else if c.is_ascii() {
if !prev_underscore {
out.push('_');
prev_underscore = true;
}
} else {
out.push(c);
prev_underscore = false;
}
}
out.trim_matches('_').to_string()
}
fn titleless_stem() -> String {
if let Ok(out) = std::process::Command::new("date")
.arg("+%Y%m%d%H%M%S")
.output()
&& out.status.success()
{
let s = String::from_utf8_lossy(&out.stdout).trim().to_string();
if s.len() == 14 && s.bytes().all(|b| b.is_ascii_digit()) {
return s;
}
}
static COUNTER: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
let secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
format!("{secs:014}-{n}")
}
pub(crate) fn filename_for_title(
title: Option<&str>,
extension: &str,
) -> String {
let stem = title
.map(|t| mangle_stem(t.trim()))
.filter(|s| !s.is_empty());
match stem {
Some(s) => format!("{s}.{extension}"),
None => format!("{}.{extension}", titleless_stem()),
}
}
pub(crate) fn suffixed_filename(filename: &str, n: u32) -> String {
match filename.find('.') {
Some(i) => format!("{}-{n}{}", &filename[..i], &filename[i..]),
None => format!("{filename}-{n}"),
}
}
pub(crate) fn join_folder_file(
folder: Option<&str>,
filename: &str,
) -> String {
match folder {
Some(f) if !f.trim().is_empty() => {
format!(
"{}/{}",
normalize_rel(f),
filename.trim_start_matches('/')
)
},
_ => filename.to_string(),
}
}
pub(super) fn split_folder_basename(path: &str) -> (String, String) {
match path.rfind('/') {
Some(i) => (path[..i].to_string(), path[i + 1..].to_string()),
None => (String::new(), path.to_string()),
}
}
pub(super) fn index_rel_for_folder(folder: &str) -> String {
if folder.is_empty() {
".index".to_string()
} else {
format!("{folder}/.index")
}
}
pub(super) fn append_index_edit(path: &str) -> PendingIndexEdit {
let (folder, basename) = split_folder_basename(path);
PendingIndexEdit {
file_rel: index_rel_for_folder(&folder),
kind: PendingIndexKind::Append { basename },
}
}
pub(super) fn blank_index_edit(path: &str) -> PendingIndexEdit {
let (folder, basename) = split_folder_basename(path);
PendingIndexEdit {
file_rel: index_rel_for_folder(&folder),
kind: PendingIndexKind::Blank { basename },
}
}
pub(super) fn move_index_edits(from: &str, to: &str) -> Vec<PendingIndexEdit> {
let (from_folder, from_base) = split_folder_basename(from);
let (to_folder, to_base) = split_folder_basename(to);
if from_folder == to_folder {
vec![PendingIndexEdit {
file_rel: index_rel_for_folder(&from_folder),
kind: PendingIndexKind::Rename {
old_basename: from_base,
new_basename: to_base,
},
}]
} else {
vec![
PendingIndexEdit {
file_rel: index_rel_for_folder(&from_folder),
kind: PendingIndexKind::Blank {
basename: from_base,
},
},
PendingIndexEdit {
file_rel: index_rel_for_folder(&to_folder),
kind: PendingIndexKind::Append { basename: to_base },
},
]
}
}
pub(super) fn parse_index_lines(bytes: &[u8]) -> Vec<String> {
if bytes.is_empty() {
return Vec::new();
}
let text = String::from_utf8_lossy(bytes);
let mut lines: Vec<String> =
text.split('\n').map(str::to_string).collect();
if text.ends_with('\n') {
lines.pop();
}
lines
}
pub(super) fn render_index_lines(lines: &[String]) -> Vec<u8> {
if lines.is_empty() {
return Vec::new();
}
let mut out = lines.join("\n").into_bytes();
out.push(b'\n');
out
}
pub(super) fn numeric_selector(
notebook: &str,
folder: &str,
id: u32,
) -> String {
if folder.is_empty() {
format!("{notebook}:{id}")
} else {
format!("{notebook}:{folder}/{id}")
}
}
pub(super) fn is_index_rel(rel: &str) -> bool {
rel == ".index" || rel.ends_with("/.index")
}
pub(super) struct IndexLockGuard {
path: PathBuf,
nonce: String,
heartbeat: Option<tokio::task::JoinHandle<()>>,
}
impl Drop for IndexLockGuard {
fn drop(&mut self) {
if let Some(handle) = self.heartbeat.take() {
handle.abort();
}
if std::fs::read_to_string(&self.path)
.map(|content| content.trim_end() == self.nonce)
.unwrap_or(false)
{
let _ = std::fs::remove_file(&self.path);
}
}
}
fn index_lock_path(notebook_root: &Path) -> PathBuf {
notebook_root.join(".nb-api-index.lock")
}
fn new_lock_nonce() -> String {
static COUNTER: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
format!("pid={} nonce={n} ns={nanos}", std::process::id())
}
fn try_remove_lock_if_stale(path: &Path, timeout: Duration) -> bool {
let content_before = std::fs::read_to_string(path).ok();
let stale = std::fs::metadata(path)
.and_then(|m| m.modified())
.map(|t| t.elapsed().unwrap_or(Duration::ZERO) > timeout)
.unwrap_or(false);
if !stale {
return content_before.is_none();
}
let content_after = std::fs::read_to_string(path).ok();
if content_after != content_before {
return false;
}
let still_stale = std::fs::metadata(path)
.and_then(|m| m.modified())
.map(|t| t.elapsed().unwrap_or(Duration::ZERO) > timeout)
.unwrap_or(true);
if content_before.is_none() || !still_stale {
return content_before.is_none();
}
std::fs::remove_file(path).is_ok()
}
pub(super) fn reap_stale_index_lock(notebook_root: &Path, timeout: Duration) {
let path = index_lock_path(notebook_root);
if path.exists() {
try_remove_lock_if_stale(&path, timeout);
}
}
pub(super) async fn acquire_index_lock(
notebook_root: &Path,
timeout: Duration,
) -> Result<IndexLockGuard, NbError> {
let path = index_lock_path(notebook_root);
let start = Instant::now();
loop {
match OpenOptions::new().write(true).create_new(true).open(&path) {
Ok(mut f) => {
let nonce = new_lock_nonce();
let _ = writeln!(f, "{nonce}");
drop(f);
let beat = (timeout / 3)
.clamp(Duration::from_millis(5), Duration::from_secs(30));
let heartbeat_path = path.clone();
let heartbeat_nonce = nonce.clone();
let heartbeat = tokio::spawn(async move {
loop {
tokio::time::sleep(beat).await;
let owned = std::fs::read_to_string(&heartbeat_path)
.map(|content| {
content.trim_end() == heartbeat_nonce
})
.unwrap_or(false);
if !owned {
break;
}
if std::fs::File::options()
.write(true)
.open(&heartbeat_path)
.and_then(|f| f.set_modified(SystemTime::now()))
.is_err()
{
break;
}
}
});
return Ok(IndexLockGuard {
path,
nonce,
heartbeat: Some(heartbeat),
});
},
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
if try_remove_lock_if_stale(&path, timeout) {
continue;
}
if start.elapsed() >= timeout {
return Err(NbError::IndexLockTimeout {
path: path.to_string_lossy().into_owned(),
timeout_ms: timeout.as_millis() as u64,
});
}
tokio::time::sleep(Duration::from_millis(10)).await;
},
Err(e) => {
return Err(NbError::Io {
path: path.clone(),
source: e.into(),
});
},
}
}
}
#[derive(Debug, Clone)]
pub(super) struct IndexPlacement {
pub(super) op_index: u32,
pub(super) folder: String,
pub(super) id: u32,
}
pub(super) fn apply_index_edits(
notebook_root: &Path,
pending: &[(u32, PendingIndexEdit)],
contention_timeout: Duration,
) -> Result<Vec<IndexPlacement>, NbError> {
let mut order: Vec<String> = Vec::new();
let mut grouped: HashMap<String, Vec<(u32, &PendingIndexEdit)>> =
HashMap::new();
for (op_index, edit) in pending {
grouped
.entry(edit.file_rel.clone())
.or_insert_with(|| {
order.push(edit.file_rel.clone());
Vec::new()
})
.push((*op_index, edit));
}
let mut placements: Vec<IndexPlacement> = Vec::new();
for file_rel in &order {
let edits = &grouped[file_rel];
let abs = notebook_root.join(file_rel);
let append_only = edits
.iter()
.all(|(_, e)| matches!(e.kind, PendingIndexKind::Append { .. }));
if append_only {
if let Some(parent) = abs.parent() {
std::fs::create_dir_all(parent).map_err(|e| NbError::Io {
path: parent.to_path_buf(),
source: e.into(),
})?;
}
let mut f = OpenOptions::new()
.create(true)
.append(true)
.open(&abs)
.map_err(|e| NbError::Io {
path: abs.clone(),
source: e.into(),
})?;
for (op_index, edit) in edits {
let PendingIndexKind::Append { basename } = &edit.kind else {
unreachable!()
};
writeln!(f, "{basename}").map_err(|e| NbError::Io {
path: abs.clone(),
source: e.into(),
})?;
let _ = op_index;
}
} else {
if let Some(parent) = abs.parent() {
std::fs::create_dir_all(parent).map_err(|e| NbError::Io {
path: parent.to_path_buf(),
source: e.into(),
})?;
}
let appends: Vec<&String> = edits
.iter()
.filter_map(|(_, e)| match &e.kind {
PendingIndexKind::Append { basename } => Some(basename),
_ => None,
})
.collect();
let mutations: Vec<&PendingIndexEdit> = edits
.iter()
.map(|(_, e)| *e)
.filter(|e| !matches!(e.kind, PendingIndexKind::Append { .. }))
.collect();
merge_index_rewrite(
&abs,
&appends,
&mutations,
contention_timeout,
)?;
}
let fresh = std::fs::read(&abs).map_err(|e| NbError::Io {
path: abs.clone(),
source: e.into(),
})?;
let lines = parse_index_lines(&fresh);
for (op_index, edit) in edits {
let (folder, basename) = match &edit.kind {
PendingIndexKind::Append { basename } => {
let (folder, _) = split_folder_basename(
&pending_basename_path(file_rel, basename),
);
(folder, basename.clone())
},
PendingIndexKind::Blank { basename } => {
let (folder, _) = split_folder_basename(
&pending_basename_path(file_rel, basename),
);
(folder, basename.clone())
},
PendingIndexKind::Rename { new_basename, .. } => {
let (folder, _) = split_folder_basename(
&pending_basename_path(file_rel, new_basename),
);
(folder, new_basename.clone())
},
};
if let Some(pos) = lines.iter().rposition(|l| l == &basename) {
placements.push(IndexPlacement {
op_index: *op_index,
folder,
id: pos as u32 + 1,
});
} else if let PendingIndexKind::Blank { .. } = &edit.kind {
let _ = folder;
}
}
}
Ok(placements)
}
fn merge_index_rewrite(
abs: &Path,
appends: &[&String],
mutations: &[&PendingIndexEdit],
contention_timeout: Duration,
) -> Result<(), NbError> {
use std::io::{Read as _, Seek as _, Write as _};
let io_err = |e: std::io::Error| NbError::Io {
path: abs.to_path_buf(),
source: e.into(),
};
let mut fd = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(abs)
.map_err(io_err)?;
for _ in 0..25 {
let s0 = fd.metadata().map_err(io_err)?.len();
fd.seek(std::io::SeekFrom::Start(0)).map_err(io_err)?;
let mut base = Vec::new();
fd.read_to_end(&mut base).map_err(io_err)?;
if fd.metadata().map_err(io_err)?.len() != s0 {
continue; }
#[cfg(feature = "testing")]
if let Ok(ms) = std::env::var("NB_API_INDEX_SPLICE_PAUSE_MS")
&& let Ok(ms) = ms.parse::<u64>()
{
std::thread::sleep(Duration::from_millis(ms));
}
let mut lines = parse_index_lines(&base);
for edit in mutations {
match &edit.kind {
PendingIndexKind::Append { .. } => {},
PendingIndexKind::Blank { basename } => {
if let Some(pos) = lines.iter().position(|l| l == basename)
{
lines[pos] = String::new();
}
},
PendingIndexKind::Rename {
old_basename,
new_basename,
} => {
if let Some(pos) =
lines.iter().position(|l| l == old_basename)
{
lines[pos] = new_basename.clone();
}
},
}
}
for basename in appends {
if !lines.iter().any(|l| l == *basename) {
lines.push((*basename).clone());
}
}
let rendered = render_index_lines(&lines);
if rendered.as_slice() == base.as_slice() {
return Ok(()); }
if rendered.len() as u64 > s0 {
fd.seek(std::io::SeekFrom::Start(0)).map_err(io_err)?;
fd.write_all(&rendered).map_err(io_err)?;
fd.set_len(rendered.len() as u64).map_err(io_err)?;
if fd.metadata().map_err(io_err)?.len() == rendered.len() as u64 {
return Ok(());
}
continue;
}
let prefix_len = rendered.len() as u64;
fd.seek(std::io::SeekFrom::Start(0)).map_err(io_err)?;
fd.write_all(&rendered).map_err(io_err)?;
if fd.metadata().map_err(io_err)?.len() != s0 {
continue; }
let s1 = fd.metadata().map_err(io_err)?.len();
fd.seek(std::io::SeekFrom::Start(s0)).map_err(io_err)?;
if fd.metadata().map_err(io_err)?.len() != s1 {
continue;
}
let mut tail = Vec::new();
let mut take = (s1 - s0) as usize;
while take > 0 {
let mut chunk = vec![0u8; take.min(8192)];
let n = fd.read(&mut chunk).map_err(io_err)?;
if n == 0 {
break;
}
tail.extend_from_slice(&chunk[..n]);
take -= n;
}
if fd.metadata().map_err(io_err)?.len() != s1 {
continue;
}
#[cfg(feature = "testing")]
if let Ok(ms) = std::env::var("NB_API_INDEX_SPLICE_PAUSE_MS")
&& let Ok(ms) = ms.parse::<u64>()
{
std::thread::sleep(Duration::from_millis(ms));
}
fd.seek(std::io::SeekFrom::Start(prefix_len))
.map_err(io_err)?;
fd.write_all(&tail).map_err(io_err)?;
let tail_len = tail.len() as u64;
let stale_len = s0.saturating_sub(prefix_len + tail_len);
let expected = prefix_len + (s1 - s0);
let collapsed =
collapse_stale_range(&fd, prefix_len + tail_len, stale_len);
if !collapsed {
fd.set_len(prefix_len + tail_len).map_err(io_err)?;
}
if fd.metadata().map_err(io_err)?.len() == expected {
return Ok(());
}
}
Err(NbError::IndexLockTimeout {
path: abs.to_string_lossy().into_owned(),
timeout_ms: contention_timeout.as_millis() as u64,
})
}
#[cfg(target_os = "linux")]
fn collapse_stale_range(fd: &std::fs::File, offset: u64, len: u64) -> bool {
use std::os::fd::AsRawFd;
if len == 0 {
return true;
}
const FALLOC_FL_COLLAPSE_RANGE: libc::c_int = 0x08;
let r = unsafe {
libc::fallocate(
fd.as_raw_fd(),
FALLOC_FL_COLLAPSE_RANGE,
offset as libc::off_t,
len as libc::off_t,
)
};
r == 0
}
#[cfg(not(target_os = "linux"))]
fn collapse_stale_range(_fd: &std::fs::File, _offset: u64, _len: u64) -> bool {
false
}
fn pending_basename_path(file_rel: &str, basename: &str) -> String {
if file_rel == ".index" {
basename.to_string()
} else if let Some(folder) = file_rel.strip_suffix("/.index") {
format!("{folder}/{basename}")
} else {
basename.to_string()
}
}
pub(super) fn index_id_on_disk(
notebook_root: &Path,
rel_path: &str,
) -> Option<u32> {
let (folder, basename) = split_folder_basename(rel_path);
index_id_in_folder(notebook_root, &folder, &basename)
}
pub(crate) fn index_id_in_folder(
notebook_root: &Path,
folder: &str,
basename: &str,
) -> Option<u32> {
let abs = notebook_root.join(index_rel_for_folder(folder));
let bytes = std::fs::read(abs).ok()?;
parse_index_lines(&bytes)
.iter()
.position(|l| l == basename)
.map(|p| p as u32 + 1)
}