use anyhow::{Context, Result};
use memmap2::Mmap;
use std::fs::{File, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
const MAGIC: &[u8; 4] = b"RFCT";
const VERSION: u32 = 2;
const ENTRY_SIZE: usize = 28;
const HEADER_SIZE: usize = 32;
#[derive(Debug, Clone)]
pub struct FileEntry {
pub path: PathBuf,
pub offset: u64,
pub length: u64,
}
pub struct ContentWriter {
files: Vec<FileEntry>,
writer: Option<std::io::BufWriter<File>>,
current_offset: u64,
file_path: Option<PathBuf>,
content: Vec<u8>,
write_error: Option<std::io::Error>,
}
impl ContentWriter {
pub fn new() -> Self {
Self {
files: Vec::new(),
writer: None,
current_offset: 0,
file_path: None,
content: Vec::new(),
write_error: None,
}
}
pub fn init(&mut self, path: PathBuf) -> Result<()> {
let tmp_path = crate::atomic_write::tmp_path_for(&path);
let file = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(&tmp_path)
.with_context(|| format!("Failed to create {}", tmp_path.display()))?;
let mut writer = std::io::BufWriter::with_capacity(16 * 1024 * 1024, file);
writer.write_all(MAGIC)?;
writer.write_all(&VERSION.to_le_bytes())?;
writer.write_all(&0u64.to_le_bytes())?; writer.write_all(&0u64.to_le_bytes())?; writer.write_all(&[0u8; 8])?;
self.writer = Some(writer);
self.current_offset = 0; self.file_path = Some(path);
Ok(())
}
pub fn add_file(&mut self, path: PathBuf, content: &str) -> u32 {
let file_id = self.files.len() as u32;
let content_bytes = content.as_bytes();
let length = content_bytes.len() as u64;
if let Some(ref mut w) = self.writer {
let offset = self.current_offset;
if let Err(e) = w.write_all(content_bytes)
&& self.write_error.is_none()
{
self.write_error = Some(e);
}
self.current_offset += length;
self.files.push(FileEntry {
path,
offset,
length,
});
} else {
let offset = self.content.len() as u64;
self.content.extend_from_slice(content_bytes);
self.files.push(FileEntry {
path,
offset,
length,
});
}
file_id
}
pub fn write(&mut self, path: impl AsRef<Path>) -> Result<()> {
let path = path.as_ref();
if self.writer.is_none() && self.file_path.is_none() {
return self.write_legacy(path);
}
self.finalize_if_needed()?;
Ok(())
}
fn write_legacy(&self, path: impl AsRef<Path>) -> Result<()> {
let path = path.as_ref();
let tmp_path = crate::atomic_write::tmp_path_for(path);
let file = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(&tmp_path)
.with_context(|| format!("Failed to create {}", tmp_path.display()))?;
let mut writer = std::io::BufWriter::with_capacity(8 * 1024 * 1024, file);
let index_offset = HEADER_SIZE as u64 + self.content.len() as u64;
writer.write_all(MAGIC)?;
writer.write_all(&VERSION.to_le_bytes())?;
writer.write_all(&(self.files.len() as u64).to_le_bytes())?;
writer.write_all(&index_offset.to_le_bytes())?;
writer.write_all(&[0u8; 8])?;
writer.write_all(&self.content)?;
write_file_index(&mut writer, &self.files, index_offset)?;
writer.flush()?;
writer.get_ref().sync_all()?;
crate::atomic_write::atomic_replace(&tmp_path, path)
.with_context(|| format!("Failed to move {} into place", path.display()))?;
Ok(())
}
fn finalize(&mut self) -> Result<()> {
let mut writer = self
.writer
.take()
.ok_or_else(|| anyhow::anyhow!("ContentWriter not initialized"))?;
let final_path = self
.file_path
.clone()
.ok_or_else(|| anyhow::anyhow!("ContentWriter has no output path"))?;
let tmp_path = crate::atomic_write::tmp_path_for(&final_path);
if let Some(e) = self.write_error.take() {
let _ = std::fs::remove_file(&tmp_path);
return Err(anyhow::Error::new(e).context(format!(
"Failed to write file content to {}",
tmp_path.display()
)));
}
let index_offset = HEADER_SIZE as u64 + self.current_offset;
write_file_index(&mut writer, &self.files, index_offset)?;
let mut file = writer
.into_inner()
.map_err(|e| anyhow::anyhow!("Failed to flush BufWriter: {}", e.error()))?;
use std::io::Seek;
file.seek(std::io::SeekFrom::Start(0))?;
file.write_all(MAGIC)?;
file.write_all(&VERSION.to_le_bytes())?;
file.write_all(&(self.files.len() as u64).to_le_bytes())?;
file.write_all(&index_offset.to_le_bytes())?;
file.write_all(&[0u8; 8])?;
file.sync_all()?;
drop(file);
crate::atomic_write::atomic_replace(&tmp_path, &final_path)
.with_context(|| format!("Failed to move {} into place", final_path.display()))?;
log::debug!(
"Finalized content.bin: {} files, {} bytes of content",
self.files.len(),
self.current_offset
);
Ok(())
}
pub fn file_count(&self) -> usize {
self.files.len()
}
pub fn content_size(&self) -> usize {
if self.writer.is_some() || self.file_path.is_some() {
self.current_offset as usize
} else {
self.content.len()
}
}
pub fn finalize_if_needed(&mut self) -> Result<()> {
if self.writer.is_some() {
self.finalize()?;
self.writer = None;
}
Ok(())
}
}
impl Default for ContentWriter {
fn default() -> Self {
Self::new()
}
}
fn write_file_index<W: Write>(
writer: &mut W,
files: &[FileEntry],
index_offset: u64,
) -> Result<()> {
let blob_start = index_offset + (files.len() * ENTRY_SIZE) as u64;
let mut path_pos = blob_start;
for entry in files {
let path_len = entry.path.to_string_lossy().len() as u64;
writer.write_all(&entry.offset.to_le_bytes())?;
writer.write_all(&entry.length.to_le_bytes())?;
writer.write_all(&path_pos.to_le_bytes())?;
writer.write_all(&(path_len as u32).to_le_bytes())?;
path_pos += path_len;
}
for entry in files {
writer.write_all(entry.path.to_string_lossy().as_bytes())?;
}
Ok(())
}
pub struct ContentReader {
_file: File,
mmap: Mmap,
num_files: usize,
index_offset: usize,
}
#[derive(Debug, Clone, Copy)]
struct Entry<'a> {
offset: u64,
length: u64,
path: &'a str,
}
impl ContentReader {
pub fn open(path: impl AsRef<Path>) -> Result<Self> {
let path = path.as_ref();
let file =
File::open(path).with_context(|| format!("Failed to open {}", path.display()))?;
let mmap = unsafe {
Mmap::map(&file).with_context(|| format!("Failed to mmap {}", path.display()))?
};
if mmap.len() < HEADER_SIZE {
anyhow::bail!(
"content.bin too small (expected at least {} bytes)",
HEADER_SIZE
);
}
if &mmap[0..4] != MAGIC {
anyhow::bail!("Invalid content.bin (wrong magic bytes)");
}
let version = u32::from_le_bytes([mmap[4], mmap[5], mmap[6], mmap[7]]);
if version != VERSION {
anyhow::bail!("Unsupported content.bin version: {}", version);
}
let num_files = u64::from_le_bytes([
mmap[8], mmap[9], mmap[10], mmap[11], mmap[12], mmap[13], mmap[14], mmap[15],
]);
let index_offset = u64::from_le_bytes([
mmap[16], mmap[17], mmap[18], mmap[19], mmap[20], mmap[21], mmap[22], mmap[23],
]) as usize;
let num_files = num_files as usize;
let table_end = index_offset.saturating_add(num_files.saturating_mul(ENTRY_SIZE));
if table_end > mmap.len() {
anyhow::bail!(
"Truncated file index (index_offset={}, num_files={}, mmap.len()={})",
index_offset,
num_files,
mmap.len()
);
}
let reader = Self {
_file: file,
mmap,
num_files,
index_offset,
};
if num_files > 0 {
for id in [0u32, (num_files - 1) as u32] {
if reader.entry(id).is_none() {
anyhow::bail!("Truncated file entry at file {}", id);
}
}
}
Ok(reader)
}
fn entry(&self, file_id: u32) -> Option<Entry<'_>> {
let id = file_id as usize;
if id >= self.num_files {
return None;
}
let at = self.index_offset + id * ENTRY_SIZE;
let b = self.mmap.get(at..at + ENTRY_SIZE)?;
let u64_at = |i: usize| u64::from_le_bytes(b[i..i + 8].try_into().unwrap());
let offset = u64_at(0);
let length = u64_at(8);
let path_pos = u64_at(16) as usize;
let path_len = u32::from_le_bytes(b[24..28].try_into().unwrap()) as usize;
let path = std::str::from_utf8(self.mmap.get(path_pos..path_pos + path_len)?).ok()?;
Some(Entry {
offset,
length,
path,
})
}
pub fn release_pages(&self) {
#[cfg(unix)]
unsafe {
let _ = self
.mmap
.unchecked_advise(memmap2::UncheckedAdvice::DontNeed);
}
}
pub fn file_len(&self, file_id: u32) -> Option<u64> {
self.entry(file_id).map(|e| e.length)
}
pub fn get_file_content(&self, file_id: u32) -> Result<&str> {
let entry = self
.entry(file_id)
.ok_or_else(|| anyhow::anyhow!("Invalid file_id: {}", file_id))?;
let start = HEADER_SIZE + entry.offset as usize;
let end = start + entry.length as usize;
if end > self.mmap.len() {
anyhow::bail!("File content out of bounds");
}
let bytes = &self.mmap[start..end];
std::str::from_utf8(bytes).context("Invalid UTF-8 in file content")
}
pub fn get_file_path(&self, file_id: u32) -> Option<&Path> {
self.entry(file_id).map(|e| Path::new(e.path))
}
pub fn file_count(&self) -> usize {
self.num_files
}
pub fn get_file_id_by_path(&self, path: &str) -> Option<u32> {
let normalized_input = path.strip_prefix("./").unwrap_or(path);
(0..self.num_files as u32).find(|&id| {
self.entry(id).is_some_and(|entry| {
entry.path.strip_prefix("./").unwrap_or(entry.path) == normalized_input
})
})
}
pub fn get_content_at_offset(
&self,
file_id: u32,
byte_offset: u32,
length: usize,
) -> Result<&str> {
let entry = self
.entry(file_id)
.ok_or_else(|| anyhow::anyhow!("Invalid file_id: {}", file_id))?;
let start = HEADER_SIZE + entry.offset as usize + byte_offset as usize;
let end = start + length;
if end > self.mmap.len() {
anyhow::bail!("Content out of bounds");
}
let bytes = &self.mmap[start..end];
std::str::from_utf8(bytes).context("Invalid UTF-8 in content")
}
pub fn get_context(
&self,
file_id: u32,
byte_offset: u32,
context_lines: usize,
) -> Result<(Vec<String>, String, Vec<String>)> {
let content = self.get_file_content(file_id)?;
let lines: Vec<&str> = content.lines().collect();
let mut current_offset = 0;
let mut line_idx = 0;
for (idx, line) in lines.iter().enumerate() {
let line_end = current_offset + line.len() + 1; if byte_offset as usize >= current_offset && (byte_offset as usize) < line_end {
line_idx = idx;
break;
}
current_offset = line_end;
}
let start = line_idx.saturating_sub(context_lines);
let end = (line_idx + context_lines + 1).min(lines.len());
let before: Vec<String> = lines[start..line_idx]
.iter()
.map(|s| s.to_string())
.collect();
let matching = lines
.get(line_idx)
.map(|s| s.to_string())
.unwrap_or_default();
let after: Vec<String> = lines[line_idx + 1..end]
.iter()
.map(|s| s.to_string())
.collect();
Ok((before, matching, after))
}
pub fn get_context_by_line(
&self,
file_id: u32,
line_number: usize,
context_lines: usize,
) -> Result<(Vec<String>, Vec<String>)> {
let content = self.get_file_content(file_id)?;
let lines: Vec<&str> = content.lines().collect();
let line_idx = line_number.saturating_sub(1);
let start = line_idx.saturating_sub(context_lines);
let end = (line_idx + context_lines + 1).min(lines.len());
let before: Vec<String> = lines[start..line_idx]
.iter()
.map(|s| s.to_string())
.collect();
let after: Vec<String> = lines[line_idx + 1..end]
.iter()
.map(|s| s.to_string())
.collect();
Ok((before, after))
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
#[test]
fn test_content_writer_basic() {
let mut writer = ContentWriter::new();
let file1_id = writer.add_file(PathBuf::from("test1.txt"), "Hello, world!");
let file2_id = writer.add_file(PathBuf::from("test2.txt"), "Goodbye, world!");
assert_eq!(file1_id, 0);
assert_eq!(file2_id, 1);
assert_eq!(writer.file_count(), 2);
}
#[test]
fn test_content_roundtrip() {
let temp = TempDir::new().unwrap();
let content_path = temp.path().join("content.bin");
let mut writer = ContentWriter::new();
writer.add_file(PathBuf::from("file1.txt"), "First file content");
writer.add_file(PathBuf::from("file2.txt"), "Second file content");
writer.write(&content_path).unwrap();
let reader = ContentReader::open(&content_path).unwrap();
assert_eq!(reader.file_count(), 2);
assert_eq!(reader.get_file_content(0).unwrap(), "First file content");
assert_eq!(reader.get_file_content(1).unwrap(), "Second file content");
assert_eq!(reader.get_file_path(0).unwrap(), Path::new("file1.txt"));
assert_eq!(reader.get_file_path(1).unwrap(), Path::new("file2.txt"));
}
#[test]
fn test_get_context() {
let temp = TempDir::new().unwrap();
let content_path = temp.path().join("content.bin");
let mut writer = ContentWriter::new();
writer.add_file(
PathBuf::from("test.txt"),
"Line 1\nLine 2\nLine 3 with match\nLine 4\nLine 5",
);
writer.write(&content_path).unwrap();
let reader = ContentReader::open(&content_path).unwrap();
let (before, matching, after) = reader.get_context(0, 14, 1).unwrap();
assert_eq!(before.len(), 1);
assert_eq!(before[0], "Line 2");
assert_eq!(matching, "Line 3 with match");
assert_eq!(after.len(), 1);
assert_eq!(after[0], "Line 4");
}
#[test]
fn test_streaming_roundtrip() {
let temp = TempDir::new().unwrap();
let content_path = temp.path().join("content.bin");
let mut writer = ContentWriter::new();
writer.init(content_path.clone()).unwrap();
writer.add_file(PathBuf::from("src/main.rs"), "fn main() {}\n");
writer.add_file(
PathBuf::from("src/lib.rs"),
"pub fn hello() -> &'static str { \"hi\" }\n",
);
writer.finalize_if_needed().unwrap();
let reader = ContentReader::open(&content_path).unwrap();
assert_eq!(reader.file_count(), 2);
assert_eq!(reader.get_file_content(0).unwrap(), "fn main() {}\n");
assert_eq!(
reader.get_file_content(1).unwrap(),
"pub fn hello() -> &'static str { \"hi\" }\n"
);
assert_eq!(reader.get_file_path(0).unwrap(), Path::new("src/main.rs"));
assert_eq!(reader.get_file_path(1).unwrap(), Path::new("src/lib.rs"));
}
#[test]
fn test_multiline_file() {
let temp = TempDir::new().unwrap();
let content_path = temp.path().join("content.bin");
let content = "fn main() {\n println!(\"Hello\");\n}\n";
let mut writer = ContentWriter::new();
writer.add_file(PathBuf::from("main.rs"), content);
writer.write(&content_path).unwrap();
let reader = ContentReader::open(&content_path).unwrap();
assert_eq!(reader.get_file_content(0).unwrap(), content);
}
}