use std::{
collections::{HashMap, VecDeque},
fs,
path::{Path, PathBuf},
sync::{Arc, Mutex},
};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use uuid::Uuid;
use rho_tools::tool::ToolError;
const MEMORY_ENTRY_LIMIT: usize = 32;
const MEMORY_BYTE_LIMIT: usize = 16 * 1024 * 1024;
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub(super) struct StoredContent {
pub(super) kind: String,
pub(super) items: Vec<StoredItem>,
}
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub(super) struct StoredItem {
pub(super) url: Option<String>,
pub(super) query: Option<String>,
pub(super) title: Option<String>,
pub(super) content: String,
pub(super) metadata: Value,
}
#[derive(Clone, Debug, Default)]
pub struct WebAccessStore {
state: Arc<Mutex<WebAccessStoreState>>,
}
#[derive(Debug, Default)]
struct WebAccessStoreState {
session_root: Option<PathBuf>,
memory: MemoryCache,
#[cfg(test)]
override_root: Option<PathBuf>,
}
#[derive(Debug)]
struct MemoryCache {
entries: HashMap<String, CachedContent>,
order: VecDeque<String>,
bytes: usize,
entry_limit: usize,
byte_limit: usize,
}
#[derive(Clone, Debug)]
struct CachedContent {
content: StoredContent,
bytes: usize,
}
impl Default for MemoryCache {
fn default() -> Self {
Self::new(MEMORY_ENTRY_LIMIT, MEMORY_BYTE_LIMIT)
}
}
impl MemoryCache {
fn new(entry_limit: usize, byte_limit: usize) -> Self {
Self {
entries: HashMap::new(),
order: VecDeque::new(),
bytes: 0,
entry_limit: entry_limit.max(1),
byte_limit: byte_limit.max(1),
}
}
fn get(&mut self, response_id: &str) -> Option<StoredContent> {
let content = self.entries.get(response_id)?.content.clone();
self.touch(response_id);
Some(content)
}
fn insert(&mut self, response_id: String, content: StoredContent) {
let bytes = memory_bytes(&content);
self.remove(&response_id);
if bytes > self.byte_limit {
return;
}
while !self.entries.is_empty()
&& (self.entries.len() >= self.entry_limit
|| self.bytes.saturating_add(bytes) > self.byte_limit)
{
self.evict_oldest();
}
self.bytes = self.bytes.saturating_add(bytes);
self.order.push_back(response_id.clone());
self.entries
.insert(response_id, CachedContent { content, bytes });
}
#[cfg(test)]
fn contains(&self, response_id: &str) -> bool {
self.entries.contains_key(response_id)
}
fn touch(&mut self, response_id: &str) {
if let Some(index) = self
.order
.iter()
.position(|existing| existing == response_id)
{
if let Some(id) = self.order.remove(index) {
self.order.push_back(id);
}
}
}
fn evict_oldest(&mut self) {
if let Some(id) = self.order.pop_front() {
self.remove_entry(&id);
}
}
fn remove(&mut self, response_id: &str) {
self.order.retain(|existing| existing != response_id);
self.remove_entry(response_id);
}
fn remove_entry(&mut self, response_id: &str) {
if let Some(entry) = self.entries.remove(response_id) {
self.bytes = self.bytes.saturating_sub(entry.bytes);
}
}
}
fn memory_bytes(content: &StoredContent) -> usize {
content.kind.len()
+ content
.items
.iter()
.map(|item| {
item.content.len()
+ item.url.as_deref().map_or(0, str::len)
+ item.query.as_deref().map_or(0, str::len)
+ item.title.as_deref().map_or(0, str::len)
})
.sum::<usize>()
}
impl WebAccessStore {
pub fn new() -> Self {
Self::default()
}
pub fn bind_session(&self, root: Option<PathBuf>) {
let mut state = self.state.lock().expect("web access store lock poisoned");
state.session_root = root;
state.memory = MemoryCache::default();
}
pub fn root(&self) -> PathBuf {
let state = self.state.lock().expect("web access store lock poisoned");
#[cfg(test)]
if let Some(path) = state.override_root.clone() {
return path;
}
if let Some(path) = state.session_root.clone() {
return path;
}
default_web_access_cache_root()
}
pub(super) fn store(
&self,
response_id: String,
content: StoredContent,
) -> Result<(), ToolError> {
write_at(&self.root(), &response_id, &content)?;
self.state
.lock()
.expect("web access store lock poisoned")
.memory
.insert(response_id, content);
Ok(())
}
pub(super) fn load(&self, response_id: &str) -> Result<StoredContent, ToolError> {
validate_response_id(response_id)?;
{
let mut state = self.state.lock().expect("web access store lock poisoned");
if let Some(content) = state.memory.get(response_id) {
return Ok(content);
}
}
let content = read_at(&self.root(), response_id)?;
self.state
.lock()
.expect("web access store lock poisoned")
.memory
.insert(response_id.to_owned(), content.clone());
Ok(content)
}
pub(super) fn create_private_dir_all(&self, path: &Path) -> Result<(), ToolError> {
fs::create_dir_all(path)?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let root = self.root();
if root.exists() {
fs::set_permissions(&root, fs::Permissions::from_mode(0o700))?;
}
if path.exists() {
fs::set_permissions(path, fs::Permissions::from_mode(0o700))?;
}
}
Ok(())
}
#[cfg(test)]
pub(super) fn with_root(path: PathBuf) -> Self {
let store = Self::new();
store
.state
.lock()
.expect("web access store lock poisoned")
.override_root = Some(path);
store
}
#[cfg(test)]
fn memory_contains(&self, response_id: &str) -> bool {
self.state
.lock()
.expect("web access store lock poisoned")
.memory
.contains(response_id)
}
}
pub(super) fn new_response_id() -> String {
Uuid::new_v4().simple().to_string()
}
pub(super) fn validate_response_id(response_id: &str) -> Result<(), ToolError> {
let valid = response_id.len() == 32
&& response_id
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte));
if valid {
Ok(())
} else {
Err(ToolError::Message(
"invalid responseId: expected 32 lowercase hexadecimal characters".into(),
))
}
}
pub(super) fn available_selectors(stored: &StoredContent) -> String {
if stored.items.is_empty() {
return "no stored items".into();
}
let mut lines = Vec::with_capacity(stored.items.len());
let mut query_index = 0usize;
for (index, item) in stored.items.iter().enumerate() {
let mut parts = vec![format!("urlIndex={index}")];
if let Some(url) = item.url.as_deref() {
parts.push(format!("url={url}"));
}
if let Some(query) = item.query.as_deref() {
parts.push(format!("query={query:?}"));
parts.push(format!("queryIndex={query_index}"));
query_index += 1;
}
lines.push(format!("- {}", parts.join(" ")));
}
lines.join("\n")
}
fn default_web_access_cache_root() -> PathBuf {
crate::paths::rho_dir()
.map(|dir| dir.join("web-access"))
.unwrap_or_else(|_| std::env::temp_dir().join("rho-web-access"))
}
fn write_at(root: &Path, response_id: &str, content: &StoredContent) -> Result<(), ToolError> {
let path = stored_content_path(root, response_id)?;
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
if root.exists() {
fs::set_permissions(root, fs::Permissions::from_mode(0o700))?;
}
if parent.exists() {
fs::set_permissions(parent, fs::Permissions::from_mode(0o700))?;
}
}
}
let serialized = serde_json::to_string(content)
.map_err(|err| ToolError::Message(format!("failed to serialize stored content: {err}")))?;
write_private_file(&path, serialized.as_bytes())
}
fn read_at(root: &Path, response_id: &str) -> Result<StoredContent, ToolError> {
let path = stored_content_path(root, response_id)?;
match fs::read_to_string(&path) {
Ok(content) => parse_stored_content(&content),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => read_legacy_temp(response_id),
Err(error) => Err(ToolError::Message(format!(
"failed to read stored web content {}: {error}",
path.display()
))),
}
}
fn read_legacy_temp(response_id: &str) -> Result<StoredContent, ToolError> {
let legacy = std::env::temp_dir()
.join("rho-web-access")
.join("content")
.join(format!("{response_id}.json"));
let content = fs::read_to_string(&legacy).map_err(|_| {
ToolError::Message(format!(
"unknown responseId: {response_id}. Stored web content is a sidecar blob under the Rho data directory and is available only while that cache file exists. Re-run fetch_content or web_search for the original URL or query."
))
})?;
parse_stored_content(&content)
}
fn parse_stored_content(content: &str) -> Result<StoredContent, ToolError> {
serde_json::from_str(content)
.map_err(|err| ToolError::Message(format!("stored content was not valid JSON: {err}")))
}
fn stored_content_path(root: &Path, response_id: &str) -> Result<PathBuf, ToolError> {
validate_response_id(response_id)?;
Ok(root.join("content").join(format!("{response_id}.json")))
}
fn write_private_file(path: &Path, contents: &[u8]) -> Result<(), ToolError> {
#[cfg(unix)]
{
use std::io::Write;
use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};
let mut file = fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.mode(0o600)
.open(path)?;
file.write_all(contents)?;
fs::set_permissions(path, fs::Permissions::from_mode(0o600))?;
Ok(())
}
#[cfg(not(unix))]
{
fs::write(path, contents)?;
Ok(())
}
}
#[cfg(test)]
#[path = "storage_tests.rs"]
mod tests;