#[cfg(feature = "lineage")]
pub mod lineage;
use serde::{Deserialize, Serialize};
use crate::manifest::{NoFetcher, delete_manifest};
use crate::schema::{
Hash, ManifestId, ModerationState, PrivacyClass, ProvenanceOrigin, ProvenanceRecord, SchemaRef,
Timestamp, TrustEnvelope, TrustLevel,
};
use crate::typed::{TypedPayload, list_typed, load_typed, save_typed};
use crate::{Result, Store};
const BROWSING_TRACE_SCHEMA_PAYLOAD: &[u8] = br#"{"format":"mere-native","schema_id":"eidetic.BrowsingTrace/v1","body":{"version":1,"description":"A chronological segment of one owner's browsing traversals.","required":["owner","events","started_at_ms","ended_at_ms"],"fields":{"owner":{"type":"string"},"events":{"type":"array"},"started_at_ms":{"type":"integer"},"ended_at_ms":{"type":"integer"}}}}"#;
pub static BROWSING_TRACE_SCHEMA_REF: std::sync::LazyLock<SchemaRef> =
std::sync::LazyLock::new(|| {
SchemaRef::from_id(ManifestId::from_hash(Hash::of(
BROWSING_TRACE_SCHEMA_PAYLOAD,
)))
});
pub async fn bootstrap_browsing_schema(store: &mut dyn Store) -> Result<()> {
let id = BROWSING_TRACE_SCHEMA_REF.0;
if crate::manifest::load_manifest(store, id).await?.is_some() {
return Ok(());
}
let local_key = format!("blob:{}", Hash::of(BROWSING_TRACE_SCHEMA_PAYLOAD).to_hex());
store.put(&local_key, BROWSING_TRACE_SCHEMA_PAYLOAD).await?;
let manifest = crate::manifest::BlobManifest {
id,
schema: *crate::schema_def::META_SCHEMA_REF,
content_hash: Hash::of(BROWSING_TRACE_SCHEMA_PAYLOAD),
byte_size: BROWSING_TRACE_SCHEMA_PAYLOAD.len() as u64,
created_at: Timestamp::ZERO,
last_accessed: None,
sources: vec![crate::manifest::BlobSource::Local { key: local_key }],
privacy: PrivacyClass::PublicPortable,
provenance: ProvenanceRecord {
origin: ProvenanceOrigin::Generated,
upstream: Vec::new(),
tooling: Some(format!("eidetic/{}", env!("CARGO_PKG_VERSION"))),
generated_at: Timestamp::ZERO,
},
trust: TrustEnvelope {
level: TrustLevel::CheckpointAccepted,
signatures: Vec::new(),
moderation_state: ModerationState::Accepted,
},
schema_metadata: serde_json::Value::Null,
manifest_version: crate::manifest::BlobManifest::CURRENT_VERSION,
};
crate::manifest::save_manifest(store, &manifest).await
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PageRef {
pub url: String,
pub title: Option<String>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum TraceTransition {
LinkClick,
UrlTyped,
Back,
Forward,
Reload,
Redirect,
TabSpawn,
Restore,
Imported,
Unknown,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TraceEvent {
pub from: Option<PageRef>,
pub to: PageRef,
pub transition: TraceTransition,
pub at_ms: u64,
pub dwell_ms: Option<u64>,
#[serde(default)]
pub candidates: Vec<PageRef>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BrowsingTrace {
pub owner: String,
pub events: Vec<TraceEvent>,
pub started_at_ms: u64,
pub ended_at_ms: u64,
}
impl BrowsingTrace {
pub fn from_events(owner: impl Into<String>, events: Vec<TraceEvent>) -> Self {
let started_at_ms = events.first().map(|e| e.at_ms).unwrap_or(0);
let ended_at_ms = events.last().map(|e| e.at_ms).unwrap_or(0);
Self {
owner: owner.into(),
events,
started_at_ms,
ended_at_ms,
}
}
}
impl TypedPayload for BrowsingTrace {
fn schema_ref() -> SchemaRef {
*BROWSING_TRACE_SCHEMA_REF
}
}
pub async fn save_trace(
store: &mut dyn Store,
trace: &BrowsingTrace,
now_ms: u64,
) -> Result<ManifestId> {
save_typed(
store,
trace,
Vec::new(),
PrivacyClass::LocalOnly,
ProvenanceRecord {
origin: ProvenanceOrigin::Generated,
upstream: Vec::new(),
tooling: Some(format!("eidetic-browsing/{}", env!("CARGO_PKG_VERSION"))),
generated_at: Timestamp(now_ms),
},
TrustEnvelope {
level: TrustLevel::SelfAsserted,
signatures: Vec::new(),
moderation_state: ModerationState::Unreviewed,
},
Timestamp(now_ms),
)
.await
}
#[derive(Clone)]
pub struct BrowsingMemory {
traces: Vec<(Option<ManifestId>, BrowsingTrace)>,
open: std::collections::BTreeMap<String, Vec<TraceEvent>>,
segment_size: usize,
}
impl BrowsingMemory {
pub fn new(segment_size: usize) -> Self {
Self {
traces: Vec::new(),
open: std::collections::BTreeMap::new(),
segment_size: segment_size.max(1),
}
}
pub async fn load(store: &mut dyn Store, segment_size: usize) -> Result<Self> {
let mut memory = Self::new(segment_size);
let mut fetcher = NoFetcher;
for manifest in list_typed::<BrowsingTrace>(store).await? {
if let Some(trace) =
load_typed::<BrowsingTrace>(store, &mut fetcher, manifest.id).await?
{
memory.traces.push((Some(manifest.id), trace));
}
}
memory.sort_traces();
Ok(memory)
}
fn sort_traces(&mut self) {
self.traces.sort_by_key(|(_, t)| t.ended_at_ms);
}
pub fn record_traversal(&mut self, owner: &str, event: TraceEvent) -> bool {
let segment = self.open.entry(owner.to_string()).or_default();
segment.push(event);
segment.len() >= self.segment_size
}
pub async fn flush(&mut self, store: &mut dyn Store, now_ms: u64) -> Result<Vec<ManifestId>> {
let mut ids = Vec::new();
let owners: Vec<String> = self.open.keys().cloned().collect();
for owner in owners {
let events = self.open.remove(&owner).unwrap_or_default();
if events.is_empty() {
continue;
}
let trace = BrowsingTrace::from_events(owner, events);
let id = save_trace(store, &trace, now_ms).await?;
ids.push(id);
self.traces.push((Some(id), trace));
}
self.sort_traces();
Ok(ids)
}
pub fn adopt(&mut self, trace: BrowsingTrace) {
self.traces.push((None, trace));
self.sort_traces();
}
fn all_events(&self) -> impl Iterator<Item = &TraceEvent> {
self.traces
.iter()
.flat_map(|(_, t)| t.events.iter())
.chain(self.open.values().flatten())
}
pub fn recent_corridor(&self, n: usize) -> Vec<TraceEvent> {
let all: Vec<&TraceEvent> = self.all_events().collect();
let start = all.len().saturating_sub(n);
all[start..].iter().map(|e| (*e).clone()).collect()
}
pub fn nodes_visited_in_window(&self, start_ms: u64, end_ms: u64) -> Vec<PageRef> {
let mut seen = std::collections::HashSet::new();
let mut out = Vec::new();
for event in self.all_events() {
if event.at_ms < start_ms || event.at_ms > end_ms {
continue;
}
if seen.insert(event.to.url.clone()) {
out.push(event.to.clone());
}
}
out
}
pub fn co_occurrence(&self, a: &str, b: &str) -> u32 {
self.all_events()
.filter(|e| {
let Some(from) = &e.from else { return false };
(from.url == a && e.to.url == b) || (from.url == b && e.to.url == a)
})
.count() as u32
}
pub fn traces(&self) -> impl Iterator<Item = &BrowsingTrace> {
self.traces.iter().map(|(_, t)| t)
}
pub async fn apply_quota(&mut self, store: &mut dyn Store, keep_n: usize) -> Result<usize> {
let stored: Vec<ManifestId> = self.traces.iter().filter_map(|(id, _)| *id).collect();
if stored.len() <= keep_n {
return Ok(0);
}
let overflow: Vec<ManifestId> = stored[..stored.len() - keep_n].to_vec();
let mut deleted = 0;
for id in &overflow {
if delete_manifest(store, *id).await? {
deleted += 1;
}
}
self.traces
.retain(|(id, _)| id.is_none_or(|id| !overflow.contains(&id)));
Ok(deleted)
}
pub async fn forget_url(&mut self, store: &mut dyn Store, url: &str) -> Result<usize> {
let mentions = |trace: &BrowsingTrace| {
trace
.events
.iter()
.any(|e| e.to.url == url || e.from.as_ref().is_some_and(|f| f.url == url))
};
let mut forgotten = 0;
let mut remaining = Vec::with_capacity(self.traces.len());
for (id, trace) in std::mem::take(&mut self.traces) {
if mentions(&trace) {
if let Some(id) = id {
delete_manifest(store, id).await?;
}
forgotten += 1;
} else {
remaining.push((id, trace));
}
}
self.traces = remaining;
for events in self.open.values_mut() {
events.retain(|e| !(e.to.url == url || e.from.as_ref().is_some_and(|f| f.url == url)));
}
Ok(forgotten)
}
}
#[cfg(test)]
mod tests;