use std::collections::BTreeSet;
use std::fs;
use std::io::Read;
use std::path::Path;
use anyhow::{Context, Result, bail};
use chrono::{DateTime, Utc};
use serde::de::IgnoredAny;
use serde::{Deserialize, Serialize};
use unicode_width::UnicodeWidthStr;
use uuid::Uuid;
use crate::entities::attachment::format_bytes;
use crate::entities::chat::visible_row_count;
use crate::entities::chat_file::sha256_hex;
use crate::entities::message::MessageRole;
use crate::features::backup::{self, ArchiveReader, BackupManifest};
use crate::shared::i18n::Locale;
use crate::shared::paths::Paths;
use crate::shared::storage::db::{self, DbStats, KeyedRow};
use crate::shared::storage::schema::CHAT_SCHEMA;
pub const SNAPSHOT_FORMAT: u32 = 2;
const FINGERPRINT_HEX: usize = 16;
const PROFILES_ENTRY: &str = "profiles.json";
const DB_ENTRY: &str = "data.db";
const CHATS_PREFIX: &str = "chats/";
const BACKUP_MARKERS: &[&str] = &["manifest.json", "settings.json", PROFILES_ENTRY, DB_ENTRY];
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DataStats {
pub format: u32,
pub app_version: String,
pub taken_at: DateTime<Utc>,
pub fingerprint: String,
pub source: Source,
pub profiles: ProfileTotals,
pub chats: ChatTotals,
pub database: DatabaseStats,
pub sizes: Sizes,
pub chat_list: Vec<ChatRow>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Source {
DataRoot {
path: String,
},
Archive {
path: String,
manifest: Option<BackupManifest>,
newer_than_this_app: bool,
},
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProfileTotals {
pub total: usize,
pub deleted: usize,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ChatTotals {
pub total: usize,
pub deleted: usize,
pub unreadable: Vec<String>,
pub messages: usize,
pub messages_in_deleted_chats: usize,
pub message_rows: usize,
pub deleted_messages: usize,
pub deleted_exchanges: usize,
pub attachments: usize,
pub stored_files: usize,
pub images: usize,
pub projects: usize,
pub chats_with_project: usize,
pub subagent_runs: usize,
pub subagent_messages: usize,
pub last_message_at: Option<DateTime<Utc>>,
pub last_change_at: Option<DateTime<Utc>>,
pub newest_schema: u32,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "status", rename_all = "snake_case")]
pub enum DatabaseStats {
Missing,
Ok(DbStats),
Unreadable {
error: String,
},
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Sizes {
pub chats_bytes: u64,
pub database_bytes: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ChatRow {
pub id: Uuid,
pub profile_id: Option<Uuid>,
pub title: String,
pub deleted: bool,
pub created_at: Option<DateTime<Utc>>,
pub modified_at: Option<DateTime<Utc>>,
pub messages: usize,
pub message_rows: usize,
pub deleted_messages: usize,
pub last_message_at: Option<DateTime<Utc>>,
pub attachments: usize,
pub stored_files: usize,
pub message_index: Vec<String>,
pub deleted_index: Vec<String>,
}
#[derive(Debug, PartialEq, Eq, Serialize)]
struct ChatIdentity<'a> {
id: Uuid,
title: &'a str,
deleted: bool,
attachments: usize,
stored_files: usize,
message_index: &'a [String],
deleted_index: &'a [String],
}
impl ChatRow {
fn identity(&self) -> ChatIdentity<'_> {
ChatIdentity {
id: self.id,
title: &self.title,
deleted: self.deleted,
attachments: self.attachments,
stored_files: self.stored_files,
message_index: &self.message_index,
deleted_index: &self.deleted_index,
}
}
}
fn keyed_lists(database: &DatabaseStats) -> Option<[&[KeyedRow]; 3]> {
match database {
DatabaseStats::Missing => Some([&[], &[], &[]]),
DatabaseStats::Ok(d) => Some([&d.note_list, &d.source_list, &d.self_model_list]),
DatabaseStats::Unreadable { .. } => None,
}
}
impl DataStats {
fn fingerprint(chat_list: &[ChatRow], database: &DatabaseStats) -> String {
let chats: Vec<_> = chat_list.iter().map(ChatRow::identity).collect();
let rows: Vec<Vec<(&str, &str)>> = keyed_lists(database)
.unwrap_or_default()
.iter()
.map(|list| {
list.iter()
.map(|row| (row.key.as_str(), row.digest.as_str()))
.collect()
})
.collect();
let material = serde_json::to_vec(&(chats, rows)).unwrap_or_default();
let mut digest = sha256_hex(&material);
digest.truncate(FINGERPRINT_HEX);
digest
}
pub fn is_empty(&self) -> bool {
self.profiles.total == 0
&& self.chats.total == 0
&& self.chats.unreadable.is_empty()
&& self.database == DatabaseStats::Missing
}
}
#[derive(Deserialize)]
struct ChatLite {
#[serde(default = "chat_schema_v1")]
v: u32,
#[serde(default)]
profile_id: Option<Uuid>,
#[serde(default)]
title: String,
#[serde(default)]
created_at: Option<DateTime<Utc>>,
#[serde(default)]
modified_at: Option<DateTime<Utc>>,
#[serde(default)]
messages: Vec<MessageLite>,
#[serde(default)]
attachments: Vec<IgnoredAny>,
#[serde(default)]
files: Vec<IgnoredAny>,
#[serde(default)]
deleted: Vec<DeletedLite>,
#[serde(default)]
workspace: Option<WorkspaceLite>,
#[serde(default)]
is_hidden: bool,
}
fn chat_schema_v1() -> u32 {
1
}
#[derive(Deserialize)]
struct MessageLite {
#[serde(default)]
id: Option<Uuid>,
#[serde(default)]
text: Length,
#[serde(default)]
thoughts: Length,
#[serde(default)]
role: RoleLite,
#[serde(default)]
new_bubble: bool,
#[serde(default)]
timestamp: Option<DateTime<Utc>>,
#[serde(default)]
images: Vec<IgnoredAny>,
#[serde(default)]
tool_calls: Vec<ToolCallLite>,
}
#[derive(Default, Clone, Copy)]
struct Length(usize);
impl<'de> Deserialize<'de> for Length {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct Measure;
impl serde::de::Visitor<'_> for Measure {
type Value = Length;
fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.write_str("a string or null")
}
fn visit_str<E>(self, text: &str) -> Result<Length, E> {
Ok(Length(text.chars().count()))
}
fn visit_unit<E>(self) -> Result<Length, E> {
Ok(Length(0))
}
}
deserializer.deserialize_any(Measure)
}
}
fn index(messages: &[MessageLite]) -> Vec<String> {
messages
.iter()
.map(|m| {
let id = m.id.unwrap_or_default();
format!("{id}:{}", m.text.0 + m.thoughts.0)
})
.collect()
}
#[derive(Deserialize, Clone, Copy, Default)]
#[serde(rename_all = "lowercase")]
enum RoleLite {
System,
User,
Assistant,
Tool,
#[default]
#[serde(other)]
Other,
}
impl RoleLite {
fn role(self) -> MessageRole {
match self {
Self::User => MessageRole::User,
Self::Assistant => MessageRole::Assistant,
Self::Tool => MessageRole::Tool,
Self::System | Self::Other => MessageRole::System,
}
}
}
#[derive(Deserialize)]
struct ToolCallLite {
#[serde(default)]
subagent: Option<RunLite>,
}
#[derive(Deserialize)]
struct RunLite {
#[serde(default)]
messages: Vec<MessageLite>,
}
#[derive(Deserialize)]
struct DeletedLite {
#[serde(default)]
messages: Vec<MessageLite>,
}
#[derive(Deserialize)]
struct WorkspaceLite {
#[serde(default)]
root: String,
}
fn bubbles(messages: &[MessageLite]) -> usize {
visible_row_count(messages.iter().map(|m| (m.role.role(), m.new_bubble)))
}
fn runs(messages: &[MessageLite]) -> (usize, usize) {
messages
.iter()
.flat_map(|m| &m.tool_calls)
.filter_map(|call| call.subagent.as_ref())
.fold((0, 0), |(count, held), run| {
(count + 1, held + bubbles(&run.messages))
})
}
#[derive(Default)]
struct Tally {
totals: ChatTotals,
project_roots: BTreeSet<String>,
rows: Vec<ChatRow>,
bytes: u64,
}
impl Tally {
fn add(&mut self, name: &str, id: Uuid, content: &[u8]) {
self.bytes += content.len() as u64;
match serde_json::from_slice::<ChatLite>(content) {
Ok(chat) => self.count(id, chat),
Err(_) => self.unreadable(name),
}
}
fn unreadable(&mut self, name: &str) {
self.totals.unreadable.push(name.to_string());
}
fn count(&mut self, id: Uuid, chat: ChatLite) {
let t = &mut self.totals;
let messages = bubbles(&chat.messages);
let deleted_messages: usize = chat.deleted.iter().map(|d| bubbles(&d.messages)).sum();
let last_message_at = chat.messages.iter().filter_map(|m| m.timestamp).max();
let (run_count, run_messages) = runs(&chat.messages);
t.total += 1;
t.messages += messages;
t.message_rows += chat.messages.len();
if chat.is_hidden {
t.deleted += 1;
t.messages_in_deleted_chats += messages;
}
t.deleted_messages += deleted_messages;
t.deleted_exchanges += chat.deleted.len();
t.attachments += chat.attachments.len();
t.stored_files += chat.files.len();
t.images += chat.messages.iter().map(|m| m.images.len()).sum::<usize>();
t.subagent_runs += run_count;
t.subagent_messages += run_messages;
t.last_message_at = t.last_message_at.max(last_message_at);
t.last_change_at = t.last_change_at.max(chat.modified_at);
t.newest_schema = t.newest_schema.max(chat.v);
if let Some(workspace) = &chat.workspace {
t.chats_with_project += 1;
self.project_roots.insert(workspace.root.clone());
}
self.rows.push(ChatRow {
id,
profile_id: chat.profile_id,
title: chat.title,
deleted: chat.is_hidden,
created_at: chat.created_at,
modified_at: chat.modified_at,
messages,
message_rows: chat.messages.len(),
deleted_messages,
last_message_at,
attachments: chat.attachments.len(),
stored_files: chat.files.len(),
message_index: index(&chat.messages),
deleted_index: chat
.deleted
.iter()
.flat_map(|d| index(&d.messages))
.collect(),
});
}
fn finish(mut self) -> (ChatTotals, Vec<ChatRow>, u64) {
self.totals.projects = self.project_roots.len();
self.totals.unreadable.sort();
self.rows.sort_by_key(|row| row.id);
(self.totals, self.rows, self.bytes)
}
}
fn chat_id(file_name: &str) -> Option<Uuid> {
Uuid::parse_str(file_name.strip_suffix(".json")?).ok()
}
fn profile_totals(content: &[u8]) -> ProfileTotals {
let Ok(value) = serde_json::from_slice::<serde_json::Value>(content) else {
return ProfileTotals::default();
};
let list = value
.as_array()
.or_else(|| value.get("profiles").and_then(|p| p.as_array()));
let Some(list) = list else {
return ProfileTotals::default();
};
let hidden =
|p: &&serde_json::Value| p.get("is_hidden").and_then(|h| h.as_bool()) == Some(true);
ProfileTotals {
total: list.len(),
deleted: list.iter().filter(hidden).count(),
}
}
fn assemble(
source: Source,
profiles: ProfileTotals,
tally: Tally,
database: DatabaseStats,
database_bytes: u64,
) -> DataStats {
let (chats, chat_list, chats_bytes) = tally.finish();
DataStats {
format: SNAPSHOT_FORMAT,
app_version: env!("CARGO_PKG_VERSION").to_string(),
taken_at: Utc::now(),
fingerprint: DataStats::fingerprint(&chat_list, &database),
source,
profiles,
chats,
database,
sizes: Sizes {
chats_bytes,
database_bytes,
},
chat_list,
}
}
pub fn collect_root(paths: &Paths, loc: &Locale) -> Result<DataStats> {
let mut tally = Tally::default();
let dir = paths.chats_dir();
if dir.is_dir() {
let listing = fs::read_dir(&dir).with_context(|| {
loc.tf(
"cli.stats.ctx.list_chats",
&[("path", &dir.display().to_string())],
)
})?;
for entry in listing.flatten() {
let name = entry.file_name().to_string_lossy().into_owned();
let Some(id) = chat_id(&name) else { continue };
match fs::read(entry.path()) {
Ok(content) => tally.add(&name, id, &content),
Err(_) => tally.unreadable(&name),
}
}
}
let profiles = fs::read(paths.profiles_file())
.map(|content| profile_totals(&content))
.unwrap_or_default();
let db_path = paths.data_db();
let (database, database_bytes) = if db_path.is_file() {
let size = fs::metadata(&db_path).map(|m| m.len()).unwrap_or(0);
(database_stats(db::stats_of_file(&db_path)), size)
} else {
(DatabaseStats::Missing, 0)
};
let source = Source::DataRoot {
path: paths.root().display().to_string(),
};
Ok(assemble(source, profiles, tally, database, database_bytes))
}
pub fn collect_archive(archive: &Path, password: Option<&str>, loc: &Locale) -> Result<DataStats> {
let mut reader = ArchiveReader::open(archive, password, loc)?;
let entries = reader.entries();
let is_backup = entries
.iter()
.any(|e| BACKUP_MARKERS.contains(&e.name.as_str()) || e.name.starts_with(CHATS_PREFIX));
if !is_backup {
bail!(
"{}",
loc.tf(
"cli.stats.err.not_a_backup",
&[("path", &archive.display().to_string())]
)
);
}
let mut tally = Tally::default();
for entry in &entries {
let Some(name) = entry.name.strip_prefix(CHATS_PREFIX) else {
continue;
};
let Some(id) = chat_id(name) else { continue };
match reader.with_entry(&entry.name, |stream, _| read_all(stream)) {
Ok(Some(content)) => tally.add(name, id, &content),
_ => tally.unreadable(name),
}
}
let profiles = reader
.with_entry(PROFILES_ENTRY, |stream, _| read_all(stream))
.ok()
.flatten()
.map(|content| profile_totals(&content))
.unwrap_or_default();
let database_bytes = entries.iter().find(|e| e.name == DB_ENTRY).map(|e| e.size);
let database = match database_bytes {
None => DatabaseStats::Missing,
Some(_) => database_stats(
reader
.with_entry(DB_ENTRY, |stream, size| {
db::stats_of_image(stream, usize::try_from(size)?)
})
.and_then(|found| found.context("the archive lists data.db and does not hold it")),
),
};
let manifest = backup::read_manifest(archive).ok().flatten();
let source = Source::Archive {
path: archive.display().to_string(),
newer_than_this_app: manifest
.as_ref()
.is_some_and(BackupManifest::is_newer_than_current),
manifest,
};
Ok(assemble(
source,
profiles,
tally,
database,
database_bytes.unwrap_or(0),
))
}
fn read_all(stream: &mut dyn Read) -> Result<Vec<u8>> {
let mut content = Vec::new();
stream.read_to_end(&mut content)?;
Ok(content)
}
fn database_stats(read: Result<DbStats>) -> DatabaseStats {
match read {
Ok(stats) => DatabaseStats::Ok(stats),
Err(err) => DatabaseStats::Unreadable {
error: format!("{err:#}"),
},
}
}
pub fn render_json(stats: &DataStats) -> String {
serde_json::to_string_pretty(stats).unwrap_or_default()
}
fn utc(at: Option<DateTime<Utc>>, loc: &Locale) -> String {
match at {
Some(at) => at.format("%Y-%m-%d %H:%M:%S UTC").to_string(),
None => loc.t("cli.stats.none").to_string(),
}
}
pub fn render_text(stats: &DataStats, loc: &Locale) -> String {
let mut out = vec![loc.tf(
"cli.stats.title",
&[("version", stats.app_version.as_str())],
)];
out.extend(source_lines(&stats.source, loc));
out.push(String::new());
if stats.is_empty() {
out.push(loc.t("cli.stats.empty").to_string());
return out.join("\n");
}
let rows = table_rows(stats, loc);
let width = rows
.iter()
.map(|(label, _)| label.width())
.max()
.unwrap_or(0);
out.extend(rows.into_iter().map(|(label, value)| {
let pad = " ".repeat(width - label.width());
format!("{label}:{pad} {value}")
}));
let c = &stats.chats;
if !c.unreadable.is_empty() {
out.push(String::new());
out.push(loc.tf(
"cli.stats.unreadable",
&[
("count", &c.unreadable.len().to_string()),
("names", &c.unreadable.join(", ")),
],
));
}
if c.newest_schema > CHAT_SCHEMA {
out.push(String::new());
out.push(loc.t("cli.stats.newer_data").to_string());
}
out.join("\n")
}
fn source_lines(source: &Source, loc: &Locale) -> Vec<String> {
match source {
Source::DataRoot { path } => vec![loc.tf("cli.stats.source.root", &[("path", path)])],
Source::Archive {
path,
manifest,
newer_than_this_app,
} => {
let mut lines = vec![loc.tf("cli.stats.source.archive", &[("path", path)])];
if let Some(m) = manifest {
let created = DateTime::parse_from_rfc3339(&m.created_at)
.map(|at| utc(Some(at.with_timezone(&Utc)), loc))
.unwrap_or_else(|_| m.created_at.clone());
lines.push(loc.tf(
"cli.stats.source.archive_made",
&[("created", &created), ("version", &m.app_version)],
));
}
if *newer_than_this_app {
lines.push(loc.t("cli.stats.newer_data").to_string());
}
lines
}
}
}
fn table_rows(stats: &DataStats, loc: &Locale) -> Vec<(String, String)> {
let c = &stats.chats;
let n = |value: usize| value.to_string();
let mut rows = vec![
("cli.stats.label.last_message", utc(c.last_message_at, loc)),
("cli.stats.label.last_change", utc(c.last_change_at, loc)),
("cli.stats.label.fingerprint", stats.fingerprint.clone()),
(
"cli.stats.label.profiles",
with_deleted(loc, stats.profiles.total, stats.profiles.deleted),
),
(
"cli.stats.label.chats",
with_deleted(loc, c.total, c.deleted),
),
(
"cli.stats.label.messages",
loc.tf(
"cli.stats.val.messages",
&[
("total", &n(c.messages)),
("deleted", &n(c.messages_in_deleted_chats)),
("rows", &n(c.message_rows)),
],
),
),
(
"cli.stats.label.deleted_messages",
loc.tf(
"cli.stats.val.deleted_messages",
&[
("total", &n(c.deleted_messages)),
("exchanges", &n(c.deleted_exchanges)),
],
),
),
("cli.stats.label.attachments", n(c.attachments)),
("cli.stats.label.stored_files", n(c.stored_files)),
("cli.stats.label.images", n(c.images)),
(
"cli.stats.label.projects",
loc.tf(
"cli.stats.val.projects",
&[
("total", &n(c.projects)),
("chats", &n(c.chats_with_project)),
],
),
),
(
"cli.stats.label.subagents",
loc.tf(
"cli.stats.val.subagents",
&[
("total", &n(c.subagent_runs)),
("messages", &n(c.subagent_messages)),
],
),
),
];
rows.extend(database_rows(&stats.database, loc));
rows.push((
"cli.stats.label.size",
loc.tf(
"cli.stats.val.size",
&[
("chats", &size(stats.sizes.chats_bytes)),
("database", &size(stats.sizes.database_bytes)),
],
),
));
rows.into_iter()
.map(|(label, value)| (loc.t(label).to_string(), value))
.collect()
}
fn database_rows(database: &DatabaseStats, loc: &Locale) -> Vec<(&'static str, String)> {
let n = |value: u64| value.to_string();
match database {
DatabaseStats::Missing => vec![(
"cli.stats.label.database",
loc.t("cli.stats.val.database_missing").to_string(),
)],
DatabaseStats::Unreadable { error } => vec![(
"cli.stats.label.database",
loc.tf("cli.stats.val.database_unreadable", &[("err", error)]),
)],
DatabaseStats::Ok(d) => vec![
(
"cli.stats.label.notes",
loc.tf(
"cli.stats.val.notes",
&[
("total", &n(d.notes)),
("superseded", &n(d.notes_superseded)),
("links", &n(d.note_links)),
],
),
),
("cli.stats.label.last_note", utc(d.last_note_change, loc)),
(
"cli.stats.label.rag",
loc.tf(
"cli.stats.val.rag",
&[("sources", &n(d.rag_sources)), ("chunks", &n(d.rag_chunks))],
),
),
("cli.stats.label.self_models", n(d.self_models)),
],
}
}
fn with_deleted(loc: &Locale, total: usize, deleted: usize) -> String {
loc.tf(
"cli.stats.val.with_deleted",
&[
("total", &total.to_string()),
("deleted", &deleted.to_string()),
],
)
}
fn size(bytes: u64) -> String {
format_bytes(usize::try_from(bytes).unwrap_or(usize::MAX))
}
mod compare;
mod snapshot;
pub use compare::{compare, render_comparison_json, render_comparison_text};
pub use snapshot::{OtherCopy, other_copy, read_snapshot};
#[cfg(test)]
mod tests;