1use std::{
23 fs::{self, File, OpenOptions},
24 io::{self, BufRead as _, BufReader, Read as _, Write as _},
25 path::{Path, PathBuf},
26 time::Duration,
27};
28
29use serde::{Deserialize, Serialize};
30
31pub const MAX_TEXT_BYTES: usize = 256 * 1024;
33const CUT_NOTE: &str = "\n[cut: longer than the chat log keeps]";
34const MAX_LINE_BYTES: usize = 2 * MAX_TEXT_BYTES;
36const KEPT_DIR: &str = "files";
38const MAX_CLOCK_SKEW_MS: u64 = 5 * 60 * 1000;
42const NEWEST_WEEKS: usize = 3;
46
47#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
49#[serde(tag = "type", rename_all = "snake_case")]
50pub enum Record {
51 Message(Entry),
53 End {
55 at: u64,
56 #[serde(default)]
57 local: String,
58 },
59 #[serde(other)]
61 Unknown,
62}
63
64#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
66pub struct Entry {
67 pub at: u64,
69 #[serde(default)]
71 pub local: String,
72 pub role: Role,
73 #[serde(default, skip_serializing_if = "String::is_empty")]
74 pub text: String,
75 #[serde(default, skip_serializing_if = "String::is_empty")]
77 pub quote: String,
78 #[serde(default, skip_serializing_if = "Vec::is_empty")]
79 pub files: Vec<FileRef>,
80 #[serde(default, skip_serializing_if = "Vec::is_empty")]
82 pub notes: Vec<String>,
83 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
85 pub report: bool,
86}
87
88#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
90#[serde(rename_all = "snake_case")]
91pub enum Role {
92 #[default]
94 Owner,
95 Scv,
97 System,
99 #[serde(other)]
101 Unknown,
102}
103
104#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
106pub struct FileRef {
107 pub kind: String,
109 pub name: String,
110 #[serde(default, skip_serializing_if = "String::is_empty")]
113 pub path: String,
114 #[serde(default, skip_serializing_if = "String::is_empty")]
115 pub mime: String,
116 #[serde(default, skip_serializing_if = "is_zero")]
117 pub size: u64,
118 #[serde(default, skip_serializing_if = "String::is_empty")]
120 pub transcript: String,
121}
122
123#[allow(
124 clippy::trivially_copy_pass_by_ref,
125 reason = "serde's skip_serializing_if passes a reference"
126)]
127fn is_zero(value: &u64) -> bool {
128 *value == 0
129}
130
131#[derive(Debug, Clone, Copy, PartialEq, Eq)]
133pub struct LocalTime {
134 year: i64,
135 month: u32,
136 day: u32,
137 hour: u32,
138 minute: u32,
139 second: u32,
140 offset: i32,
142}
143
144impl LocalTime {
145 pub fn at(unix: i64, offset: i32) -> Self {
147 let local = unix.saturating_add(i64::from(offset));
148 let days = local.div_euclid(86_400);
149 let seconds = local.rem_euclid(86_400);
150 let (year, month, day) = civil_from_days(days);
151 Self {
152 year,
153 month,
154 day,
155 hour: (seconds / 3600) as u32,
156 minute: (seconds / 60 % 60) as u32,
157 second: (seconds % 60) as u32,
158 offset,
159 }
160 }
161
162 pub fn year(&self) -> i64 {
163 self.year
164 }
165
166 pub fn stamp(&self) -> String {
168 let sign = if self.offset < 0 { '-' } else { '+' };
169 let offset = self.offset.unsigned_abs();
170 format!(
171 "{} {:02}:{:02}:{:02} {sign}{:02}:{:02}",
172 self.date(),
173 self.hour,
174 self.minute,
175 self.second,
176 offset / 3600,
177 offset / 60 % 60
178 )
179 }
180
181 pub fn date(&self) -> String {
183 format_date(self.year, self.month, self.day)
184 }
185
186 fn week(&self) -> String {
188 let days = days_from_civil(self.year, self.month, self.day);
189 let monday = days - (days + 3).rem_euclid(7);
191 let (y1, m1, d1) = civil_from_days(monday);
192 let (y2, m2, d2) = civil_from_days(monday + 6);
193 format!("{}_{}", format_date(y1, m1, d1), format_date(y2, m2, d2))
194 }
195
196 fn file_stem(&self) -> String {
198 format!(
199 "{}T{:02}-{:02}-{:02}",
200 self.date(),
201 self.hour,
202 self.minute,
203 self.second
204 )
205 }
206}
207
208fn format_date(year: i64, month: u32, day: u32) -> String {
209 format!("{year:04}-{month:02}-{day:02}")
210}
211
212fn days_from_civil(year: i64, month: u32, day: u32) -> i64 {
215 let year = if month <= 2 { year - 1 } else { year };
216 let era = year.div_euclid(400);
217 let year_of_era = year - era * 400;
218 let month_from_march = i64::from((month + 9) % 12);
219 let day_of_year = (153 * month_from_march + 2) / 5 + i64::from(day) - 1;
220 let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year;
221 era * 146_097 + day_of_era - 719_468
222}
223
224fn civil_from_days(days: i64) -> (i64, u32, u32) {
226 let days = days + 719_468;
227 let era = days.div_euclid(146_097);
228 let day_of_era = days - era * 146_097;
229 let year_of_era =
230 (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
231 let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
232 let month_from_march = (5 * day_of_year + 2) / 153;
233 let day = (day_of_year - (153 * month_from_march + 2) / 5 + 1) as u32;
234 let month = if month_from_march < 10 {
235 month_from_march + 3
236 } else {
237 month_from_march - 9
238 } as u32;
239 let year = year_of_era + era * 400 + i64::from(month <= 2);
240 (year, month, day)
241}
242
243pub fn valid_part(part: &str) -> bool {
246 !part.is_empty()
247 && part.len() <= 64
248 && part
249 .bytes()
250 .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_')
251}
252
253pub fn conversation_path(channel: &str, account: &str, conversation: &str) -> Option<PathBuf> {
257 [channel, account, conversation]
258 .iter()
259 .all(|part| valid_part(part))
260 .then(|| Path::new(channel).join(account).join(conversation))
261}
262
263pub fn kept_dir(archive: &Path, conversation: &Path) -> PathBuf {
267 archive.join(conversation).join(KEPT_DIR)
268}
269
270pub struct Log {
272 dir: PathBuf,
273 gap: Duration,
274 newest: Option<Newest>,
276 scanned: bool,
277}
278
279#[derive(Debug, Clone)]
280struct Newest {
281 path: PathBuf,
282 last_at: u64,
283 ended: bool,
284}
285
286impl Log {
287 pub fn new(dir: PathBuf, gap: Duration) -> Self {
289 Self {
290 dir,
291 gap,
292 newest: None,
293 scanned: false,
294 }
295 }
296
297 pub fn dir(&self) -> &Path {
298 &self.dir
299 }
300
301 pub fn append(&mut self, mut entry: Entry, local: &LocalTime) -> io::Result<()> {
305 entry.local = local.stamp();
306 cut(&mut entry.text);
307 cut(&mut entry.quote);
308 let at = entry.at;
309 let path = match self.newest(at)? {
310 Some(newest) if is_open(&newest, self.gap, at) => newest.path,
311 _ => self.start(local)?,
312 };
313 write_record(&path, &Record::Message(entry))?;
314 self.newest = Some(Newest {
315 path,
316 last_at: at,
317 ended: false,
318 });
319 Ok(())
320 }
321
322 pub fn end(&mut self, at: u64, local: &LocalTime) -> io::Result<bool> {
325 let Some(newest) = self.newest(at)? else {
326 return Ok(false);
327 };
328 if !is_open(&newest, self.gap, at) {
329 return Ok(false);
330 }
331 write_record(
332 &newest.path,
333 &Record::End {
334 at,
335 local: local.stamp(),
336 },
337 )?;
338 self.newest = Some(Newest {
339 ended: true,
340 ..newest
341 });
342 Ok(true)
343 }
344
345 fn newest(&mut self, now: u64) -> io::Result<Option<Newest>> {
346 if self.scanned {
347 return Ok(self.newest.clone());
348 }
349 let newest = match newest_episode(&self.dir, now)? {
350 Some(path) => {
351 let (last_at, ended) = tail(&path)?;
352 Some(Newest {
353 path,
354 last_at,
355 ended,
356 })
357 }
358 None => None,
359 };
360 self.newest.clone_from(&newest);
361 self.scanned = true;
362 Ok(newest)
363 }
364
365 fn start(&self, local: &LocalTime) -> io::Result<PathBuf> {
367 let dir = self
368 .dir
369 .join(format!("{:04}", local.year))
370 .join(local.week());
371 create_private_dir(&dir)?;
372 let stem = local.file_stem();
373 for attempt in 1..1000 {
374 let name = if attempt == 1 {
375 format!("{stem}.jsonl")
376 } else {
377 format!("{stem}-{attempt}.jsonl")
378 };
379 let path = dir.join(name);
380 let mut options = OpenOptions::new();
381 options.write(true).create_new(true);
382 #[cfg(unix)]
383 std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
384 match options.open(&path) {
385 Ok(_) => return Ok(path),
386 Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}
387 Err(error) => return Err(error),
388 }
389 }
390 Err(io::Error::other(
391 "too many episodes start in the same second",
392 ))
393 }
394}
395
396fn is_open(newest: &Newest, gap: Duration, now: u64) -> bool {
397 !newest.ended
398 && newest.last_at <= now.saturating_add(MAX_CLOCK_SKEW_MS)
399 && u128::from(now.saturating_sub(newest.last_at)) < gap.as_millis()
400}
401
402fn cut(text: &mut String) {
403 if text.len() > MAX_TEXT_BYTES {
404 let kept = crate::text::utf8_prefix(text, MAX_TEXT_BYTES - CUT_NOTE.len()).len();
405 text.truncate(kept);
406 text.push_str(CUT_NOTE);
407 }
408}
409
410fn create_private_dir(dir: &Path) -> io::Result<()> {
411 let mut builder = fs::DirBuilder::new();
412 builder.recursive(true);
413 #[cfg(unix)]
414 std::os::unix::fs::DirBuilderExt::mode(&mut builder, 0o700);
415 builder.create(dir)
416}
417
418fn write_record(path: &Path, record: &Record) -> io::Result<()> {
419 let mut line = serde_json::to_vec(record).map_err(io::Error::other)?;
420 line.push(b'\n');
421 let mut options = OpenOptions::new();
422 options.append(true).create(true);
423 #[cfg(unix)]
424 std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
425 options.open(path)?.write_all(&line)
427}
428
429fn records(path: &Path) -> io::Result<Vec<Record>> {
431 let mut reader = BufReader::new(File::open(path)?);
432 let mut records = Vec::new();
433 let mut line = Vec::new();
434 loop {
435 line.clear();
436 let read = (&mut reader)
437 .take(MAX_LINE_BYTES as u64 + 1)
438 .read_until(b'\n', &mut line)?;
439 if read == 0 {
440 return Ok(records);
441 }
442 if line.last() != Some(&b'\n') && read > MAX_LINE_BYTES {
443 let mut rest = Vec::new();
445 reader.read_until(b'\n', &mut rest)?;
446 continue;
447 }
448 if let Ok(record) = serde_json::from_slice(&line) {
449 records.push(record);
450 }
451 }
452}
453
454fn tail(path: &Path) -> io::Result<(u64, bool)> {
456 let mut last_at = 0;
457 let mut ended = false;
458 for record in records(path)? {
459 match record {
460 Record::Message(entry) => {
461 last_at = last_at.max(entry.at);
462 ended = false;
463 }
464 Record::End { .. } => ended = true,
465 Record::Unknown => {}
466 }
467 }
468 Ok((last_at, ended))
469}
470
471fn first_at(path: &Path) -> io::Result<u64> {
474 let mut reader = BufReader::new(File::open(path)?);
475 let mut line = Vec::new();
476 (&mut reader)
477 .take(MAX_LINE_BYTES as u64)
478 .read_until(b'\n', &mut line)?;
479 Ok(match serde_json::from_slice(&line) {
480 Ok(Record::Message(entry)) => entry.at,
481 Ok(Record::End { at, .. }) => at,
482 _ => 0,
483 })
484}
485
486fn names(dir: &Path, keep: impl Fn(&str) -> bool) -> io::Result<Vec<String>> {
489 let entries = match fs::read_dir(dir) {
490 Ok(entries) => entries,
491 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
492 Err(error) => return Err(error),
493 };
494 let mut names = Vec::new();
495 for entry in entries {
496 let entry = entry?;
497 if let Some(name) = entry.file_name().to_str()
498 && keep(name)
499 {
500 names.push(name.to_owned());
501 }
502 }
503 names.sort();
504 Ok(names)
505}
506
507fn is_year(name: &str) -> bool {
508 name.len() == 4 && name.bytes().all(|byte| byte.is_ascii_digit())
509}
510
511fn is_week(name: &str) -> bool {
512 name.len() == 21 && name.as_bytes()[10] == b'_' && valid_part(name)
513}
514
515fn is_episode_file(name: &str) -> bool {
516 name.strip_suffix(".jsonl").is_some_and(valid_part)
517}
518
519fn all_episodes(dir: &Path) -> io::Result<Vec<(String, PathBuf)>> {
522 let mut episodes = Vec::new();
523 for year in names(dir, is_year)? {
524 for week in names(&dir.join(&year), is_week)? {
525 let week_dir = dir.join(&year).join(&week);
526 for file in names(&week_dir, is_episode_file)? {
527 let stem = file.trim_end_matches(".jsonl");
528 episodes.push((format!("{year}/{week}/{stem}"), week_dir.join(&file)));
529 }
530 }
531 }
532 episodes.sort_by(|a, b| b.0.cmp(&a.0));
533 Ok(episodes)
534}
535
536fn newest_episode(dir: &Path, now: u64) -> io::Result<Option<PathBuf>> {
540 let mut weeks = Vec::new();
541 for year in names(dir, is_year)?.into_iter().rev().take(2) {
542 for week in names(&dir.join(&year), is_week)? {
543 weeks.push((week, year.clone()));
544 }
545 }
546 weeks.sort();
547 let mut newest: Option<((bool, u64), PathBuf)> = None;
549 for (week, year) in weeks.into_iter().rev().take(NEWEST_WEEKS) {
550 let week_dir = dir.join(year).join(week);
551 for file in names(&week_dir, is_episode_file)? {
552 let path = week_dir.join(file);
553 let started = first_at(&path)?;
554 let rank = (started <= now.saturating_add(MAX_CLOCK_SKEW_MS), started);
555 if newest.as_ref().is_none_or(|(best, _)| rank >= *best) {
556 newest = Some((rank, path));
557 }
558 }
559 }
560 Ok(newest.map(|(_, path)| path))
561}
562
563fn episode_path(dir: &Path, id: &str) -> Option<PathBuf> {
565 let mut parts = id.split('/');
566 let (Some(year), Some(week), Some(stem), None) =
567 (parts.next(), parts.next(), parts.next(), parts.next())
568 else {
569 return None;
570 };
571 (is_year(year) && is_week(week) && valid_part(stem))
572 .then(|| dir.join(year).join(week).join(format!("{stem}.jsonl")))
573}
574
575#[derive(Debug, Clone, PartialEq)]
577pub struct Episode {
578 pub id: String,
579 pub messages: Vec<Entry>,
580}
581
582pub fn open_episode(dir: &Path, gap: Duration, now: u64) -> io::Result<Option<Episode>> {
585 let Some(path) = newest_episode(dir, now)? else {
586 return Ok(None);
587 };
588 let mut messages = Vec::new();
589 let mut ended = false;
590 for record in records(&path)? {
591 match record {
592 Record::Message(entry) => {
593 ended = false;
594 messages.push(entry);
595 }
596 Record::End { .. } => ended = true,
597 Record::Unknown => {}
598 }
599 }
600 let last_at = messages.iter().map(|entry| entry.at).max().unwrap_or(0);
601 let newest = Newest {
602 path: path.clone(),
603 last_at,
604 ended,
605 };
606 if messages.is_empty() || !is_open(&newest, gap, now) {
607 return Ok(None);
608 }
609 Ok(Some(Episode {
610 id: episode_id(dir, &path),
611 messages,
612 }))
613}
614
615fn episode_id(dir: &Path, path: &Path) -> String {
616 let relative = path.strip_prefix(dir).unwrap_or(path);
617 relative
618 .with_extension("")
619 .components()
620 .map(|part| part.as_os_str().to_string_lossy())
621 .collect::<Vec<_>>()
622 .join("/")
623}
624
625#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
627pub struct Summary {
628 pub id: String,
629 pub started: String,
631 pub last: String,
632 pub messages: usize,
633 pub opening: String,
635 #[serde(skip_serializing_if = "std::ops::Not::not")]
637 pub ended: bool,
638}
639
640const EXCERPT_CHARS: usize = 240;
642
643pub fn episodes(
647 dir: &Path,
648 before: Option<&str>,
649 after: Option<&str>,
650 limit: usize,
651) -> io::Result<(Vec<Summary>, bool)> {
652 let mut summaries = Vec::new();
653 let mut more = false;
654 for (id, path) in all_episodes(dir)? {
655 let day = id.rsplit('/').next().unwrap_or("").get(..10).unwrap_or("");
656 if before.is_some_and(|before| day >= before) || after.is_some_and(|after| day < after) {
657 continue;
658 }
659 if summaries.len() == limit {
660 more = true;
661 break;
662 }
663 let mut summary = Summary {
664 id,
665 started: String::new(),
666 last: String::new(),
667 messages: 0,
668 opening: String::new(),
669 ended: false,
670 };
671 for record in records(&path)? {
672 match record {
673 Record::Message(entry) => {
674 if summary.started.is_empty() {
675 summary.started.clone_from(&entry.local);
676 }
677 if summary.opening.is_empty() && entry.role == Role::Owner {
678 summary.opening = excerpt(&entry.text, 0);
679 }
680 summary.last = entry.local;
681 summary.messages += 1;
682 summary.ended = false;
683 }
684 Record::End { .. } => summary.ended = true,
685 Record::Unknown => {}
686 }
687 }
688 summaries.push(summary);
689 }
690 Ok((summaries, more))
691}
692
693pub fn read_episode(
696 dir: &Path,
697 id: &str,
698 offset: usize,
699 limit: usize,
700) -> io::Result<Option<(Vec<Entry>, usize)>> {
701 let Some(path) = episode_path(dir, id) else {
702 return Ok(None);
703 };
704 let messages: Vec<Entry> = match records(&path) {
705 Ok(records) => records
706 .into_iter()
707 .filter_map(|record| match record {
708 Record::Message(entry) => Some(entry),
709 _ => None,
710 })
711 .collect(),
712 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
713 Err(error) => return Err(error),
714 };
715 let total = messages.len();
716 Ok(Some((
717 messages.into_iter().skip(offset).take(limit).collect(),
718 total,
719 )))
720}
721
722#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
724pub struct Hit {
725 pub episode: String,
726 pub index: usize,
728 pub local: String,
729 pub role: Role,
730 pub excerpt: String,
731}
732
733pub fn search(
738 dir: &Path,
739 query: &str,
740 limit: usize,
741 max_bytes: u64,
742) -> io::Result<(Vec<Hit>, bool)> {
743 let terms: Vec<String> = query.split_whitespace().map(str::to_lowercase).collect();
744 let mut hits = Vec::new();
745 if terms.is_empty() {
746 return Ok((hits, false));
747 }
748 let mut read = 0u64;
749 for (id, path) in all_episodes(dir)? {
750 if read >= max_bytes {
751 return Ok((hits, true));
752 }
753 read = read.saturating_add(fs::metadata(&path).map_or(0, |metadata| metadata.len()));
754 let messages: Vec<Entry> = records(&path)?
755 .into_iter()
756 .filter_map(|record| match record {
757 Record::Message(entry) => Some(entry),
758 _ => None,
759 })
760 .collect();
761 for (index, entry) in messages.iter().enumerate().rev() {
762 let mut haystack = entry.text.to_lowercase();
763 let files = entry
764 .files
765 .iter()
766 .flat_map(|file| [&file.name, &file.transcript]);
767 for extra in std::iter::once(&entry.quote).chain(files) {
768 haystack.push('\n');
769 haystack.push_str(&extra.to_lowercase());
770 }
771 if !terms.iter().all(|term| haystack.contains(term.as_str())) {
772 continue;
773 }
774 let lower = entry.text.to_lowercase();
775 let from = lower
776 .find(terms[0].as_str())
777 .map_or(0, |byte| lower[..byte].chars().count());
778 hits.push(Hit {
779 episode: id.clone(),
780 index,
781 local: entry.local.clone(),
782 role: entry.role,
783 excerpt: excerpt(&entry.text, from),
784 });
785 if hits.len() == limit {
786 return Ok((hits, true));
787 }
788 }
789 }
790 Ok((hits, false))
791}
792
793fn excerpt(text: &str, from: usize) -> String {
795 let start = from.saturating_sub(EXCERPT_CHARS / 4);
796 let mut excerpt: String = text.chars().skip(start).take(EXCERPT_CHARS).collect();
797 if start > 0 {
798 excerpt.insert(0, '…');
799 }
800 if text.chars().count() > start + EXCERPT_CHARS {
801 excerpt.push('…');
802 }
803 excerpt
804}
805
806pub fn prune_years(account_dir: &Path, oldest_year: i64) -> usize {
809 let mut removed = 0;
810 let Ok(conversations) = names(account_dir, valid_part) else {
811 return 0;
812 };
813 for conversation in conversations {
814 let dir = account_dir.join(conversation);
815 let Ok(years) = names(&dir, is_year) else {
816 continue;
817 };
818 for year in years {
819 if year.parse::<i64>().is_ok_and(|year| year < oldest_year)
820 && fs::remove_dir_all(dir.join(&year)).is_ok()
821 {
822 removed += 1;
823 }
824 }
825 }
826 removed
827}
828
829#[cfg(test)]
830mod tests;