1use std::collections::{BTreeMap, HashMap};
5use std::io;
6use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
7
8use rig_core::conversation::{Change, ChangeRecord, Conversation, Lineage, MessageId};
9use serde_json::Value;
10
11use crate::blobs::{PutBlob, put_blob, read_blob, store_change};
12use crate::fork::turn_cut;
13use crate::format::{Header, LogLines, Meta, SESSION, write_change, write_meta};
14use crate::session::{
15 End, Fold, Folded, ends, fold_all, fold_lines, log_lines, read_header, read_logs, repairs,
16 retired, session_of,
17};
18use crate::store::now;
19use crate::{LogWriteError, Repair, SavedAgent, Session, SessionError, Stamp, Store};
20use rig_core::now_ms;
21use rig_core::tool::content_hash;
22
23#[derive(Default)]
25pub(crate) struct Log {
26 depth: usize,
28 next_seq: u64,
29 started: bool,
31 pending: Vec<u8>,
32 state: BTreeMap<String, Value>,
35 lineage: Lineage,
37 changes: u64,
39 len: u64,
42}
43
44#[derive(Default)]
46pub(crate) struct Book {
47 next_n: u64,
49 live: bool,
51 failure: Option<String>,
54 wrote: bool,
56 logs: HashMap<String, Log>,
57 torn: HashMap<String, u64>,
60}
61
62impl Book {
63 pub fn after(folded: &[Folded]) -> Self {
65 let parents: HashMap<&str, Option<&str>> = folded
66 .iter()
67 .map(|log| (log.name.as_str(), log.header.parent.as_deref()))
68 .collect();
69 let mut book = Self::default();
70 for log in folded {
71 let ancestors = std::iter::successors(log.header.parent.as_deref(), |parent| {
73 parents.get(parent).copied().flatten()
74 });
75 let known = ancestors.take_while(|parent| parents.contains_key(parent));
76 let kept = Log {
77 depth: known.take(folded.len()).count(),
78 next_seq: log.next_seq,
79 started: true,
80 state: log.state.clone(),
81 lineage: log.header.lineage,
82 changes: log.conversation.revision(),
83 len: log.len,
84 ..Log::default()
85 };
86 book.logs.insert(log.name.clone(), kept);
87 book.next_n = book.next_n.max(log.next_n);
88 }
89 book
90 }
91
92 pub fn written(&self) -> impl Iterator<Item = (String, End)> + '_ {
94 let written = self.logs.iter();
95 let written = written.filter(|(_, log)| log.len > 0 && log.pending.is_empty());
96 written.map(|(name, log)| (name.clone(), (log.lineage, log.next_seq)))
97 }
98
99 pub fn set_aside(&mut self, torn: impl IntoIterator<Item = (String, u64)>) {
102 self.torn.extend(torn);
103 }
104
105 pub fn start(&mut self) {
106 self.live = true;
107 }
108
109 fn is_live(&self) -> bool {
110 self.live && self.failure.is_none()
111 }
112
113 pub fn opened(&mut self, name: &str, parent: Option<&str>) -> Option<&mut Log> {
116 if !self.is_live() {
117 return None;
118 }
119 if !self.logs.contains_key(name) {
120 let depth = |parent| self.logs.get(parent).map_or(0, |log: &Log| log.depth + 1);
121 let header = Header::new(name, parent);
122 let mut log = Log {
123 depth: parent.map_or(0, depth),
124 lineage: header.lineage,
125 ..Log::default()
126 };
127 let header = Meta::Header(header);
128 let stamp = Stamp {
129 n: self.next_n,
130 seq: 0,
131 t: now_ms(),
132 };
133 if let Err(failure) = write_meta(&mut log.pending, stamp, &header) {
134 self.failure = Some(failure.to_string());
135 return None;
136 }
137 log.next_seq += 1;
138 self.next_n += 1;
139 self.logs.insert(name.to_owned(), log);
140 }
141 self.logs.get_mut(name)
142 }
143
144 fn meta(&mut self, name: &str, meta: &Meta) -> bool {
146 if self.opened(name, None).is_none() {
147 return false;
148 }
149 let n = self.next_n;
150 let Some(log) = self.logs.get_mut(name) else {
151 return false;
152 };
153 let stamp = Stamp {
154 n,
155 seq: log.next_seq,
156 t: now_ms(),
157 };
158 match write_meta(&mut log.pending, stamp, meta) {
159 Ok(()) => {
160 log.next_seq += 1;
161 self.next_n += 1;
162 true
163 }
164 Err(failure) => {
165 self.failure = Some(failure.to_string());
166 false
167 }
168 }
169 }
170
171 pub fn save(
179 &mut self,
180 put: &mut PutBlob<'_>,
181 name: &str,
182 conversation: &mut Conversation,
183 ) -> Vec<ChangeRecord> {
184 let Some(log) = self.opened(name, None) else {
185 return stamp_all(conversation.take_unsaved());
186 };
187 let (lineage, held) = (log.lineage, log.changes);
188 let continues = conversation.continues(lineage, held);
189 let unsaved = conversation.take_unsaved();
190 let changes = if continues {
191 unsaved
192 } else {
193 vec![ChangeRecord {
194 change: Change::Reset {
195 state: Box::new(conversation.view()),
196 },
197 by: None,
198 at: None,
199 }]
200 };
201 if changes.is_empty() {
202 conversation.continue_log(lineage, held);
203 return changes;
204 }
205 let mut queued = Vec::with_capacity(changes.len());
206 for recorded in stamp_all(changes) {
207 if let Err(failure) = self.queue(put, name, &recorded) {
208 self.failure = Some(failure);
209 break;
210 }
211 queued.push(recorded);
212 }
213 let held = held.saturating_add(queued.len() as u64);
214 if let Some(log) = self.logs.get_mut(name) {
215 log.started = true;
216 log.changes = held;
217 }
218 conversation.continue_log(lineage, held);
219 queued
220 }
221
222 fn queue(
224 &mut self,
225 put: &mut PutBlob<'_>,
226 name: &str,
227 recorded: &ChangeRecord,
228 ) -> Result<(), String> {
229 let stored = store_change(&recorded.change, put)
230 .map_err(|failure| format!("storing an image failed: {failure}"))?;
231 let change = stored.as_ref().unwrap_or(&recorded.change);
232 let n = self.next_n;
233 let log = self.logs.get_mut(name).ok_or("no log")?;
234 let stamp = Stamp {
235 n,
236 seq: log.next_seq,
237 t: recorded.at.unwrap_or_default(),
238 };
239 write_change(&mut log.pending, stamp, recorded.by.as_ref(), change)
240 .map_err(|failure| failure.to_string())?;
241 log.next_seq += 1;
242 self.next_n += 1;
243 Ok(())
244 }
245
246 pub fn save_value(&mut self, name: &str, key: &str, value: Option<Value>, starts: bool) {
250 let value = value.filter(|value| !value.is_null());
251 let known = self.logs.get(name).and_then(|log| log.state.get(key));
252 if known == value.as_ref() {
253 return;
254 }
255 let meta = Meta::State {
256 key: key.to_owned(),
257 value: value.clone(),
258 };
259 if self.meta(name, &meta)
260 && let Some(log) = self.logs.get_mut(name)
261 {
262 log.started |= starts;
263 match value {
264 Some(value) => log.state.insert(key.to_owned(), value),
265 None => log.state.remove(key),
266 };
267 }
268 }
269
270 pub fn states(
273 &mut self,
274 name: &str,
275 known: Option<&BTreeMap<String, Value>>,
276 values: &BTreeMap<String, Value>,
277 ) {
278 let removed = known.into_iter().flat_map(BTreeMap::keys);
279 let removed: Vec<String> = removed
280 .filter(|key| !values.contains_key(*key))
281 .cloned()
282 .collect();
283 for key in removed {
284 self.save_value(name, &key, None, true);
285 }
286 for (key, value) in values {
287 self.save_value(name, key, Some(value.clone()), true);
288 }
289 }
290
291 pub fn fork(
297 &mut self,
298 store: &dyn Store,
299 (from, to): (&str, &str),
300 through: Option<MessageId>,
301 ) -> Result<Folded, SessionError> {
302 let refuse = |why: String| SessionError::from(io::Error::other(why));
303 if !self.is_live() {
304 return Err(refuse("the session is not logged".to_owned()));
305 }
306 if self.logs.contains_key(to) || log_lines(store, to)?.is_some() {
307 return Err(refuse(format!("there is a log `{to}` already")));
308 }
309 let lines = log_lines(store, from)?.unwrap_or_else(|| LogLines::parse(Vec::new()));
310 let mut bytes = Vec::new();
311 if let Some((_, start, first)) = lines.iter().next() {
312 let (stamp, _) = read_header(from, first)?;
313 let cut = turn_cut(from, &lines, through)?;
314 let stamp = Stamp {
315 t: now_ms(),
316 ..stamp
317 };
318 let header = Meta::Header(Header::new(to, None));
319 write_meta(&mut bytes, stamp, &header).map_err(io::Error::from)?;
320 let start = usize::try_from(start).unwrap_or(usize::MAX);
321 bytes.extend_from_slice(lines.prefix(cut).get(start..).unwrap_or_default());
322 } else if through.is_some() {
323 return Err(refuse(format!("the log `{from}` holds no message")));
324 }
325 if bytes.is_empty() {
326 let _ = self
327 .opened(to, None)
328 .ok_or_else(|| refuse("logging stopped".to_owned()))?;
329 let folded = Folded {
330 name: to.to_owned(),
331 ..Folded::default()
332 };
333 return Ok(folded);
334 }
335 let copy = LogLines::parse(bytes);
336 let read = |blob: &str| read_blob(store, blob);
337 let folded = fold_lines(to, ©, Fold::WHOLE, &read)?
338 .ok_or_else(|| refuse(format!("the copy of the log `{from}` is empty")))?;
339 refused(to, store.append_log(to, 0, copy.prefix(u64::MAX)))?;
340 let log = Log {
341 next_seq: folded.next_seq,
342 started: true,
343 state: folded.state.clone(),
344 lineage: folded.header.lineage,
345 changes: folded.conversation.revision(),
346 len: copy.len(),
347 ..Log::default()
348 };
349 self.logs.insert(to.to_owned(), log);
350 Ok(folded)
351 }
352
353 pub fn retire(&mut self, name: &str) {
354 self.meta(name, &Meta::Retired);
355 }
356
357 pub fn flush(&mut self, store: &dyn Store) -> Result<(), SessionError> {
363 now(self.flush_async(store))
364 }
365
366 pub async fn flush_async<S: crate::AsyncStore + ?Sized>(
368 &mut self,
369 store: &S,
370 ) -> Result<(), SessionError> {
371 if let Some(failure) = &self.failure {
372 return Err(io::Error::other(failure.clone()).into());
373 }
374 let any = self.logs.values().any(|log| log.started);
375 let mut due: Vec<(usize, String)> = self
376 .logs
377 .iter()
378 .filter(|(name, log)| log.started || any && name.as_str() == SESSION)
379 .filter(|(_, log)| !log.pending.is_empty())
380 .map(|(name, log)| (log.depth, name.clone()))
381 .collect();
382 due.sort_by(|a, b| b.cmp(a));
383 for (_, name) in due {
384 let Some(log) = self.logs.get_mut(&name) else {
385 continue;
386 };
387 let mut written = Ok(());
388 if let Some(cut) = self.torn.remove(&name) {
389 written = set_aside_tail_async(store, &name, log.len, cut)
390 .await
391 .map(drop);
392 }
393 if written.is_ok() {
394 let appended = store.append_log_async(&name, log.len, &log.pending).await;
395 written = refused(&name, appended);
396 }
397 if let Err(failure) = written {
398 let why = format!("writing the log {name} failed: {failure}");
399 self.failure = Some(why.clone());
400 return Err(match failure {
401 SessionError::Changed { .. } => failure,
402 _ => io::Error::other(why).into(),
403 });
404 }
405 log.len += log.pending.len() as u64;
406 log.pending.clear();
407 self.wrote = true;
408 }
409 Ok(())
410 }
411}
412
413#[derive(Clone, Default)]
418pub struct Writer(Arc<Mutex<Inner>>);
419
420#[derive(Default)]
421struct Inner {
422 store: Option<Arc<dyn Store>>,
423 book: Book,
424}
425
426impl Writer {
427 fn inner(&self) -> MutexGuard<'_, Inner> {
428 self.0.lock().unwrap_or_else(PoisonError::into_inner)
429 }
430
431 pub fn open(store: Arc<dyn Store>) -> Result<(Self, Session), SessionError> {
444 let mut folded = fold_all(&*store, &read_logs(&*store)?)?;
445 let mut repairs = repairs(&*store, &folded)?;
446 for repair in &mut repairs {
447 if let Repair::TornTail {
448 log,
449 kept,
450 cut,
451 set_aside,
452 } = repair
453 {
454 *set_aside = Some(set_aside_tail(&*store, log, *kept, *cut)?);
455 }
456 }
457 let book = Book::after(&folded);
458 let ends = ends(&folded);
459 let skipped = retired(&folded);
460 folded.retain(|log| !skipped.contains(&log.name));
461 let writer = Self(Arc::new(Mutex::new(Inner {
462 store: Some(store),
463 book,
464 })));
465 let session = Session {
466 ends,
467 ..session_of(folded, repairs)
468 };
469 Ok((writer, session))
470 }
471
472 pub fn start(&self) {
475 let mut inner = self.inner();
476 if inner.store.is_some() {
477 inner.book.start();
478 }
479 }
480
481 pub fn is_live(&self) -> bool {
484 self.inner().book.is_live()
485 }
486
487 pub fn failure(&self) -> Option<String> {
489 self.inner().book.failure.clone()
490 }
491
492 pub fn has_written(&self) -> bool {
495 self.inner().book.wrote
496 }
497
498 pub fn open_log(&self, log: &str, parent: Option<&str>) {
501 let _ = self.inner().book.opened(log, parent);
502 }
503
504 pub fn save(&self, log: &str, conversation: &mut Conversation) -> Vec<ChangeRecord> {
511 let mut inner = self.inner();
512 let Inner { store, book } = &mut *inner;
513 match store {
514 Some(store) => {
515 let put = &mut |name: String, bytes: Vec<u8>| put_blob(&**store, &name, &bytes);
516 book.save(put, log, conversation)
517 }
518 None => stamp_all(conversation.take_unsaved()),
519 }
520 }
521
522 pub fn save_value(&self, log: &str, key: &str, value: Option<Value>, starts: bool) {
526 self.inner().book.save_value(log, key, value, starts);
527 }
528
529 pub fn fork_log(
545 &self,
546 from: &str,
547 to: &str,
548 through: Option<MessageId>,
549 ) -> Result<SavedAgent, SessionError> {
550 let mut inner = self.inner();
551 let Inner { store, book } = &mut *inner;
552 let Some(store) = store else {
553 return Err(io::Error::other("the session is not stored").into());
554 };
555 book.flush(&**store)?;
556 let folded = book.fork(&**store, (from, to), through)?;
557 Ok(SavedAgent {
558 id: folded.name,
559 started_ms: folded.started_ms,
560 conversation: folded.conversation,
561 saved: folded.state,
562 ..SavedAgent::default()
563 })
564 }
565
566 pub fn retire(&self, log: &str) {
568 self.inner().book.retire(log);
569 }
570
571 pub fn flush(&self) -> Result<(), SessionError> {
583 let mut inner = self.inner();
584 let Inner { store, book } = &mut *inner;
585 match store {
586 Some(store) => book.flush(&**store),
587 None => Ok(()),
588 }
589 }
590}
591
592pub(crate) fn set_aside_tail(
603 store: &dyn Store,
604 log: &str,
605 kept: u64,
606 cut: u64,
607) -> Result<String, SessionError> {
608 now(set_aside_tail_async(store, log, kept, cut))
609}
610
611async fn set_aside_tail_async<S: crate::AsyncStore + ?Sized>(
613 store: &S,
614 log: &str,
615 kept: u64,
616 cut: u64,
617) -> Result<String, SessionError> {
618 let bytes = store.read_log_async(log).await?;
619 let split = usize::try_from(kept)
620 .ok()
621 .and_then(|kept| bytes.split_at_checked(kept));
622 let Some((prefix, tail)) = split.filter(|(_, tail)| tail.len() as u64 == cut) else {
623 let log = log.to_owned();
624 return Err(SessionError::Changed { log });
625 };
626 let name = format!("recovered/{log}-{}.part", content_hash(tail).0);
627 store.write_file_async(&name, tail).await?;
628 let replaced = store.replace_log_async(log, kept + cut, prefix).await;
629 refused(log, replaced)?;
630 Ok(name)
631}
632
633pub(crate) fn refused(log: &str, written: Result<(), LogWriteError>) -> Result<(), SessionError> {
636 written.map_err(|failure| match failure {
637 LogWriteError::Changed => SessionError::Changed {
638 log: log.to_owned(),
639 },
640 LogWriteError::Io(failure) => failure.into(),
641 })
642}
643
644fn stamp_all(changes: Vec<ChangeRecord>) -> Vec<ChangeRecord> {
646 let now = now_ms();
647 let stamp = |recorded: ChangeRecord| ChangeRecord {
648 at: Some(recorded.at.unwrap_or(now)),
649 ..recorded
650 };
651 changes.into_iter().map(stamp).collect()
652}