1use std::collections::{BTreeSet, HashMap};
5use std::io;
6use std::sync::Arc;
7
8use rig_core::completion::Message;
9use rig_core::conversation::{Aside, Author, Change, Conversation, Lineage, ResultFacts};
10use rig_core::json_utils::from_value;
11use serde::Serialize;
12use serde::de::DeserializeOwned;
13use serde_json::Value;
14
15use crate::blobs::{BLOBS, blob_names, named_blobs, put_blob, read_blob, store_change};
16use crate::format::{Body, Header, LogLines, LogRewrite, Meta, parse, write_change, write_meta};
17use crate::session::{Fold, Folded, fold_lines, read_header, read_logs};
18use crate::writer::{refused, set_aside_tail};
19use crate::{Files, MemoryStore, SessionError, Stamp, Store};
20use rig_core::now_ms;
21
22#[derive(Clone, Copy, Debug)]
24pub enum Place<'a> {
25 Record {
28 log: &'a str,
30 stamp: Stamp,
32 by: Option<&'a Author>,
34 },
35 File {
38 name: &'a str,
40 },
41}
42
43pub trait Rewrite: Send + Sync {
50 fn message(&self, at: &Place<'_>, message: &mut Message) -> bool {
54 let _ = (at, message);
55 false
56 }
57
58 fn aside(&self, at: &Place<'_>, aside: &mut Aside) -> bool {
60 let _ = (at, aside);
61 false
62 }
63
64 fn facts(&self, at: &Place<'_>, facts: &mut ResultFacts) -> bool {
67 let _ = (at, facts);
68 false
69 }
70
71 fn value(&self, at: &Place<'_>, key: &str, value: &mut Value) -> bool {
74 let _ = (at, key, value);
75 false
76 }
77
78 fn text(&self, at: &Place<'_>, text: &mut String) -> bool {
80 let _ = (at, text);
81 false
82 }
83
84 fn file(&self, name: &str, bytes: &mut Vec<u8>) -> bool {
86 map_file(self, name, bytes)
87 }
88}
89
90pub struct Strings<F: Fn(&mut String) -> bool + Send + Sync>(pub F);
119
120impl<F: Fn(&mut String) -> bool + Send + Sync> Rewrite for Strings<F> {
121 fn message(&self, _: &Place<'_>, message: &mut Message) -> bool {
122 map_serde(message, &self.0)
123 }
124
125 fn aside(&self, _: &Place<'_>, aside: &mut Aside) -> bool {
126 map_serde(aside, &self.0)
127 }
128
129 fn facts(&self, _: &Place<'_>, facts: &mut ResultFacts) -> bool {
130 map_serde(facts, &self.0)
131 }
132
133 fn value(&self, _: &Place<'_>, _: &str, value: &mut Value) -> bool {
134 map_strings(value, &self.0)
135 }
136
137 fn text(&self, _: &Place<'_>, text: &mut String) -> bool {
138 (self.0)(text)
139 }
140}
141
142pub struct Messages<F: Fn(&mut Message) -> bool + Send + Sync>(pub F);
144
145impl<F: Fn(&mut Message) -> bool + Send + Sync> Rewrite for Messages<F> {
146 fn message(&self, _: &Place<'_>, message: &mut Message) -> bool {
147 (self.0)(message)
148 }
149
150 fn file(&self, _: &str, _: &mut Vec<u8>) -> bool {
151 false
152 }
153}
154
155pub fn map_strings(value: &mut Value, f: &dyn Fn(&mut String) -> bool) -> bool {
158 match value {
159 Value::String(text) => f(text),
160 Value::Array(items) => items
161 .iter_mut()
162 .fold(false, |changed, item| map_strings(item, f) | changed),
163 Value::Object(map) => {
164 let mut changed = map
165 .values_mut()
166 .fold(false, |changed, item| map_strings(item, f) | changed);
167 let renamed: Vec<(String, String)> = map
168 .keys()
169 .filter_map(|key| {
170 let mut new = key.clone();
171 f(&mut new).then(|| (key.clone(), new))
172 })
173 .collect();
174 if !renamed.is_empty() {
175 let renamed: HashMap<String, String> = renamed.into_iter().collect();
176 *map = std::mem::take(map)
177 .into_iter()
178 .map(|(key, item)| match renamed.get(&key) {
179 Some(new) => (new.clone(), item),
180 None => (key, item),
181 })
182 .collect();
183 changed = true;
184 }
185 changed
186 }
187 Value::Null | Value::Bool(_) | Value::Number(_) => false,
188 }
189}
190
191fn map_serde<T: Serialize + DeserializeOwned>(
196 item: &mut T,
197 f: &dyn Fn(&mut String) -> bool,
198) -> bool {
199 let Ok(mut value) = serde_json::to_value(&*item) else {
200 return false;
201 };
202 if !map_strings(&mut value, f) {
203 return false;
204 }
205 if let Ok(read) = from_value(&value) {
206 *item = read;
207 return true;
208 }
209 let Ok(mut kept) = serde_json::to_value(&*item) else {
210 return false;
211 };
212 let mut leaves = Vec::new();
213 strings_of(&kept, &mut Vec::new(), f, &mut leaves);
214 let mut changed = false;
215 for leaf in leaves {
216 let mut trial = kept.clone();
217 if map_leaf(&mut trial, &leaf, f)
218 && let Ok(read) = from_value(&trial)
219 {
220 *item = read;
221 kept = trial;
222 changed = true;
223 }
224 }
225 changed
226}
227
228#[derive(Clone)]
230enum Step {
231 Index(usize),
232 Key(String),
233}
234
235struct Leaf {
238 path: Vec<Step>,
239 key: bool,
240}
241
242fn strings_of(
245 value: &Value,
246 path: &mut Vec<Step>,
247 f: &dyn Fn(&mut String) -> bool,
248 leaves: &mut Vec<Leaf>,
249) {
250 let changes = |text: &str| f(&mut text.to_owned());
251 match value {
252 Value::String(text) if changes(text) => leaves.push(Leaf {
253 path: path.clone(),
254 key: false,
255 }),
256 Value::Array(items) => {
257 for (at, item) in items.iter().enumerate() {
258 path.push(Step::Index(at));
259 strings_of(item, path, f, leaves);
260 path.pop();
261 }
262 }
263 Value::Object(map) => {
264 for (key, item) in map {
265 path.push(Step::Key(key.clone()));
266 strings_of(item, path, f, leaves);
267 if changes(key) {
268 leaves.push(Leaf {
269 path: path.clone(),
270 key: true,
271 });
272 }
273 path.pop();
274 }
275 }
276 _ => {}
277 }
278}
279
280fn map_leaf(value: &mut Value, leaf: &Leaf, f: &dyn Fn(&mut String) -> bool) -> bool {
282 let (last, parent) = match leaf.path.split_last() {
283 Some((last, parent)) if leaf.key => (Some(last), parent),
284 _ => (None, leaf.path.as_slice()),
285 };
286 let mut at = value;
287 for step in parent {
288 let next = match step {
289 Step::Index(index) => at.get_mut(*index),
290 Step::Key(key) => at.get_mut(key.as_str()),
291 };
292 let Some(next) = next else {
293 return false;
294 };
295 at = next;
296 }
297 match (last, at) {
298 (None, Value::String(text)) => f(text),
299 (Some(Step::Key(key)), Value::Object(map)) => {
300 let mut new = key.clone();
301 if !f(&mut new) || map.contains_key(&new) || !map.contains_key(key) {
302 return false;
303 }
304 *map = std::mem::take(map)
305 .into_iter()
306 .map(|(old, item)| match old == *key {
307 true => (new.clone(), item),
308 false => (old, item),
309 })
310 .collect();
311 true
312 }
313 _ => false,
314 }
315}
316
317pub fn rewrite_file(files: &dyn Files, name: &str, with: &dyn Rewrite) -> io::Result<bool> {
327 let mut bytes = match files.read_file(name) {
328 Ok(bytes) => bytes,
329 Err(failure) if failure.kind() == io::ErrorKind::NotFound => return Ok(false),
330 Err(failure) => return Err(failure),
331 };
332 if !with.file(name, &mut bytes) {
333 return Ok(false);
334 }
335 files.write_file(name, &bytes)?;
336 Ok(true)
337}
338
339fn map_file<R: Rewrite + ?Sized>(with: &R, name: &str, bytes: &mut Vec<u8>) -> bool {
341 let at = Place::File { name };
342 if let Ok(mut value) = serde_json::from_slice::<Value>(bytes) {
343 if !with.value(&at, name, &mut value) {
344 return false;
345 }
346 let pretty = bytes.trim_ascii_end().contains(&b'\n');
347 let written = match pretty {
348 true => serde_json::to_vec_pretty(&value),
349 false => serde_json::to_vec(&value),
350 };
351 let Ok(mut written) = written else {
352 return false;
353 };
354 if bytes.ends_with(b"\n") {
355 written.push(b'\n');
356 }
357 *bytes = written;
358 return true;
359 }
360 let Ok(text) = std::str::from_utf8(bytes) else {
361 return false;
362 };
363 let lines: Vec<(&str, &str, Option<Value>)> = text
364 .split_inclusive('\n')
365 .map(|line| {
366 let (body, end) = line
367 .strip_suffix('\n')
368 .map_or((line, ""), |body| (body, "\n"));
369 (body, end, serde_json::from_str(body).ok())
370 })
371 .collect();
372 let complete: Vec<_> = lines
373 .iter()
374 .filter(|(body, end, _)| !end.is_empty() && !body.trim().is_empty())
375 .collect();
376 let json_lines = !complete.is_empty() && complete.iter().all(|line| line.2.is_some());
377 if !json_lines {
378 let mut text = text.to_owned();
379 let changed = with.text(&at, &mut text);
380 if changed {
381 *bytes = text.into_bytes();
382 }
383 return changed;
384 }
385 let mut out = String::with_capacity(text.len());
386 let mut changed = false;
387 for (body, end, value) in lines {
388 match value {
389 Some(mut value) => match with.value(&at, name, &mut value) {
390 true => {
391 match serde_json::to_string(&value) {
392 Ok(line) => out.push_str(&line),
393 Err(_) => out.push_str(body),
394 }
395 changed = true;
396 }
397 false => out.push_str(body),
398 },
399 None => {
400 let mut line = body.to_owned();
401 changed |= with.text(&at, &mut line);
402 out.push_str(&line);
403 }
404 }
405 out.push_str(end);
406 }
407 if changed {
408 *bytes = out.into_bytes();
409 }
410 changed
411}
412
413#[derive(Clone, Debug, Default)]
415pub struct Rewritten {
416 pub logs: Vec<RewrittenLog>,
418 pub files: Vec<String>,
420 pub blobs_removed: Vec<String>,
422}
423
424#[derive(Clone, Debug)]
426pub struct RewrittenLog {
427 pub log: String,
429 pub records: usize,
431}
432
433pub(crate) struct Mapped {
435 pub header: (Stamp, Header),
437 pub body: Vec<u8>,
440 pub records: usize,
442}
443
444pub(crate) fn map_log(
448 name: &str,
449 lines: &LogLines,
450 with: &dyn Rewrite,
451 blobs: &dyn Store,
452) -> Result<Option<Mapped>, SessionError> {
453 let refuse = |line, why| SessionError::record(name, line, why);
454 let Some((_, _, first)) = lines.iter().next() else {
455 return Ok(None);
456 };
457 let (stamp, header) = read_header(name, first)?;
458 let mut mapped = Mapped {
459 header: (stamp, header),
460 body: Vec::new(),
461 records: 0,
462 };
463 for (line, _, bytes) in lines.iter().skip(1) {
464 let (stamp, body) = parse(bytes).map_err(|why| refuse(line, why))?;
465 match map_record(name, stamp, body, with, blobs).map_err(|why| refuse(line, why))? {
466 Some(record) => {
467 mapped.body.extend_from_slice(&record);
468 mapped.records += 1;
469 }
470 None => {
471 mapped.body.extend_from_slice(bytes);
472 mapped.body.push(b'\n');
473 }
474 }
475 }
476 Ok(Some(mapped))
477}
478
479pub(crate) fn map_record(
482 log: &str,
483 stamp: Stamp,
484 body: Body,
485 with: &dyn Rewrite,
486 blobs: &dyn Store,
487) -> Result<Option<Vec<u8>>, String> {
488 let mut line = Vec::new();
489 match body {
490 Body::Meta(Meta::State {
491 key,
492 value: Some(mut value),
493 }) if !value.is_null() => {
494 let at = Place::Record {
495 log,
496 stamp,
497 by: None,
498 };
499 if !with.value(&at, &key, &mut value) {
500 return Ok(None);
501 }
502 let state = Meta::State {
503 key,
504 value: Some(value),
505 };
506 write_meta(&mut line, stamp, &state).map_err(|failure| failure.to_string())?;
507 }
508 Body::Meta(_) => return Ok(None),
509 Body::Change(mut recorded) => {
510 let at = Place::Record {
511 log,
512 stamp,
513 by: recorded.by.as_ref(),
514 };
515 if !map_change(&mut recorded.change, &at, with)? {
516 return Ok(None);
517 }
518 let put = &mut |name: String, bytes: Vec<u8>| put_blob(blobs, &name, &bytes);
519 let stored = store_change(&recorded.change, put)
520 .map_err(|failure| format!("storing an image failed: {failure}"))?;
521 let change = stored.as_ref().unwrap_or(&recorded.change);
522 write_change(&mut line, stamp, recorded.by.as_ref(), change)
523 .map_err(|failure| failure.to_string())?;
524 }
525 }
526 Ok(Some(line))
527}
528
529pub(crate) fn map_change(
532 change: &mut Change,
533 at: &Place<'_>,
534 with: &dyn Rewrite,
535) -> Result<bool, String> {
536 Ok(match change {
537 Change::Push { entries } | Change::Splice { with: entries, .. } => {
538 entries.iter_mut().fold(false, |changed, entry| {
539 let message = with.message(at, Arc::make_mut(&mut entry.message));
540 message | with.facts(at, &mut entry.facts) | changed
541 })
542 }
543 Change::Aside {
544 aside: Some(aside), ..
545 } => with.aside(at, aside),
546 Change::Reset { state } => map_state(state, at, with)?,
547 Change::Aside { aside: None, .. } | Change::Halt { .. } => false,
548 })
549}
550
551fn map_state(state: &mut Conversation, at: &Place<'_>, with: &dyn Rewrite) -> Result<bool, String> {
554 let fail = |failure: serde_json::Error| failure.to_string();
555 let mut value = serde_json::to_value(&*state).map_err(fail)?;
556 let mut changed = false;
557 if let Some(Value::Array(messages)) = value.get_mut("messages") {
558 for message in messages {
559 let mut typed: Message = from_value(message).map_err(fail)?;
560 if with.message(at, &mut typed) {
561 *message = serde_json::to_value(&typed).map_err(fail)?;
562 changed = true;
563 }
564 }
565 }
566 if let Some(Value::Array(metas)) = value.get_mut("meta") {
567 for facts in metas.iter_mut().filter_map(|meta| meta.get_mut("facts")) {
568 let mut typed: ResultFacts = from_value(facts).map_err(fail)?;
569 if with.facts(at, &mut typed) {
570 *facts = serde_json::to_value(&typed).map_err(fail)?;
571 changed = true;
572 }
573 }
574 }
575 if let Some(Value::Array(asides)) = value.get_mut("asides") {
576 for aside in asides.iter_mut().filter_map(|pair| pair.get_mut(1)) {
577 let mut typed: Aside = from_value(aside).map_err(fail)?;
578 if with.aside(at, &mut typed) {
579 *aside = serde_json::to_value(&typed).map_err(fail)?;
580 changed = true;
581 }
582 }
583 }
584 if changed {
585 *state = from_value(&value).map_err(fail)?;
586 }
587 Ok(changed)
588}
589
590pub fn rewrite(
626 store: &dyn Store,
627 by: Author,
628 with: &dyn Rewrite,
629) -> Result<Rewritten, SessionError> {
630 let mut logs: Vec<(String, LogLines, Folded)> = Vec::new();
631 for (name, lines) in read_logs(store)? {
632 if let Some(folded) = fold_lines(&name, &lines, Fold::default(), &|_| {
633 Err(io::ErrorKind::NotFound.into())
634 })? {
635 logs.push((name, lines, folded));
636 }
637 }
638 logs.sort_by(|a, b| a.0.cmp(&b.0));
639 let t = now_ms();
640 let blobs = MemoryStore::default();
641 let blob = |name: &str| read_blob(&blobs, name).or_else(|_| read_blob(store, name));
642 let mut rewritten = Rewritten::default();
643 let mut replaced: Vec<(u64, Vec<u8>)> = Vec::new();
644 for (name, lines, _) in &logs {
645 let Some(mapped) = map_log(name, lines, with, &blobs)? else {
646 continue;
647 };
648 if mapped.records == 0 {
649 continue;
650 }
651 let (stamp, mut header) = mapped.header;
652 header.lineage = Lineage::fresh();
653 header.rewrites.push(LogRewrite { t, by: by.clone() });
654 let mut bytes = Vec::with_capacity(mapped.body.len() + 256);
655 write_meta(&mut bytes, stamp, &Meta::Header(header)).map_err(io::Error::from)?;
656 bytes.extend_from_slice(&mapped.body);
657 let new = LogLines::parse(bytes);
658 if fold_lines(name, &new, Fold::WHOLE, &blob)?.is_none() {
659 let why = "the rewritten log is empty".to_owned();
660 return Err(SessionError::record(name, 1, why));
661 }
662 rewritten.logs.push(RewrittenLog {
663 log: name.clone(),
664 records: mapped.records,
665 });
666 replaced.push((lines.len(), new.prefix(u64::MAX).to_vec()));
667 }
668
669 for (name, _, old) in &logs {
671 if let Some((kept, cut)) = old.torn {
672 set_aside_tail(store, name, kept, cut)?;
673 }
674 }
675 for name in blob_names(&blobs)? {
676 put_blob(store, &name, &read_blob(&blobs, &name)?)?;
677 }
678 for (log, (len, bytes)) in rewritten.logs.iter().zip(&replaced) {
679 refused(&log.log, store.replace_log(&log.log, *len, bytes))?;
680 }
681 for name in store.files()? {
682 if !name.starts_with(BLOBS) && rewrite_file(store, &name, with)? {
683 rewritten.files.push(name);
684 }
685 }
686 rewritten.files.sort();
687 let mut named = BTreeSet::new();
688 for (name, lines, _) in &logs {
689 let log = rewritten.logs.iter().position(|log| &log.log == name);
690 let bytes = match log.and_then(|at| replaced.get(at)) {
691 Some((_, bytes)) => bytes.as_slice(),
692 None => lines.prefix(u64::MAX),
693 };
694 named_blobs(bytes, &mut named);
695 }
696 let mut stored = blob_names(store)?;
697 stored.sort();
698 for name in stored {
699 if !named.contains(&name) {
700 store.remove_file(&format!("{BLOBS}{name}"))?;
701 rewritten.blobs_removed.push(name);
702 }
703 }
704 Ok(rewritten)
705}
706
707#[cfg(test)]
708mod tests;