1use alloc::collections::BTreeMap;
4use alloc::format;
5use alloc::string::{String, ToString};
6use alloc::vec::Vec;
7use core::fmt;
8
9use std::fs::{File, OpenOptions};
10use std::io::{BufRead, BufReader, Write};
11use std::path::{Path, PathBuf};
12
13use serde::{Deserialize, Serialize};
14
15use super::queue::{MessageStore, QueuedCall, Seq, StoreError};
16
17#[derive(Serialize, Deserialize)]
23#[serde(tag = "op", rename_all = "lowercase")]
24enum Record {
25 Push {
27 seq: Seq,
28 #[serde(flatten)]
29 entry: StoredCall,
30 },
31 Ack { seq: Seq },
33 Attempts { seq: Seq, attempts: u32 },
35}
36
37#[derive(Clone, Serialize, Deserialize)]
39struct StoredCall {
40 action: String,
41 payload: String,
45 send: bool,
46 attempts: u32,
47 transactional: bool,
48}
49
50impl StoredCall {
51 fn from_call(entry: &QueuedCall) -> Self {
52 Self {
53 action: entry.action.clone(),
54 payload: entry.payload.get().to_string(),
55 send: entry.kind == crate::message::MessageKind::Send,
56 attempts: entry.attempts,
57 transactional: entry.transactional,
58 }
59 }
60
61 fn into_call(self) -> Result<QueuedCall, StoreError> {
62 let payload = serde_json::value::RawValue::from_string(self.payload)
63 .map_err(|error| StoreError::new(format!("stored payload is not JSON: {error}")))?;
64 Ok(QueuedCall {
65 action: self.action,
66 payload,
67 kind: if self.send {
68 crate::message::MessageKind::Send
69 } else {
70 crate::message::MessageKind::Call
71 },
72 attempts: self.attempts,
73 transactional: self.transactional,
74 })
75 }
76}
77
78const COMPACT_RATIO: usize = 4;
83
84const COMPACT_FLOOR: usize = 64;
86
87pub struct FileStore {
122 path: PathBuf,
123 journal: File,
124 live: BTreeMap<Seq, QueuedCall>,
125 next_seq: Seq,
126 written: usize,
128}
129
130impl fmt::Debug for FileStore {
131 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
132 f.debug_struct("FileStore")
133 .field("path", &self.path)
134 .field("queued", &self.live.len())
135 .finish_non_exhaustive()
136 }
137}
138
139impl FileStore {
140 pub fn open(path: impl AsRef<Path>) -> Result<Self, StoreError> {
145 let path = path.as_ref().to_path_buf();
146 let (live, next_seq) = Self::replay(&path)?;
147
148 let journal = OpenOptions::new()
150 .create(true)
151 .append(true)
152 .open(&path)
153 .map_err(|error| io(&path, &error))?;
154 let mut store = Self {
155 path,
156 journal,
157 live,
158 next_seq,
159 written: 0,
160 };
161 store.rewrite()?;
162 Ok(store)
163 }
164
165 #[must_use]
167 pub fn path(&self) -> &Path {
168 &self.path
169 }
170
171 pub fn compact(&mut self) -> Result<(), StoreError> {
176 self.rewrite()
177 }
178
179 fn replay(path: &Path) -> Result<(BTreeMap<Seq, QueuedCall>, Seq), StoreError> {
186 let mut live: BTreeMap<Seq, QueuedCall> = BTreeMap::new();
187 let mut next_seq: Seq = 0;
188 let mut pushed: alloc::collections::BTreeSet<Seq> = alloc::collections::BTreeSet::new();
192
193 let file = match File::open(path) {
194 Ok(file) => file,
195 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
196 return Ok((live, next_seq));
197 }
198 Err(error) => return Err(io(path, &error)),
199 };
200
201 for read in BufReader::new(file).lines() {
202 let Ok(text) = read else { break };
203 if text.trim().is_empty() {
204 continue;
205 }
206 let Ok(record) = serde_json::from_str::<Record>(&text) else {
207 break;
208 };
209 match record {
210 Record::Push { seq, entry } => {
211 if !pushed.insert(seq) {
221 return Err(StoreError::new(format!(
222 "{}: sequence number {seq} was queued twice, which one writer \
223 cannot do — two processes have the same journal open. Give each \
224 one its own file.",
225 path.display(),
226 )));
227 }
228 next_seq = next_seq.max(seq + 1);
229 live.insert(seq, entry.into_call()?);
230 }
231 Record::Ack { seq } => {
232 next_seq = next_seq.max(seq + 1);
233 live.remove(&seq);
234 }
235 Record::Attempts { seq, attempts } => {
236 next_seq = next_seq.max(seq + 1);
237 if let Some(entry) = live.get_mut(&seq) {
238 entry.attempts = attempts;
239 }
240 }
241 }
242 }
243 Ok((live, next_seq))
244 }
245
246 fn rewrite(&mut self) -> Result<(), StoreError> {
248 let temporary = self.path.with_extension("compacting");
249 let mut file = File::create(&temporary).map_err(|error| io(&temporary, &error))?;
250 for (seq, entry) in &self.live {
251 let record = Record::Push {
252 seq: *seq,
253 entry: StoredCall::from_call(entry),
254 };
255 write_record(&mut file, &temporary, &record)?;
256 }
257 file.sync_data().map_err(|error| io(&temporary, &error))?;
258 drop(file);
259
260 std::fs::rename(&temporary, &self.path).map_err(|error| io(&self.path, &error))?;
263 self.journal = OpenOptions::new()
264 .create(true)
265 .append(true)
266 .open(&self.path)
267 .map_err(|error| io(&self.path, &error))?;
268 self.written = self.live.len();
269 Ok(())
270 }
271
272 fn append(&mut self, record: &Record) -> Result<(), StoreError> {
274 write_record(&mut self.journal, &self.path, record)?;
275 self.journal
278 .sync_data()
279 .map_err(|error| io(&self.path, &error))?;
280 self.written += 1;
281 Ok(())
282 }
283
284 fn compact_if_worthwhile(&mut self) -> Result<(), StoreError> {
286 if self.written >= COMPACT_FLOOR && self.written >= self.live.len() * COMPACT_RATIO {
287 self.rewrite()?;
288 }
289 Ok(())
290 }
291}
292
293impl MessageStore for FileStore {
294 fn push(&mut self, entry: &QueuedCall) -> Result<Seq, StoreError> {
295 let seq = self.next_seq;
296 self.append(&Record::Push {
297 seq,
298 entry: StoredCall::from_call(entry),
299 })?;
300 self.next_seq += 1;
301 self.live.insert(seq, entry.clone());
302 Ok(seq)
303 }
304
305 fn pending(&self) -> Result<Vec<(Seq, QueuedCall)>, StoreError> {
306 Ok(self
307 .live
308 .iter()
309 .map(|(seq, entry)| (*seq, entry.clone()))
310 .collect())
311 }
312
313 fn ack(&mut self, seq: Seq) -> Result<(), StoreError> {
314 if self.live.remove(&seq).is_none() {
315 return Ok(());
316 }
317 self.append(&Record::Ack { seq })?;
318 self.compact_if_worthwhile()
319 }
320
321 fn set_attempts(&mut self, seq: Seq, attempts: u32) -> Result<(), StoreError> {
322 let Some(entry) = self.live.get_mut(&seq) else {
323 return Ok(());
324 };
325 entry.attempts = attempts;
326 self.append(&Record::Attempts { seq, attempts })?;
327 self.compact_if_worthwhile()
328 }
329
330 fn len(&self) -> Result<usize, StoreError> {
331 Ok(self.live.len())
332 }
333}
334
335fn write_record(file: &mut File, path: &Path, record: &Record) -> Result<(), StoreError> {
336 let mut line = serde_json::to_vec(record)
337 .map_err(|error| StoreError::new(format!("could not encode a journal record: {error}")))?;
338 line.push(b'\n');
339 file.write_all(&line).map_err(|error| io(path, &error))
340}
341
342fn io(path: &Path, error: &std::io::Error) -> StoreError {
343 StoreError::new(format!("{}: {error}", path.display()))
344}
345
346#[cfg(test)]
347mod tests {
348 use super::*;
349 use crate::message::MessageKind;
350 use serde_json::value::RawValue;
351
352 fn call(seq_no: u32) -> QueuedCall {
353 QueuedCall {
354 action: "TransactionEvent".into(),
355 payload: RawValue::from_string(format!(r#"{{"seqNo":{seq_no}}}"#)).unwrap(),
356 kind: MessageKind::Call,
357 attempts: 0,
358 transactional: true,
359 }
360 }
361
362 fn temp_dir() -> std::path::PathBuf {
363 let dir = std::env::temp_dir().join(format!(
364 "ocpp-kit-store-{}-{:?}",
365 std::process::id(),
366 std::thread::current().id()
367 ));
368 std::fs::create_dir_all(&dir).unwrap();
369 dir
370 }
371
372 #[test]
375 fn a_power_cut_does_not_lose_what_was_queued() {
376 let path = temp_dir().join("replay.jsonl");
377 let _ = std::fs::remove_file(&path);
378
379 let mut store = FileStore::open(&path).unwrap();
380 let first = store.push(&call(0)).unwrap();
381 store.push(&call(1)).unwrap();
382 store.push(&call(2)).unwrap();
383 store.ack(first).unwrap();
384 store.set_attempts(1, 2).unwrap();
385 drop(store);
387
388 let store = FileStore::open(&path).unwrap();
389 let pending = store.pending().unwrap();
390 assert_eq!(
391 pending.len(),
392 2,
393 "the acknowledged one is gone, the rest stay"
394 );
395 assert_eq!(pending[0].0, 1, "and they come back oldest first");
396 assert_eq!(pending[0].1.attempts, 2, "with the attempts they had made");
397 assert!(pending[0].1.payload.get().contains(r#""seqNo":1"#));
398 assert_eq!(pending[1].0, 2);
399 std::fs::remove_file(&path).unwrap();
400 }
401
402 #[test]
407 fn a_torn_final_record_is_discarded_rather_than_poisoning_the_queue() {
408 let path = temp_dir().join("torn.jsonl");
409 let _ = std::fs::remove_file(&path);
410
411 let mut store = FileStore::open(&path).unwrap();
412 store.push(&call(0)).unwrap();
413 store.push(&call(1)).unwrap();
414 drop(store);
415
416 let mut file = OpenOptions::new().append(true).open(&path).unwrap();
418 file.write_all(br#"{"op":"push","seq":2,"action":"Transact"#)
419 .unwrap();
420 drop(file);
421
422 let store = FileStore::open(&path).unwrap();
423 assert_eq!(store.len().unwrap(), 2);
424 std::fs::remove_file(&path).unwrap();
425 }
426
427 #[test]
430 fn the_journal_does_not_grow_without_bound() {
431 let path = temp_dir().join("compact.jsonl");
432 let _ = std::fs::remove_file(&path);
433
434 let mut store = FileStore::open(&path).unwrap();
435 for _ in 0..200 {
436 let seq = store.push(&call(0)).unwrap();
437 store.ack(seq).unwrap();
438 }
439 assert_eq!(store.len().unwrap(), 0);
440
441 let lines = std::fs::read_to_string(&path).unwrap().lines().count();
442 assert!(
443 lines <= COMPACT_FLOOR * 2,
444 "400 records should have been compacted away, {lines} lines remain"
445 );
446
447 store.push(&call(9)).unwrap();
449 drop(store);
450 let store = FileStore::open(&path).unwrap();
451 assert_eq!(store.len().unwrap(), 1);
452 std::fs::remove_file(&path).unwrap();
453 }
454
455 #[test]
456 fn a_send_keeps_its_kind_across_a_reboot() {
457 let path = temp_dir().join("kinds.jsonl");
458 let _ = std::fs::remove_file(&path);
459
460 let mut store = FileStore::open(&path).unwrap();
461 let mut send = call(0);
462 send.kind = MessageKind::Send;
463 store.push(&send).unwrap();
464 drop(store);
465
466 let store = FileStore::open(&path).unwrap();
467 assert_eq!(store.pending().unwrap()[0].1.kind, MessageKind::Send);
468 std::fs::remove_file(&path).unwrap();
469 }
470
471 #[test]
472 fn two_writers_on_one_journal_are_refused_rather_than_merged() {
473 let path = std::env::temp_dir().join("ocpp-kit-two-writers.jsonl");
474 let _ = std::fs::remove_file(&path);
475
476 let entry = |action: &str| {
479 format!(
480 r#"{{"op":"push","seq":0,"action":"{action}","payload":"{{}}","send":false,"attempts":0,"transactional":true}}"#
481 )
482 };
483 std::fs::write(
484 &path,
485 format!(
486 "{}\n{}\n",
487 entry("TransactionEvent"),
488 entry("StopTransaction")
489 ),
490 )
491 .unwrap();
492
493 let error = FileStore::open(&path).expect_err("a duplicate seq is not recoverable");
494 assert!(
495 error.reason.contains("queued twice"),
496 "the error has to name the cause, or an operator debugs the wrong thing: {}",
497 error.reason
498 );
499
500 std::fs::write(
503 &path,
504 format!(
505 "{}\n{{\"op\":\"ack\",\"seq\":0}}\n{}\n",
506 entry("TransactionEvent"),
507 entry("StopTransaction")
508 ),
509 )
510 .unwrap();
511 assert!(FileStore::open(&path).is_err());
512
513 std::fs::remove_file(&path).unwrap();
514 }
515}