Skip to main content

ocpp_kit/engine/
store_file.rs

1//! A durable [`MessageStore`] backed by an append-only journal.
2
3use alloc::collections::BTreeMap;
4use alloc::format;
5use alloc::string::{String, ToString};
6use alloc::vec::Vec;
7use core::fmt;
8
9use std::fs::{File, OpenOptions};
10use std::io::{BufRead, BufReader, Write};
11use std::path::{Path, PathBuf};
12
13use serde::{Deserialize, Serialize};
14
15use super::queue::{MessageStore, QueuedCall, Seq, StoreError};
16
17/// One line of the journal.
18///
19/// A record per *change*, not a snapshot per write: appending a line and flushing it is one
20/// sequential write, which is what makes the cost of durability bearable on the flash a
21/// charging station actually has.
22#[derive(Serialize, Deserialize)]
23#[serde(tag = "op", rename_all = "lowercase")]
24enum Record {
25    /// A message was queued.
26    Push {
27        seq: Seq,
28        #[serde(flatten)]
29        entry: StoredCall,
30    },
31    /// A message was delivered or definitively abandoned.
32    Ack { seq: Seq },
33    /// A further transmission was attempted.
34    Attempts { seq: Seq, attempts: u32 },
35}
36
37/// A [`QueuedCall`] in a form that survives a round trip through JSON.
38#[derive(Clone, Serialize, Deserialize)]
39struct StoredCall {
40    action: String,
41    /// The payload as its raw JSON text, so it is byte-identical after a reboot — a
42    /// re-serialized payload could differ, and a `TransactionEvent` that changes shape between
43    /// attempts is one a CSMS cannot deduplicate.
44    payload: String,
45    send: bool,
46    attempts: u32,
47    transactional: bool,
48}
49
50impl StoredCall {
51    fn from_call(entry: &QueuedCall) -> Self {
52        Self {
53            action: entry.action.clone(),
54            payload: entry.payload.get().to_string(),
55            send: entry.kind == crate::message::MessageKind::Send,
56            attempts: entry.attempts,
57            transactional: entry.transactional,
58        }
59    }
60
61    fn into_call(self) -> Result<QueuedCall, StoreError> {
62        let payload = serde_json::value::RawValue::from_string(self.payload)
63            .map_err(|error| StoreError::new(format!("stored payload is not JSON: {error}")))?;
64        Ok(QueuedCall {
65            action: self.action,
66            payload,
67            kind: if self.send {
68                crate::message::MessageKind::Send
69            } else {
70                crate::message::MessageKind::Call
71            },
72            attempts: self.attempts,
73            transactional: self.transactional,
74        })
75    }
76}
77
78/// How much of the journal may be dead records before it is rewritten.
79///
80/// A station queues and acknowledges the same handful of messages for years, so without
81/// compaction the file grows without bound while the live set stays tiny.
82const COMPACT_RATIO: usize = 4;
83
84/// The smallest journal worth rewriting. Below this, compaction costs more than it saves.
85const COMPACT_FLOOR: usize = 64;
86
87/// A [`MessageStore`] that survives a power cut, for the station side.
88///
89/// An append-only journal, one line of JSON per change. A Charging Station must replay the
90/// transaction messages an outage interrupted (E04.FR.01–03, E08.FR.05–07, E12.FR.01–02),
91/// which [`MemStore`](super::MemStore) cannot.
92///
93/// ```no_run
94/// use ocpp_kit::Version;
95/// use ocpp_kit::engine::{Engine, EngineConfig, FileStore, Role};
96///
97/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
98/// // Whatever the outage interrupted is back in the queue by the time this returns.
99/// let store = FileStore::open("/var/lib/ocpp/queue.jsonl")?;
100/// let engine = Engine::with_store(
101///     EngineConfig::new(Role::ChargingStation, Version::V2_1),
102///     store,
103/// )?;
104/// # let _ = engine;
105/// # Ok(()) }
106/// ```
107///
108/// # Guarantees
109///
110/// [`push`](MessageStore::push) returns only once the record has reached the device
111/// (`sync_data`). A crash mid-append leaves a partial final line, discarded on the next open:
112/// it described a message that was never reported as queued.
113///
114/// One process, one file — checked, not assumed. A second writer produces a sequence number
115/// twice, which one forward-only counter cannot, and [`open`](Self::open) refuses the journal
116/// and names the cause. Refused rather than repaired: the interleaved records are each
117/// well-formed, so a repair would be a guess about whose queue is real.
118///
119/// No defence against a corrupt filesystem. For anything else, implement [`MessageStore`]; it
120/// is four synchronous methods.
121pub struct FileStore {
122    path: PathBuf,
123    journal: File,
124    live: BTreeMap<Seq, QueuedCall>,
125    next_seq: Seq,
126    /// Records written since the last compaction, live or not.
127    written: usize,
128}
129
130impl fmt::Debug for FileStore {
131    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
132        f.debug_struct("FileStore")
133            .field("path", &self.path)
134            .field("queued", &self.live.len())
135            .finish_non_exhaustive()
136    }
137}
138
139impl FileStore {
140    /// Opens (or creates) the journal at `path`, replaying whatever an outage left behind.
141    ///
142    /// The parent directory must exist. Replaying compacts: the file that comes back holds
143    /// exactly the messages that are still queued.
144    pub fn open(path: impl AsRef<Path>) -> Result<Self, StoreError> {
145        let path = path.as_ref().to_path_buf();
146        let (live, next_seq) = Self::replay(&path)?;
147
148        // Replaced immediately by `rewrite`; opening it here keeps the field non-optional.
149        let journal = OpenOptions::new()
150            .create(true)
151            .append(true)
152            .open(&path)
153            .map_err(|error| io(&path, &error))?;
154        let mut store = Self {
155            path,
156            journal,
157            live,
158            next_seq,
159            written: 0,
160        };
161        store.rewrite()?;
162        Ok(store)
163    }
164
165    /// The journal's path.
166    #[must_use]
167    pub fn path(&self) -> &Path {
168        &self.path
169    }
170
171    /// Rewrites the journal so it holds only what is still queued.
172    ///
173    /// Called on open and whenever the dead records outnumber the live ones; exposed because a
174    /// long-running station may want to schedule it rather than pay for it mid-transaction.
175    pub fn compact(&mut self) -> Result<(), StoreError> {
176        self.rewrite()
177    }
178
179    /// Replays a journal into the set of messages still queued.
180    ///
181    /// A line that does not parse ends the replay rather than being skipped. A journal is
182    /// append-only, so the only way to reach one is a crash mid-write, and everything after
183    /// such a line is by definition not there — treating it as a gap to step over would be
184    /// inventing an ordering the file does not have.
185    fn replay(path: &Path) -> Result<(BTreeMap<Seq, QueuedCall>, Seq), StoreError> {
186        let mut live: BTreeMap<Seq, QueuedCall> = BTreeMap::new();
187        let mut next_seq: Seq = 0;
188        // Every sequence number this journal has ever pushed, not just the ones still live:
189        // an `Ack` removes an entry from `live`, so checking `live` alone would miss a
190        // duplicate that arrives after one.
191        let mut pushed: alloc::collections::BTreeSet<Seq> = alloc::collections::BTreeSet::new();
192
193        let file = match File::open(path) {
194            Ok(file) => file,
195            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
196                return Ok((live, next_seq));
197            }
198            Err(error) => return Err(io(path, &error)),
199        };
200
201        for read in BufReader::new(file).lines() {
202            let Ok(text) = read else { break };
203            if text.trim().is_empty() {
204                continue;
205            }
206            let Ok(record) = serde_json::from_str::<Record>(&text) else {
207                break;
208            };
209            match record {
210                Record::Push { seq, entry } => {
211                    // A single writer allocates sequence numbers from a counter that only
212                    // ever moves forward, so it cannot produce the same one twice. Seeing it
213                    // twice means two processes have been appending to this journal — and
214                    // their interleaved records describe a queue neither of them has.
215                    //
216                    // This is refused rather than repaired: the records are individually
217                    // well-formed, so any repair would be a guess about which writer's view
218                    // of the queue is the real one, on messages a station is obliged to
219                    // deliver exactly once.
220                    if !pushed.insert(seq) {
221                        return Err(StoreError::new(format!(
222                            "{}: sequence number {seq} was queued twice, which one writer \
223                             cannot do — two processes have the same journal open. Give each \
224                             one its own file.",
225                            path.display(),
226                        )));
227                    }
228                    next_seq = next_seq.max(seq + 1);
229                    live.insert(seq, entry.into_call()?);
230                }
231                Record::Ack { seq } => {
232                    next_seq = next_seq.max(seq + 1);
233                    live.remove(&seq);
234                }
235                Record::Attempts { seq, attempts } => {
236                    next_seq = next_seq.max(seq + 1);
237                    if let Some(entry) = live.get_mut(&seq) {
238                        entry.attempts = attempts;
239                    }
240                }
241            }
242        }
243        Ok((live, next_seq))
244    }
245
246    /// Writes a fresh journal containing only the live entries, then swaps it in.
247    fn rewrite(&mut self) -> Result<(), StoreError> {
248        let temporary = self.path.with_extension("compacting");
249        let mut file = File::create(&temporary).map_err(|error| io(&temporary, &error))?;
250        for (seq, entry) in &self.live {
251            let record = Record::Push {
252                seq: *seq,
253                entry: StoredCall::from_call(entry),
254            };
255            write_record(&mut file, &temporary, &record)?;
256        }
257        file.sync_data().map_err(|error| io(&temporary, &error))?;
258        drop(file);
259
260        // `rename` replaces atomically, so a crash here leaves either the old journal or the
261        // new one — never a half-written mixture.
262        std::fs::rename(&temporary, &self.path).map_err(|error| io(&self.path, &error))?;
263        self.journal = OpenOptions::new()
264            .create(true)
265            .append(true)
266            .open(&self.path)
267            .map_err(|error| io(&self.path, &error))?;
268        self.written = self.live.len();
269        Ok(())
270    }
271
272    /// Appends one record and flushes it to the device.
273    fn append(&mut self, record: &Record) -> Result<(), StoreError> {
274        write_record(&mut self.journal, &self.path, record)?;
275        // The whole point: a message the engine is told is queued is on the device, not in a
276        // page cache that a power cut discards.
277        self.journal
278            .sync_data()
279            .map_err(|error| io(&self.path, &error))?;
280        self.written += 1;
281        Ok(())
282    }
283
284    /// Rewrites the journal once the dead records have come to outweigh the live ones.
285    fn compact_if_worthwhile(&mut self) -> Result<(), StoreError> {
286        if self.written >= COMPACT_FLOOR && self.written >= self.live.len() * COMPACT_RATIO {
287            self.rewrite()?;
288        }
289        Ok(())
290    }
291}
292
293impl MessageStore for FileStore {
294    fn push(&mut self, entry: &QueuedCall) -> Result<Seq, StoreError> {
295        let seq = self.next_seq;
296        self.append(&Record::Push {
297            seq,
298            entry: StoredCall::from_call(entry),
299        })?;
300        self.next_seq += 1;
301        self.live.insert(seq, entry.clone());
302        Ok(seq)
303    }
304
305    fn pending(&self) -> Result<Vec<(Seq, QueuedCall)>, StoreError> {
306        Ok(self
307            .live
308            .iter()
309            .map(|(seq, entry)| (*seq, entry.clone()))
310            .collect())
311    }
312
313    fn ack(&mut self, seq: Seq) -> Result<(), StoreError> {
314        if self.live.remove(&seq).is_none() {
315            return Ok(());
316        }
317        self.append(&Record::Ack { seq })?;
318        self.compact_if_worthwhile()
319    }
320
321    fn set_attempts(&mut self, seq: Seq, attempts: u32) -> Result<(), StoreError> {
322        let Some(entry) = self.live.get_mut(&seq) else {
323            return Ok(());
324        };
325        entry.attempts = attempts;
326        self.append(&Record::Attempts { seq, attempts })?;
327        self.compact_if_worthwhile()
328    }
329
330    fn len(&self) -> Result<usize, StoreError> {
331        Ok(self.live.len())
332    }
333}
334
335fn write_record(file: &mut File, path: &Path, record: &Record) -> Result<(), StoreError> {
336    let mut line = serde_json::to_vec(record)
337        .map_err(|error| StoreError::new(format!("could not encode a journal record: {error}")))?;
338    line.push(b'\n');
339    file.write_all(&line).map_err(|error| io(path, &error))
340}
341
342fn io(path: &Path, error: &std::io::Error) -> StoreError {
343    StoreError::new(format!("{}: {error}", path.display()))
344}
345
346#[cfg(test)]
347mod tests {
348    use super::*;
349    use crate::message::MessageKind;
350    use serde_json::value::RawValue;
351
352    fn call(seq_no: u32) -> QueuedCall {
353        QueuedCall {
354            action: "TransactionEvent".into(),
355            payload: RawValue::from_string(format!(r#"{{"seqNo":{seq_no}}}"#)).unwrap(),
356            kind: MessageKind::Call,
357            attempts: 0,
358            transactional: true,
359        }
360    }
361
362    fn temp_dir() -> std::path::PathBuf {
363        let dir = std::env::temp_dir().join(format!(
364            "ocpp-kit-store-{}-{:?}",
365            std::process::id(),
366            std::thread::current().id()
367        ));
368        std::fs::create_dir_all(&dir).unwrap();
369        dir
370    }
371
372    /// The whole reason this type exists: E04/E08/E12 require the messages an outage
373    /// interrupted to be replayed, in order, after the station comes back.
374    #[test]
375    fn a_power_cut_does_not_lose_what_was_queued() {
376        let path = temp_dir().join("replay.jsonl");
377        let _ = std::fs::remove_file(&path);
378
379        let mut store = FileStore::open(&path).unwrap();
380        let first = store.push(&call(0)).unwrap();
381        store.push(&call(1)).unwrap();
382        store.push(&call(2)).unwrap();
383        store.ack(first).unwrap();
384        store.set_attempts(1, 2).unwrap();
385        // The power goes out here: no close, no flush of our own beyond what `push` did.
386        drop(store);
387
388        let store = FileStore::open(&path).unwrap();
389        let pending = store.pending().unwrap();
390        assert_eq!(
391            pending.len(),
392            2,
393            "the acknowledged one is gone, the rest stay"
394        );
395        assert_eq!(pending[0].0, 1, "and they come back oldest first");
396        assert_eq!(pending[0].1.attempts, 2, "with the attempts they had made");
397        assert!(pending[0].1.payload.get().contains(r#""seqNo":1"#));
398        assert_eq!(pending[1].0, 2);
399        std::fs::remove_file(&path).unwrap();
400    }
401
402    /// A crash *during* an append leaves a partial line. It described a message that was never
403    /// reported as queued, so discarding it loses nothing anyone was told about — and stopping
404    /// there rather than skipping it keeps the replay from inventing an order the file does
405    /// not have.
406    #[test]
407    fn a_torn_final_record_is_discarded_rather_than_poisoning_the_queue() {
408        let path = temp_dir().join("torn.jsonl");
409        let _ = std::fs::remove_file(&path);
410
411        let mut store = FileStore::open(&path).unwrap();
412        store.push(&call(0)).unwrap();
413        store.push(&call(1)).unwrap();
414        drop(store);
415
416        // Simulate the tear: half a record, as a power cut mid-`write_all` would leave.
417        let mut file = OpenOptions::new().append(true).open(&path).unwrap();
418        file.write_all(br#"{"op":"push","seq":2,"action":"Transact"#)
419            .unwrap();
420        drop(file);
421
422        let store = FileStore::open(&path).unwrap();
423        assert_eq!(store.len().unwrap(), 2);
424        std::fs::remove_file(&path).unwrap();
425    }
426
427    /// A station queues and acknowledges the same few messages for years. Without compaction
428    /// the journal grows without bound while the live set stays tiny.
429    #[test]
430    fn the_journal_does_not_grow_without_bound() {
431        let path = temp_dir().join("compact.jsonl");
432        let _ = std::fs::remove_file(&path);
433
434        let mut store = FileStore::open(&path).unwrap();
435        for _ in 0..200 {
436            let seq = store.push(&call(0)).unwrap();
437            store.ack(seq).unwrap();
438        }
439        assert_eq!(store.len().unwrap(), 0);
440
441        let lines = std::fs::read_to_string(&path).unwrap().lines().count();
442        assert!(
443            lines <= COMPACT_FLOOR * 2,
444            "400 records should have been compacted away, {lines} lines remain"
445        );
446
447        // And the compacted file still replays correctly.
448        store.push(&call(9)).unwrap();
449        drop(store);
450        let store = FileStore::open(&path).unwrap();
451        assert_eq!(store.len().unwrap(), 1);
452        std::fs::remove_file(&path).unwrap();
453    }
454
455    #[test]
456    fn a_send_keeps_its_kind_across_a_reboot() {
457        let path = temp_dir().join("kinds.jsonl");
458        let _ = std::fs::remove_file(&path);
459
460        let mut store = FileStore::open(&path).unwrap();
461        let mut send = call(0);
462        send.kind = MessageKind::Send;
463        store.push(&send).unwrap();
464        drop(store);
465
466        let store = FileStore::open(&path).unwrap();
467        assert_eq!(store.pending().unwrap()[0].1.kind, MessageKind::Send);
468        std::fs::remove_file(&path).unwrap();
469    }
470
471    #[test]
472    fn two_writers_on_one_journal_are_refused_rather_than_merged() {
473        let path = std::env::temp_dir().join("ocpp-kit-two-writers.jsonl");
474        let _ = std::fs::remove_file(&path);
475
476        // What two processes each holding their own counter would leave behind: both allocate
477        // seq 0, and both append a well-formed record for it.
478        let entry = |action: &str| {
479            format!(
480                r#"{{"op":"push","seq":0,"action":"{action}","payload":"{{}}","send":false,"attempts":0,"transactional":true}}"#
481            )
482        };
483        std::fs::write(
484            &path,
485            format!(
486                "{}\n{}\n",
487                entry("TransactionEvent"),
488                entry("StopTransaction")
489            ),
490        )
491        .unwrap();
492
493        let error = FileStore::open(&path).expect_err("a duplicate seq is not recoverable");
494        assert!(
495            error.reason.contains("queued twice"),
496            "the error has to name the cause, or an operator debugs the wrong thing: {}",
497            error.reason
498        );
499
500        // An `Ack` in between does not hide it: the check is over everything ever pushed, not
501        // over what is still live.
502        std::fs::write(
503            &path,
504            format!(
505                "{}\n{{\"op\":\"ack\",\"seq\":0}}\n{}\n",
506                entry("TransactionEvent"),
507                entry("StopTransaction")
508            ),
509        )
510        .unwrap();
511        assert!(FileStore::open(&path).is_err());
512
513        std::fs::remove_file(&path).unwrap();
514    }
515}