Skip to main content

ocpp_kit/engine/
store_file.rs

1//! A durable [`MessageStore`] backed by an append-only journal.
2
3use alloc::collections::BTreeMap;
4use alloc::format;
5use alloc::string::{String, ToString};
6use alloc::vec::Vec;
7use core::fmt;
8
9use std::fs::{File, OpenOptions};
10use std::io::{BufRead, BufReader, Write};
11use std::path::{Path, PathBuf};
12
13use serde::{Deserialize, Serialize};
14
15use super::queue::{MessageStore, QueuedCall, Seq, StoreError};
16
17/// One line of the journal.
18///
19/// A record per *change*, not a snapshot per write: appending a line and flushing it is one
20/// sequential write, which is what makes the cost of durability bearable on the flash a
21/// charging station actually has.
22#[derive(Serialize, Deserialize)]
23#[serde(tag = "op", rename_all = "lowercase")]
24enum Record {
25    /// A message was queued.
26    Push {
27        seq: Seq,
28        #[serde(flatten)]
29        entry: StoredCall,
30    },
31    /// A message was delivered or definitively abandoned.
32    Ack { seq: Seq },
33    /// A further transmission was attempted.
34    Attempts { seq: Seq, attempts: u32 },
35}
36
37/// A [`QueuedCall`] in a form that survives a round trip through JSON.
38#[derive(Clone, Serialize, Deserialize)]
39struct StoredCall {
40    action: String,
41    /// The payload as its raw JSON text, so it is byte-identical after a reboot — a
42    /// re-serialized payload could differ, and a `TransactionEvent` that changes shape between
43    /// attempts is one a CSMS cannot deduplicate.
44    payload: String,
45    send: bool,
46    attempts: u32,
47    transactional: bool,
48}
49
50impl StoredCall {
51    fn from_call(entry: &QueuedCall) -> Self {
52        Self {
53            action: entry.action.clone(),
54            payload: entry.payload.get().to_string(),
55            send: entry.kind == crate::message::MessageKind::Send,
56            attempts: entry.attempts,
57            transactional: entry.transactional,
58        }
59    }
60
61    fn into_call(self) -> Result<QueuedCall, StoreError> {
62        let payload = serde_json::value::RawValue::from_string(self.payload)
63            .map_err(|error| StoreError::new(format!("stored payload is not JSON: {error}")))?;
64        Ok(QueuedCall {
65            action: self.action,
66            payload,
67            kind: if self.send {
68                crate::message::MessageKind::Send
69            } else {
70                crate::message::MessageKind::Call
71            },
72            attempts: self.attempts,
73            transactional: self.transactional,
74        })
75    }
76}
77
78/// How much of the journal may be dead records before it is rewritten.
79///
80/// A station queues and acknowledges the same handful of messages for years, so without
81/// compaction the file grows without bound while the live set stays tiny.
82const COMPACT_RATIO: usize = 4;
83
84/// The smallest journal worth rewriting. Below this, compaction costs more than it saves.
85const COMPACT_FLOOR: usize = 64;
86
87/// A [`MessageStore`] that survives a power cut, for the station side.
88///
89/// An append-only journal, one line of JSON per change. A Charging Station must replay the
90/// transaction messages an outage interrupted (E04.FR.01–03, E08.FR.05–07, E12.FR.01–02),
91/// which [`MemStore`](super::MemStore) cannot.
92///
93/// ```no_run
94/// use ocpp_kit::Version;
95/// use ocpp_kit::engine::{Engine, EngineConfig, FileStore, Role};
96///
97/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
98/// // Whatever the outage interrupted is back in the queue by the time this returns.
99/// let store = FileStore::open("/var/lib/ocpp/queue.jsonl")?;
100/// let engine = Engine::with_store(
101///     EngineConfig::new(Role::ChargingStation, Version::V2_1),
102///     store,
103/// )?;
104/// # let _ = engine;
105/// # Ok(()) }
106/// ```
107///
108/// # Guarantees
109///
110/// [`push`](MessageStore::push) returns only once the record has reached the device
111/// (`sync_data`). A crash mid-append leaves a partial final line, discarded on the next open:
112/// it described a message that was never reported as queued.
113///
114/// Not concurrent — one process, one file — and no defence against a corrupt filesystem. For
115/// anything else, implement [`MessageStore`]; it is four synchronous methods.
116pub struct FileStore {
117    path: PathBuf,
118    journal: File,
119    live: BTreeMap<Seq, QueuedCall>,
120    next_seq: Seq,
121    /// Records written since the last compaction, live or not.
122    written: usize,
123}
124
125impl fmt::Debug for FileStore {
126    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
127        f.debug_struct("FileStore")
128            .field("path", &self.path)
129            .field("queued", &self.live.len())
130            .finish_non_exhaustive()
131    }
132}
133
134impl FileStore {
135    /// Opens (or creates) the journal at `path`, replaying whatever an outage left behind.
136    ///
137    /// The parent directory must exist. Replaying compacts: the file that comes back holds
138    /// exactly the messages that are still queued.
139    pub fn open(path: impl AsRef<Path>) -> Result<Self, StoreError> {
140        let path = path.as_ref().to_path_buf();
141        let (live, next_seq) = Self::replay(&path)?;
142
143        // Replaced immediately by `rewrite`; opening it here keeps the field non-optional.
144        let journal = OpenOptions::new()
145            .create(true)
146            .append(true)
147            .open(&path)
148            .map_err(|error| io(&path, &error))?;
149        let mut store = Self {
150            path,
151            journal,
152            live,
153            next_seq,
154            written: 0,
155        };
156        store.rewrite()?;
157        Ok(store)
158    }
159
160    /// The journal's path.
161    #[must_use]
162    pub fn path(&self) -> &Path {
163        &self.path
164    }
165
166    /// Rewrites the journal so it holds only what is still queued.
167    ///
168    /// Called on open and whenever the dead records outnumber the live ones; exposed because a
169    /// long-running station may want to schedule it rather than pay for it mid-transaction.
170    pub fn compact(&mut self) -> Result<(), StoreError> {
171        self.rewrite()
172    }
173
174    /// Replays a journal into the set of messages still queued.
175    ///
176    /// A line that does not parse ends the replay rather than being skipped. A journal is
177    /// append-only, so the only way to reach one is a crash mid-write, and everything after
178    /// such a line is by definition not there — treating it as a gap to step over would be
179    /// inventing an ordering the file does not have.
180    fn replay(path: &Path) -> Result<(BTreeMap<Seq, QueuedCall>, Seq), StoreError> {
181        let mut live: BTreeMap<Seq, QueuedCall> = BTreeMap::new();
182        let mut next_seq: Seq = 0;
183
184        let file = match File::open(path) {
185            Ok(file) => file,
186            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
187                return Ok((live, next_seq));
188            }
189            Err(error) => return Err(io(path, &error)),
190        };
191
192        for read in BufReader::new(file).lines() {
193            let Ok(text) = read else { break };
194            if text.trim().is_empty() {
195                continue;
196            }
197            let Ok(record) = serde_json::from_str::<Record>(&text) else {
198                break;
199            };
200            match record {
201                Record::Push { seq, entry } => {
202                    next_seq = next_seq.max(seq + 1);
203                    live.insert(seq, entry.into_call()?);
204                }
205                Record::Ack { seq } => {
206                    next_seq = next_seq.max(seq + 1);
207                    live.remove(&seq);
208                }
209                Record::Attempts { seq, attempts } => {
210                    next_seq = next_seq.max(seq + 1);
211                    if let Some(entry) = live.get_mut(&seq) {
212                        entry.attempts = attempts;
213                    }
214                }
215            }
216        }
217        Ok((live, next_seq))
218    }
219
220    /// Writes a fresh journal containing only the live entries, then swaps it in.
221    fn rewrite(&mut self) -> Result<(), StoreError> {
222        let temporary = self.path.with_extension("compacting");
223        let mut file = File::create(&temporary).map_err(|error| io(&temporary, &error))?;
224        for (seq, entry) in &self.live {
225            let record = Record::Push {
226                seq: *seq,
227                entry: StoredCall::from_call(entry),
228            };
229            write_record(&mut file, &temporary, &record)?;
230        }
231        file.sync_data().map_err(|error| io(&temporary, &error))?;
232        drop(file);
233
234        // `rename` replaces atomically, so a crash here leaves either the old journal or the
235        // new one — never a half-written mixture.
236        std::fs::rename(&temporary, &self.path).map_err(|error| io(&self.path, &error))?;
237        self.journal = OpenOptions::new()
238            .create(true)
239            .append(true)
240            .open(&self.path)
241            .map_err(|error| io(&self.path, &error))?;
242        self.written = self.live.len();
243        Ok(())
244    }
245
246    /// Appends one record and flushes it to the device.
247    fn append(&mut self, record: &Record) -> Result<(), StoreError> {
248        write_record(&mut self.journal, &self.path, record)?;
249        // The whole point: a message the engine is told is queued is on the device, not in a
250        // page cache that a power cut discards.
251        self.journal
252            .sync_data()
253            .map_err(|error| io(&self.path, &error))?;
254        self.written += 1;
255        Ok(())
256    }
257
258    /// Rewrites the journal once the dead records have come to outweigh the live ones.
259    fn compact_if_worthwhile(&mut self) -> Result<(), StoreError> {
260        if self.written >= COMPACT_FLOOR && self.written >= self.live.len() * COMPACT_RATIO {
261            self.rewrite()?;
262        }
263        Ok(())
264    }
265}
266
267impl MessageStore for FileStore {
268    fn push(&mut self, entry: &QueuedCall) -> Result<Seq, StoreError> {
269        let seq = self.next_seq;
270        self.append(&Record::Push {
271            seq,
272            entry: StoredCall::from_call(entry),
273        })?;
274        self.next_seq += 1;
275        self.live.insert(seq, entry.clone());
276        Ok(seq)
277    }
278
279    fn pending(&self) -> Result<Vec<(Seq, QueuedCall)>, StoreError> {
280        Ok(self
281            .live
282            .iter()
283            .map(|(seq, entry)| (*seq, entry.clone()))
284            .collect())
285    }
286
287    fn ack(&mut self, seq: Seq) -> Result<(), StoreError> {
288        if self.live.remove(&seq).is_none() {
289            return Ok(());
290        }
291        self.append(&Record::Ack { seq })?;
292        self.compact_if_worthwhile()
293    }
294
295    fn set_attempts(&mut self, seq: Seq, attempts: u32) -> Result<(), StoreError> {
296        let Some(entry) = self.live.get_mut(&seq) else {
297            return Ok(());
298        };
299        entry.attempts = attempts;
300        self.append(&Record::Attempts { seq, attempts })?;
301        self.compact_if_worthwhile()
302    }
303
304    fn len(&self) -> Result<usize, StoreError> {
305        Ok(self.live.len())
306    }
307}
308
309fn write_record(file: &mut File, path: &Path, record: &Record) -> Result<(), StoreError> {
310    let mut line = serde_json::to_vec(record)
311        .map_err(|error| StoreError::new(format!("could not encode a journal record: {error}")))?;
312    line.push(b'\n');
313    file.write_all(&line).map_err(|error| io(path, &error))
314}
315
316fn io(path: &Path, error: &std::io::Error) -> StoreError {
317    StoreError::new(format!("{}: {error}", path.display()))
318}
319
320#[cfg(test)]
321mod tests {
322    use super::*;
323    use crate::message::MessageKind;
324    use serde_json::value::RawValue;
325
326    fn call(seq_no: u32) -> QueuedCall {
327        QueuedCall {
328            action: "TransactionEvent".into(),
329            payload: RawValue::from_string(format!(r#"{{"seqNo":{seq_no}}}"#)).unwrap(),
330            kind: MessageKind::Call,
331            attempts: 0,
332            transactional: true,
333        }
334    }
335
336    fn temp_dir() -> std::path::PathBuf {
337        let dir = std::env::temp_dir().join(format!(
338            "ocpp-kit-store-{}-{:?}",
339            std::process::id(),
340            std::thread::current().id()
341        ));
342        std::fs::create_dir_all(&dir).unwrap();
343        dir
344    }
345
346    /// The whole reason this type exists: E04/E08/E12 require the messages an outage
347    /// interrupted to be replayed, in order, after the station comes back.
348    #[test]
349    fn a_power_cut_does_not_lose_what_was_queued() {
350        let path = temp_dir().join("replay.jsonl");
351        let _ = std::fs::remove_file(&path);
352
353        let mut store = FileStore::open(&path).unwrap();
354        let first = store.push(&call(0)).unwrap();
355        store.push(&call(1)).unwrap();
356        store.push(&call(2)).unwrap();
357        store.ack(first).unwrap();
358        store.set_attempts(1, 2).unwrap();
359        // The power goes out here: no close, no flush of our own beyond what `push` did.
360        drop(store);
361
362        let store = FileStore::open(&path).unwrap();
363        let pending = store.pending().unwrap();
364        assert_eq!(
365            pending.len(),
366            2,
367            "the acknowledged one is gone, the rest stay"
368        );
369        assert_eq!(pending[0].0, 1, "and they come back oldest first");
370        assert_eq!(pending[0].1.attempts, 2, "with the attempts they had made");
371        assert!(pending[0].1.payload.get().contains(r#""seqNo":1"#));
372        assert_eq!(pending[1].0, 2);
373        std::fs::remove_file(&path).unwrap();
374    }
375
376    /// A crash *during* an append leaves a partial line. It described a message that was never
377    /// reported as queued, so discarding it loses nothing anyone was told about — and stopping
378    /// there rather than skipping it keeps the replay from inventing an order the file does
379    /// not have.
380    #[test]
381    fn a_torn_final_record_is_discarded_rather_than_poisoning_the_queue() {
382        let path = temp_dir().join("torn.jsonl");
383        let _ = std::fs::remove_file(&path);
384
385        let mut store = FileStore::open(&path).unwrap();
386        store.push(&call(0)).unwrap();
387        store.push(&call(1)).unwrap();
388        drop(store);
389
390        // Simulate the tear: half a record, as a power cut mid-`write_all` would leave.
391        let mut file = OpenOptions::new().append(true).open(&path).unwrap();
392        file.write_all(br#"{"op":"push","seq":2,"action":"Transact"#)
393            .unwrap();
394        drop(file);
395
396        let store = FileStore::open(&path).unwrap();
397        assert_eq!(store.len().unwrap(), 2);
398        std::fs::remove_file(&path).unwrap();
399    }
400
401    /// A station queues and acknowledges the same few messages for years. Without compaction
402    /// the journal grows without bound while the live set stays tiny.
403    #[test]
404    fn the_journal_does_not_grow_without_bound() {
405        let path = temp_dir().join("compact.jsonl");
406        let _ = std::fs::remove_file(&path);
407
408        let mut store = FileStore::open(&path).unwrap();
409        for _ in 0..200 {
410            let seq = store.push(&call(0)).unwrap();
411            store.ack(seq).unwrap();
412        }
413        assert_eq!(store.len().unwrap(), 0);
414
415        let lines = std::fs::read_to_string(&path).unwrap().lines().count();
416        assert!(
417            lines <= COMPACT_FLOOR * 2,
418            "400 records should have been compacted away, {lines} lines remain"
419        );
420
421        // And the compacted file still replays correctly.
422        store.push(&call(9)).unwrap();
423        drop(store);
424        let store = FileStore::open(&path).unwrap();
425        assert_eq!(store.len().unwrap(), 1);
426        std::fs::remove_file(&path).unwrap();
427    }
428
429    #[test]
430    fn a_send_keeps_its_kind_across_a_reboot() {
431        let path = temp_dir().join("kinds.jsonl");
432        let _ = std::fs::remove_file(&path);
433
434        let mut store = FileStore::open(&path).unwrap();
435        let mut send = call(0);
436        send.kind = MessageKind::Send;
437        store.push(&send).unwrap();
438        drop(store);
439
440        let store = FileStore::open(&path).unwrap();
441        assert_eq!(store.pending().unwrap()[0].1.kind, MessageKind::Send);
442        std::fs::remove_file(&path).unwrap();
443    }
444}