omry_db/
model.rs

1//! Data structures for flora's SQLite database.
2use chrono::{DateTime, FixedOffset};
3use omry_archiving::{ClientDateTimeRecord, Document, Marshal, Record, RecordParams, ToRecord};
4use std::fmt;
5use thiserror::Error;
6
7/// Data types for the database columns.
8#[allow(dead_code)]
9#[allow(clippy::missing_docs_in_private_items)]
10#[derive(Debug)]
11enum ColumnType {
12    Boolean,
13    Integer,
14    Real,
15    Text,
16    Blob,
17}
18
19impl fmt::Display for ColumnType {
20    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
21        let s = match self {
22            Self::Boolean => "BOOLEAN",
23            Self::Integer => "INTEGER",
24            Self::Real => "REAL",
25            Self::Text => "TEXT",
26            Self::Blob => "BLOB",
27        };
28        write!(fmt, "{s}")
29    }
30}
31
32/// A builder for [`ColumnDefinition`].
33struct ColumnDefinitionBuilder {
34    /// The column name.
35    name: String,
36
37    /// The data type of the column.
38    column_type: ColumnType,
39
40    /// Whether the column can be nullable.
41    not_null: Option<bool>,
42
43    /// Whether the column should be used as a SQLite primary key.
44    primary_key: Option<bool>,
45}
46
47impl ColumnDefinitionBuilder {
48    /// Builds the [`ColumnDefinition`] with the parameters set in this builder's fields.
49    fn build(self) -> ColumnDefinition {
50        ColumnDefinition {
51            name: self.name,
52            column_type: self.column_type,
53            not_null: self.not_null.unwrap_or_default(),
54            primary_key: self.primary_key.unwrap_or_default(),
55        }
56    }
57
58    /// Sets the value [`ColumnDefinitionBuilder.not_null`] to the given value.
59    fn not_null(mut self, not_null: bool) -> Self {
60        self.not_null = Some(not_null);
61        self
62    }
63
64    /// Sets the value [`ColumnDefinitionBuilder.not_null`] to the given value.
65    fn primary_key(mut self, primary_key: bool) -> Self {
66        self.primary_key = Some(primary_key);
67        self
68    }
69}
70
71/// Structure for programmatically describing a SQLite column.
72#[derive(Debug)]
73struct ColumnDefinition {
74    /// The column name.
75    name: String,
76
77    /// The data type of the column.
78    column_type: ColumnType,
79
80    /// Whether the column can be nullable.
81    not_null: bool,
82
83    /// Whether the column should be used as a SQLite primary key.
84    primary_key: bool,
85}
86
87impl ColumnDefinition {
88    /// Returns the builder for defining a column programmatically.
89    fn builder(name: String, column_type: ColumnType) -> ColumnDefinitionBuilder {
90        ColumnDefinitionBuilder {
91            name,
92            column_type,
93            not_null: None,
94            primary_key: None,
95        }
96    }
97}
98
99/// Definition of a SQLite table schema.
100#[derive(Debug)]
101pub(crate) struct Schema {
102    /// The name of the table that this schema describes.
103    pub(crate) table_name: String,
104
105    /// The definitions of the columns in this table.
106    columns: Vec<ColumnDefinition>,
107}
108
109impl fmt::Display for Schema {
110    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
111        writeln!(f, "{} (", self.table_name)?;
112
113        let mut parts = vec![];
114        let mut s = String::new();
115
116        for column in &self.columns {
117            s.push_str(format!("\t{} {}", column.name, column.column_type).as_str());
118
119            if column.primary_key {
120                s.push_str(" PRIMARY KEY");
121            }
122
123            // "According to the SQL standard,
124            // PRIMARY KEY should always imply NOT NULL.
125            // Unfortunately, due to a bug in some early versions, this is not the case in SQLite."
126            // https://www.sqlite.org/lang_createtable.html section 3.5
127            if column.primary_key || column.not_null {
128                s.push_str(" NOT NULL");
129            }
130
131            parts.push(s.clone());
132            s.clear();
133        }
134
135        let s = parts.join(",\n");
136        write!(f, "{s}\n)")
137    }
138}
139
140/// Public wrappere for database API errors.
141#[derive(Debug, Error)]
142#[error(transparent)]
143pub struct DbRecordError(#[from] DbRecordErrorRepr);
144
145/// Private representation of database API errors.
146#[derive(Debug, Error)]
147enum DbRecordErrorRepr {
148    /// Errors encountered when deserializing the web page.
149    #[error("Error (de)serializing the web page: {0}")]
150    WebArchive(String),
151}
152
153/// SQLite representation of a flora `Record`.
154#[derive(Debug)]
155pub struct DbRecord {
156    /// The SQLite row ID of this record.
157    pub(crate) id: Option<i64>,
158
159    /// The original URL that corresponds to this record.
160    pub(crate) url: String,
161
162    /// The client date and time, when the client saved the content to be archived.
163    pub(crate) client_datetime: DateTime<FixedOffset>,
164
165    /// Raw date time, in case we couldn't parse it.
166    pub(crate) client_datetime_raw: Option<String>,
167
168    /// The Omry server timestamp, when Omry first imported this record.
169    pub(crate) timestamp_flora: i64,
170
171    /// The page title that the client provided for this record.
172    pub(crate) title: String,
173
174    /// The language we extracted from this record's metadata, if any.
175    pub(crate) language: Option<String>,
176
177    /// The binary serialized web content in this record.
178    pub(crate) web_archive_serialized: Vec<u8>,
179}
180
181impl DbRecord {
182    /// The database schema.
183    pub(crate) fn schema() -> Schema {
184        let columns = vec![
185            ColumnDefinition::builder("id".to_string(), ColumnType::Integer)
186                .primary_key(true)
187                .build(),
188            ColumnDefinition::builder("url".to_string(), ColumnType::Text)
189                .not_null(true)
190                .build(),
191            ColumnDefinition::builder("client_datetime".to_string(), ColumnType::Text)
192                .not_null(true)
193                .build(),
194            ColumnDefinition::builder("client_datetime_raw".to_string(), ColumnType::Text).build(),
195            ColumnDefinition::builder("timestamp_flora".to_string(), ColumnType::Integer)
196                .not_null(true)
197                .build(),
198            ColumnDefinition::builder("title".to_string(), ColumnType::Text)
199                .not_null(true)
200                .build(),
201            ColumnDefinition::builder("language".to_string(), ColumnType::Text).build(),
202            ColumnDefinition::builder("web_archive_serialized".to_string(), ColumnType::Blob)
203                .not_null(true)
204                .build(),
205        ];
206        Schema {
207            table_name: "records".to_string(),
208            columns,
209        }
210    }
211
212    /// The SQLite row ID of this record.
213    ///
214    /// This should only return `None` if the record
215    /// hasn't yet been inserted into the database.
216    #[must_use]
217    pub fn id(&self) -> Option<i64> {
218        self.id
219    }
220}
221
222impl TryFrom<Record> for DbRecord {
223    type Error = DbRecordError;
224
225    fn try_from(value: Record) -> Result<Self, Self::Error> {
226        let id = value.id();
227        let url = value.url().to_string();
228        let title = value.title().to_string();
229
230        let (client_datetime, client_datetime_raw) = match value.client_datetime() {
231            ClientDateTimeRecord::Parsed(datetime) => (*datetime, None),
232            ClientDateTimeRecord::Interpolated { value, raw } => (*value, Some(raw.clone())),
233        };
234        let timestamp_flora = value.timestamp_archived();
235
236        let language = value.language().map(|lang| {
237            lang.primary()
238                .unwrap_or_else(|error| error.raw_record().to_string())
239        });
240        let web_archive_serialized = value.document().to_byte_vec().map_err(|error| {
241            DbRecordErrorRepr::WebArchive(format!("Couldn't serialize the archive: {error}"))
242        })?;
243
244        Ok(Self {
245            id,
246            url,
247            client_datetime,
248            client_datetime_raw,
249            timestamp_flora,
250            title,
251            language,
252            web_archive_serialized,
253        })
254    }
255}
256
257impl ToRecord for DbRecord {
258    type Error = DbRecordError;
259
260    fn to_record(&self) -> Result<Record, Self::Error> {
261        let document = Document::from_bytes(&self.web_archive_serialized)
262            .map_err(|error| DbRecordErrorRepr::WebArchive(error.to_string()))?;
263
264        let params = RecordParams {
265            id: self.id,
266            url: self.url.clone(),
267            title: self.title.clone(),
268            client_datetime: self.client_datetime.to_rfc3339(),
269            timestamp_flora: Some(self.timestamp_flora),
270            language: self.language.clone(),
271            document,
272        };
273
274        Ok(Record::new(params))
275    }
276}
277
278/// Two instances of `DbRecord` are partially equal if all their fields except `id` are equal,
279/// **and** either:
280/// - Both of their `id` fields are equal (including if both are `None`), or
281/// - One of the records' `id` fields is `None`.
282///
283/// We consider records where one `DbRecord` is `Some` and the other is `None` to be
284/// partially equal to account for a situation where a new record was just inserted into
285/// the database, and we are comparing it to the same record that we got back from the database.
286impl PartialEq for DbRecord {
287    fn eq(&self, other: &Self) -> bool {
288        // Short-circuit if the `id` fields are actually different.
289        let ids_considered_equal = if let (Some(this_id), Some(other_id)) = (self.id, other.id) {
290            this_id == other_id
291        } else {
292            // One or both `id`s are `None`
293            true
294        };
295
296        self.url == other.url
297            && self.client_datetime == other.client_datetime
298            && self.client_datetime_raw == other.client_datetime_raw
299            && self.timestamp_flora == other.timestamp_flora
300            && self.title == other.title
301            && self.language == other.language
302            && self.web_archive_serialized == other.web_archive_serialized
303            && ids_considered_equal
304    }
305}
306
307#[cfg(test)]
308pub(crate) mod tests {
309    use super::*;
310    use omry_archiving::{Document, Marshal, record::RecordParams};
311    use pretty_assertions::assert_eq;
312    use std::fs::File;
313    use std::io::prelude::*;
314
315    const TEST_ARCHIVE_PATH: &str = "../test/data/example.bincode";
316
317    fn load_test_archive_holder() -> anyhow::Result<Document> {
318        let mut file = File::open(TEST_ARCHIVE_PATH)?;
319        let mut buf = vec![];
320        file.read_to_end(&mut buf)?;
321
322        let archive_holder = Document::from_bytes(&buf)?;
323        Ok(archive_holder)
324    }
325
326    pub(crate) fn test_record_params() -> anyhow::Result<RecordParams> {
327        let document = load_test_archive_holder()?;
328
329        let url = "https://example.org/".to_string();
330        let title = "Example Domain";
331        let client_datetime = "2024-10-11T13:49:46-05:00";
332
333        #[allow(clippy::unreadable_literal)]
334        let timestamp_flora = Some(1728695243i64);
335        let language = Some("en".to_string());
336
337        Ok(RecordParams {
338            id: None,
339            url,
340            title: title.to_string(),
341            client_datetime: client_datetime.to_string(),
342            timestamp_flora,
343            language,
344            document,
345        })
346    }
347
348    #[allow(clippy::expect_used)]
349    #[test]
350    fn can_convert_record_to_db_record() -> anyhow::Result<()> {
351        let record = Record::new(test_record_params()?);
352        let same_record = Record::new(test_record_params()?);
353        let primary_language_tag = same_record.language().map(|lang| {
354            lang.primary()
355                .expect("Couldn't compute the primary language tag")
356        });
357
358        let db_record: DbRecord = record.try_into()?;
359        assert_eq!(db_record.title, same_record.title());
360        assert_eq!(db_record.language, primary_language_tag);
361        Ok(())
362    }
363
364    #[test]
365    fn can_get_schema() {
366        let schema = DbRecord::schema();
367        insta::assert_snapshot!(&schema);
368    }
369
370    #[test]
371    fn same_record_is_equal() -> anyhow::Result<()> {
372        let record: DbRecord = Record::new(test_record_params()?).try_into()?;
373        assert_eq!(record, record);
374        Ok(())
375    }
376
377    #[test]
378    fn different_records_are_not_equal() -> anyhow::Result<()> {
379        let record = Record::new(test_record_params()?);
380        let mut other_record_params = test_record_params()?;
381        other_record_params.url = "https://example.com".into();
382        let other_record = Record::new(other_record_params);
383
384        let record: DbRecord = record.try_into()?;
385        let other_record = other_record.try_into()?;
386        assert_ne!(record, other_record);
387        Ok(())
388    }
389
390    #[test]
391    fn record_can_roundtrip_through_db_record() -> anyhow::Result<()> {
392        let record = Record::new(test_record_params()?);
393        let db_record: DbRecord = record.try_into()?;
394        let roundtripped_record = db_record.to_record()?;
395
396        let same_record = Record::new(test_record_params()?);
397        assert_eq!(same_record, roundtripped_record);
398        Ok(())
399    }
400}
omry_db/model.rs

omry_db/
model.rs
