omry_db/
model.rs

1//! Data structures for flora's SQLite database.
2use chrono::{DateTime, FixedOffset};
3use omry_archiving::{ClientDateTimeRecord, Document, Marshal, Record, RecordParams, ToRecord};
4use std::fmt;
5use thiserror::Error;
6
7/// Data types for the database columns.
8#[allow(dead_code)]
9#[allow(clippy::missing_docs_in_private_items)]
10#[derive(Debug)]
11enum ColumnType {
12    Boolean,
13    Integer,
14    Real,
15    Text,
16    Blob,
17}
18
19impl fmt::Display for ColumnType {
20    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
21        let s = match self {
22            Self::Boolean => "BOOLEAN",
23            Self::Integer => "INTEGER",
24            Self::Real => "REAL",
25            Self::Text => "TEXT",
26            Self::Blob => "BLOB",
27        };
28        write!(fmt, "{s}")
29    }
30}
31
32/// A builder for [`ColumnDefinition`].
33struct ColumnDefinitionBuilder {
34    /// The column name.
35    name: String,
36
37    /// The data type of the column.
38    column_type: ColumnType,
39
40    /// Whether the column can be nullable.
41    not_null: Option<bool>,
42
43    /// Whether the column should be used as a SQLite primary key.
44    primary_key: Option<bool>,
45}
46
47impl ColumnDefinitionBuilder {
48    /// Builds the [`ColumnDefinition`] with the parameters set in this builder's fields.
49    fn build(self) -> ColumnDefinition {
50        ColumnDefinition {
51            name: self.name,
52            column_type: self.column_type,
53            not_null: self.not_null.unwrap_or_default(),
54            primary_key: self.primary_key.unwrap_or_default(),
55        }
56    }
57
58    /// Sets the value [`ColumnDefinitionBuilder.not_null`] to the given value.
59    fn not_null(mut self, not_null: bool) -> Self {
60        self.not_null = Some(not_null);
61        self
62    }
63
64    /// Sets the value [`ColumnDefinitionBuilder.not_null`] to the given value.
65    fn primary_key(mut self, primary_key: bool) -> Self {
66        self.primary_key = Some(primary_key);
67        self
68    }
69}
70
71/// Structure for programmatically describing a SQLite column.
72#[derive(Debug)]
73struct ColumnDefinition {
74    /// The column name.
75    name: String,
76
77    /// The data type of the column.
78    column_type: ColumnType,
79
80    /// Whether the column can be nullable.
81    not_null: bool,
82
83    /// Whether the column should be used as a SQLite primary key.
84    primary_key: bool,
85}
86
87impl ColumnDefinition {
88    /// Returns the builder for defining a column programmatically.
89    fn builder(name: String, column_type: ColumnType) -> ColumnDefinitionBuilder {
90        ColumnDefinitionBuilder {
91            name,
92            column_type,
93            not_null: None,
94            primary_key: None,
95        }
96    }
97}
98
99/// Definition of a SQLite table schema.
100#[derive(Debug)]
101pub(crate) struct Schema {
102    /// The name of the table that this schema describes.
103    table_name: String,
104
105    /// The definitions of the columns in this table.
106    columns: Vec<ColumnDefinition>,
107}
108
109impl Schema {
110    /// Returns the name of this schema's table.
111    pub(crate) fn table_name(&self) -> &str {
112        &self.table_name
113    }
114}
115
116impl fmt::Display for Schema {
117    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
118        writeln!(f, "{} (", self.table_name)?;
119
120        let mut parts = vec![];
121        let mut s = String::new();
122
123        for column in &self.columns {
124            s.push_str(format!("\t{} {}", column.name, column.column_type).as_str());
125
126            if column.primary_key {
127                s.push_str(" PRIMARY KEY");
128            }
129
130            // "According to the SQL standard,
131            // PRIMARY KEY should always imply NOT NULL.
132            // Unfortunately, due to a bug in some early versions, this is not the case in SQLite."
133            // https://www.sqlite.org/lang_createtable.html section 3.5
134            if column.primary_key || column.not_null {
135                s.push_str(" NOT NULL");
136            }
137
138            parts.push(s.clone());
139            s.clear();
140        }
141
142        let s = parts.join(",\n");
143        write!(f, "{s}\n)")
144    }
145}
146
147/// Public wrappere for database API errors.
148#[derive(Debug, Error)]
149#[error(transparent)]
150pub struct DbRecordError(#[from] DbRecordErrorRepr);
151
152/// Private representation of database API errors.
153#[derive(Debug, Error)]
154enum DbRecordErrorRepr {
155    /// Errors encountered when deserializing the web page.
156    #[error("Error (de)serializing the web page: {0}")]
157    WebArchive(String),
158}
159
160/// SQLite representation of a flora `Record`.
161#[derive(Debug)]
162pub struct DbRecord {
163    /// The SQLite row ID of this record.
164    pub(crate) id: Option<i64>,
165
166    /// The original URL that corresponds to this record.
167    pub(crate) url: String,
168
169    /// The client date and time, when the client saved the content to be archived.
170    pub(crate) client_datetime: DateTime<FixedOffset>,
171
172    /// Raw date time, in case we couldn't parse it.
173    pub(crate) client_datetime_raw: Option<String>,
174
175    /// The Omry server timestamp, when Omry first imported this record.
176    pub(crate) timestamp_flora: i64,
177
178    /// The page title that the client provided for this record.
179    pub(crate) title: String,
180
181    /// The language we extracted from this record's metadata, if any.
182    pub(crate) language: Option<String>,
183
184    /// The binary serialized web content in this record.
185    pub(crate) web_archive_serialized: Vec<u8>,
186}
187
188impl DbRecord {
189    /// The database schema.
190    pub(crate) fn schema() -> Schema {
191        let columns = vec![
192            ColumnDefinition::builder("id".to_string(), ColumnType::Integer)
193                .primary_key(true)
194                .build(),
195            ColumnDefinition::builder("url".to_string(), ColumnType::Text)
196                .not_null(true)
197                .build(),
198            ColumnDefinition::builder("client_datetime".to_string(), ColumnType::Text)
199                .not_null(true)
200                .build(),
201            ColumnDefinition::builder("client_datetime_raw".to_string(), ColumnType::Text).build(),
202            ColumnDefinition::builder("timestamp_flora".to_string(), ColumnType::Integer)
203                .not_null(true)
204                .build(),
205            ColumnDefinition::builder("title".to_string(), ColumnType::Text)
206                .not_null(true)
207                .build(),
208            ColumnDefinition::builder("language".to_string(), ColumnType::Text).build(),
209            ColumnDefinition::builder("web_archive_serialized".to_string(), ColumnType::Blob)
210                .not_null(true)
211                .build(),
212        ];
213        Schema {
214            table_name: "records".to_string(),
215            columns,
216        }
217    }
218
219    /// The SQLite row ID of this record.
220    ///
221    /// This should only return `None` if the record
222    /// hasn't yet been inserted into the database.
223    #[must_use]
224    pub fn id(&self) -> Option<i64> {
225        self.id
226    }
227}
228
229impl TryFrom<Record> for DbRecord {
230    type Error = DbRecordError;
231
232    fn try_from(value: Record) -> Result<Self, Self::Error> {
233        let id = value.id();
234        let url = value.url().to_string();
235        let title = value.title().to_string();
236
237        let (client_datetime, client_datetime_raw) = match value.client_datetime() {
238            ClientDateTimeRecord::Parsed(datetime) => (*datetime, None),
239            ClientDateTimeRecord::Interpolated { value, raw } => (*value, Some(raw.clone())),
240        };
241        let timestamp_flora = value.timestamp_archived();
242
243        let language = value.language().map(|lang| {
244            lang.primary()
245                .unwrap_or_else(|error| error.raw_record().to_string())
246        });
247        let web_archive_serialized = value.document().to_byte_vec().map_err(|error| {
248            DbRecordErrorRepr::WebArchive(format!("Couldn't serialize the archive: {error}"))
249        })?;
250
251        Ok(Self {
252            id,
253            url,
254            client_datetime,
255            client_datetime_raw,
256            timestamp_flora,
257            title,
258            language,
259            web_archive_serialized,
260        })
261    }
262}
263
264impl ToRecord for DbRecord {
265    type Error = DbRecordError;
266
267    fn to_record(&self) -> Result<Record, Self::Error> {
268        let document = Document::from_bytes(&self.web_archive_serialized)
269            .map_err(|error| DbRecordErrorRepr::WebArchive(error.to_string()))?;
270
271        let params = RecordParams {
272            id: self.id,
273            url: self.url.clone(),
274            title: self.title.clone(),
275            client_datetime: self.client_datetime.to_rfc3339(),
276            timestamp_flora: Some(self.timestamp_flora),
277            language: self.language.clone(),
278            document,
279        };
280
281        Ok(Record::new(params))
282    }
283}
284
285/// Two instances of `DbRecord` are partially equal if all their fields except `id` are equal,
286/// **and** either:
287/// - Both of their `id` fields are equal (including if both are `None`), or
288/// - One of the records' `id` fields is `None`.
289///
290/// We consider records where one `DbRecord` is `Some` and the other is `None` to be
291/// partially equal to account for a situation where a new record was just inserted into
292/// the database, and we are comparing it to the same record that we got back from the database.
293impl PartialEq for DbRecord {
294    fn eq(&self, other: &Self) -> bool {
295        // Short-circuit if the `id` fields are actually different.
296        let ids_considered_equal = if let (Some(this_id), Some(other_id)) = (self.id, other.id) {
297            this_id == other_id
298        } else {
299            // One or both `id`s are `None`
300            true
301        };
302
303        self.url == other.url
304            && self.client_datetime == other.client_datetime
305            && self.client_datetime_raw == other.client_datetime_raw
306            && self.timestamp_flora == other.timestamp_flora
307            && self.title == other.title
308            && self.language == other.language
309            && self.web_archive_serialized == other.web_archive_serialized
310            && ids_considered_equal
311    }
312}
313
314#[cfg(test)]
315pub(crate) mod tests {
316    use super::*;
317    use omry_archiving::{Document, Marshal, record::RecordParams};
318    use pretty_assertions::assert_eq;
319    use std::fs::File;
320    use std::io::prelude::*;
321
322    const TEST_ARCHIVE_PATH: &str = "../test/data/example.bincode";
323
324    fn load_test_archive_holder() -> anyhow::Result<Document> {
325        let mut file = File::open(TEST_ARCHIVE_PATH)?;
326        let mut buf = vec![];
327        file.read_to_end(&mut buf)?;
328
329        let archive_holder = Document::from_bytes(&buf)?;
330        Ok(archive_holder)
331    }
332
333    pub(crate) fn test_record_params() -> anyhow::Result<RecordParams> {
334        let document = load_test_archive_holder()?;
335
336        let url = "https://example.org/".to_string();
337        let title = "Example Domain";
338        let client_datetime = "2024-10-11T13:49:46-05:00";
339
340        #[allow(clippy::unreadable_literal)]
341        let timestamp_flora = Some(1728695243i64);
342        let language = Some("en".to_string());
343
344        Ok(RecordParams {
345            id: None,
346            url,
347            title: title.to_string(),
348            client_datetime: client_datetime.to_string(),
349            timestamp_flora,
350            language,
351            document,
352        })
353    }
354
355    #[allow(clippy::expect_used)]
356    #[test]
357    fn can_convert_record_to_db_record() -> anyhow::Result<()> {
358        let record = Record::new(test_record_params()?);
359        let same_record = Record::new(test_record_params()?);
360        let primary_language_tag = same_record.language().map(|lang| {
361            lang.primary()
362                .expect("Couldn't compute the primary language tag")
363        });
364
365        let db_record: DbRecord = record.try_into()?;
366        assert_eq!(db_record.title, same_record.title());
367        assert_eq!(db_record.language, primary_language_tag);
368        Ok(())
369    }
370
371    #[test]
372    fn can_get_schema() {
373        let schema = DbRecord::schema();
374        insta::assert_snapshot!(&schema);
375    }
376
377    #[test]
378    fn same_record_is_equal() -> anyhow::Result<()> {
379        let record: DbRecord = Record::new(test_record_params()?).try_into()?;
380        assert_eq!(record, record);
381        Ok(())
382    }
383
384    #[test]
385    fn different_records_are_not_equal() -> anyhow::Result<()> {
386        let record = Record::new(test_record_params()?);
387        let mut other_record_params = test_record_params()?;
388        other_record_params.url = "https://example.com".into();
389        let other_record = Record::new(other_record_params);
390
391        let record: DbRecord = record.try_into()?;
392        let other_record = other_record.try_into()?;
393        assert_ne!(record, other_record);
394        Ok(())
395    }
396
397    #[test]
398    fn record_can_roundtrip_through_db_record() -> anyhow::Result<()> {
399        let record = Record::new(test_record_params()?);
400        let db_record: DbRecord = record.try_into()?;
401        let roundtripped_record = db_record.to_record()?;
402
403        let same_record = Record::new(test_record_params()?);
404        assert_eq!(same_record, roundtripped_record);
405        Ok(())
406    }
407
408    #[test]
409    fn can_get_table_name_from_schema() {
410        let schema = DbRecord::schema();
411        insta::assert_snapshot!(schema.table_name());
412    }
413}
omry_db/model.rs

omry_db/
model.rs
