omry_db/
model.rs

1//! Data structures for flora's SQLite database.
2use chrono::{DateTime, FixedOffset};
3use omry_archiving::record::LanguageRecord;
4use omry_archiving::{
5    ClientDateTimeRecord, Document, Marshal, Record, RecordMeta, RecordParams, ToRecord,
6};
7use std::fmt;
8use thiserror::Error;
9
10/// Data types for the database columns.
11#[allow(dead_code)]
12#[allow(clippy::missing_docs_in_private_items)]
13#[derive(Debug)]
14enum ColumnType {
15    Boolean,
16    Integer,
17    Real,
18    Text,
19    Blob,
20}
21
22impl fmt::Display for ColumnType {
23    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
24        let s = match self {
25            Self::Boolean => "BOOLEAN",
26            Self::Integer => "INTEGER",
27            Self::Real => "REAL",
28            Self::Text => "TEXT",
29            Self::Blob => "BLOB",
30        };
31        write!(fmt, "{s}")
32    }
33}
34
35/// A builder for [`ColumnDefinition`].
36struct ColumnDefinitionBuilder {
37    /// The column name.
38    name: String,
39
40    /// The data type of the column.
41    column_type: ColumnType,
42
43    /// Whether the column can be nullable.
44    not_null: Option<bool>,
45
46    /// Whether the column should be used as a SQLite primary key.
47    primary_key: Option<bool>,
48}
49
50impl ColumnDefinitionBuilder {
51    /// Builds the [`ColumnDefinition`] with the parameters set in this builder's fields.
52    fn build(self) -> ColumnDefinition {
53        ColumnDefinition {
54            name: self.name,
55            column_type: self.column_type,
56            not_null: self.not_null.unwrap_or_default(),
57            primary_key: self.primary_key.unwrap_or_default(),
58        }
59    }
60
61    /// Sets the value [`ColumnDefinitionBuilder.not_null`] to the given value.
62    fn not_null(mut self, not_null: bool) -> Self {
63        self.not_null = Some(not_null);
64        self
65    }
66
67    /// Sets the value [`ColumnDefinitionBuilder.not_null`] to the given value.
68    fn primary_key(mut self, primary_key: bool) -> Self {
69        self.primary_key = Some(primary_key);
70        self
71    }
72}
73
74/// Structure for programmatically describing a SQLite column.
75#[derive(Debug)]
76struct ColumnDefinition {
77    /// The column name.
78    name: String,
79
80    /// The data type of the column.
81    column_type: ColumnType,
82
83    /// Whether the column can be nullable.
84    not_null: bool,
85
86    /// Whether the column should be used as a SQLite primary key.
87    primary_key: bool,
88}
89
90impl ColumnDefinition {
91    /// Returns the builder for defining a column programmatically.
92    fn builder(name: String, column_type: ColumnType) -> ColumnDefinitionBuilder {
93        ColumnDefinitionBuilder {
94            name,
95            column_type,
96            not_null: None,
97            primary_key: None,
98        }
99    }
100}
101
102/// Definition of a SQLite table schema.
103#[derive(Debug)]
104pub(crate) struct Schema {
105    /// The name of the table that this schema describes.
106    table_name: String,
107
108    /// The definitions of the columns in this table.
109    columns: Vec<ColumnDefinition>,
110}
111
112impl Schema {
113    /// Returns the name of this schema's table.
114    pub(crate) fn table_name(&self) -> &str {
115        &self.table_name
116    }
117}
118
119impl fmt::Display for Schema {
120    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
121        writeln!(f, "{} (", self.table_name)?;
122
123        let mut parts = vec![];
124        let mut s = String::new();
125
126        for column in &self.columns {
127            s.push_str(format!("\t{} {}", column.name, column.column_type).as_str());
128
129            if column.primary_key {
130                s.push_str(" PRIMARY KEY");
131            }
132
133            // "According to the SQL standard,
134            // PRIMARY KEY should always imply NOT NULL.
135            // Unfortunately, due to a bug in some early versions, this is not the case in SQLite."
136            // https://www.sqlite.org/lang_createtable.html section 3.5
137            if column.primary_key || column.not_null {
138                s.push_str(" NOT NULL");
139            }
140
141            parts.push(s.clone());
142            s.clear();
143        }
144
145        let s = parts.join(",\n");
146        write!(f, "{s}\n)")
147    }
148}
149
150/// Public wrappere for database API errors.
151#[derive(Debug, Error)]
152#[error(transparent)]
153pub struct DbRecordError(#[from] DbRecordErrorRepr);
154
155/// Private representation of database API errors.
156#[derive(Debug, Error)]
157enum DbRecordErrorRepr {
158    /// Errors encountered when deserializing the web page.
159    #[error("Error (de)serializing the web page: {0}")]
160    WebArchive(String),
161}
162
163/// SQLite representation of a flora `Record`.
164#[derive(Debug)]
165pub struct DbRecord {
166    /// The SQLite row ID of this record.
167    pub(crate) id: Option<i64>,
168
169    /// The original URL that corresponds to this record.
170    pub(crate) url: String,
171
172    /// The client date and time, when the client saved the content to be archived.
173    pub(crate) client_datetime: DateTime<FixedOffset>,
174
175    /// Raw date time, in case we couldn't parse it.
176    pub(crate) client_datetime_raw: Option<String>,
177
178    /// The Omry server timestamp, when Omry first imported this record.
179    pub(crate) timestamp_flora: i64,
180
181    /// The page title that the client provided for this record.
182    pub(crate) title: String,
183
184    /// The language we extracted from this record's metadata, if any.
185    pub(crate) language: Option<String>,
186
187    /// The binary serialized web content in this record.
188    pub(crate) web_archive_serialized: Vec<u8>,
189}
190
191impl DbRecord {
192    /// The database schema.
193    pub(crate) fn schema() -> Schema {
194        let columns = vec![
195            ColumnDefinition::builder("id".to_string(), ColumnType::Integer)
196                .primary_key(true)
197                .build(),
198            ColumnDefinition::builder("url".to_string(), ColumnType::Text)
199                .not_null(true)
200                .build(),
201            ColumnDefinition::builder("client_datetime".to_string(), ColumnType::Text)
202                .not_null(true)
203                .build(),
204            ColumnDefinition::builder("client_datetime_raw".to_string(), ColumnType::Text).build(),
205            ColumnDefinition::builder("timestamp_flora".to_string(), ColumnType::Integer)
206                .not_null(true)
207                .build(),
208            ColumnDefinition::builder("title".to_string(), ColumnType::Text)
209                .not_null(true)
210                .build(),
211            ColumnDefinition::builder("language".to_string(), ColumnType::Text).build(),
212            ColumnDefinition::builder("web_archive_serialized".to_string(), ColumnType::Blob)
213                .not_null(true)
214                .build(),
215        ];
216        Schema {
217            table_name: "records".to_string(),
218            columns,
219        }
220    }
221
222    /// The SQLite row ID of this record.
223    ///
224    /// This should only return `None` if the record
225    /// hasn't yet been inserted into the database.
226    #[must_use]
227    pub fn id(&self) -> Option<i64> {
228        self.id
229    }
230}
231
232impl TryFrom<Record> for DbRecord {
233    type Error = DbRecordError;
234
235    fn try_from(value: Record) -> Result<Self, Self::Error> {
236        let id = value.id();
237        let url = value.url().to_string();
238        let title = value.title().to_string();
239
240        let (client_datetime, client_datetime_raw) = match value.client_datetime() {
241            ClientDateTimeRecord::Parsed(datetime) => (*datetime, None),
242            ClientDateTimeRecord::Interpolated { value, raw } => (*value, Some(raw.clone())),
243        };
244
245        let timestamp_flora = value.timestamp_archived();
246
247        let language = value.language().map(|lang| {
248            lang.primary()
249                .unwrap_or_else(|error| error.raw_data().to_string())
250        });
251
252        let web_archive_serialized = value.document().to_byte_vec().map_err(|error| {
253            DbRecordErrorRepr::WebArchive(format!("Couldn't serialize the archive: {error}"))
254        })?;
255
256        Ok(Self {
257            id,
258            url,
259            client_datetime,
260            client_datetime_raw,
261            timestamp_flora,
262            title,
263            language,
264            web_archive_serialized,
265        })
266    }
267}
268
269impl ToRecord for DbRecord {
270    type Error = DbRecordError;
271
272    fn to_record(&self) -> Result<Record, Self::Error> {
273        let document = Document::from_bytes(&self.web_archive_serialized)
274            .map_err(|error| DbRecordErrorRepr::WebArchive(error.to_string()))?;
275
276        let params = RecordParams {
277            id: self.id,
278            url: self.url.clone(),
279            title: self.title.clone(),
280            client_datetime: self.client_datetime.to_rfc3339(),
281            timestamp_flora: Some(self.timestamp_flora),
282            language: self.language.clone(),
283            document,
284        };
285
286        Ok(Record::new(params))
287    }
288}
289
290/// Two instances of `DbRecord` are partially equal if all their fields except `id` are equal,
291/// **and** either:
292/// - Both of their `id` fields are equal (including if both are `None`), or
293/// - One of the records' `id` fields is `None`.
294///
295/// We consider records where one `DbRecord` is `Some` and the other is `None` to be
296/// partially equal to account for a situation where a new record was just inserted into
297/// the database, and we are comparing it to the same record that we got back from the database.
298impl PartialEq for DbRecord {
299    fn eq(&self, other: &Self) -> bool {
300        // Short-circuit if the `id` fields are actually different.
301        let ids_considered_equal = if let (Some(this_id), Some(other_id)) = (self.id, other.id) {
302            this_id == other_id
303        } else {
304            // One or both `id`s are `None`
305            true
306        };
307
308        self.url == other.url
309            && self.client_datetime == other.client_datetime
310            && self.client_datetime_raw == other.client_datetime_raw
311            && self.timestamp_flora == other.timestamp_flora
312            && self.title == other.title
313            && self.language == other.language
314            && self.web_archive_serialized == other.web_archive_serialized
315            && ids_considered_equal
316    }
317}
318
319/// Counterpart to [`RecordMeta`] for data as it is stored in SQLite.
320#[derive(Debug, PartialEq)]
321pub struct DbRecordMeta {
322    /// The SQLite row ID of this record.
323    pub(crate) id: i64,
324
325    /// The original URL that corresponds to this record.
326    pub(crate) url: String,
327
328    /// The client date and time, when the client saved the content to be archived.
329    pub(crate) client_datetime: DateTime<FixedOffset>,
330
331    /// The Omry server timestamp, when Omry first imported this record.
332    pub(crate) timestamp_flora: i64,
333
334    /// The page title that the client provided for this record.
335    pub(crate) title: String,
336
337    /// The language we extracted from this record's metadata, if any.
338    pub(crate) language: Option<String>,
339}
340
341impl From<DbRecordMeta> for RecordMeta {
342    fn from(record: DbRecordMeta) -> Self {
343        let DbRecordMeta {
344            id,
345            url,
346            client_datetime,
347            timestamp_flora,
348            title,
349            language,
350        } = record;
351
352        let client_datetime = ClientDateTimeRecord::new(&client_datetime.to_rfc3339());
353
354        RecordMeta {
355            id,
356            url,
357            title,
358            client_datetime,
359            timestamp_flora,
360            language: language.map(LanguageRecord::new),
361        }
362    }
363}
364
365#[cfg(test)]
366pub(crate) mod tests {
367    use anyhow::Result;
368    use omry_archiving::{Document, Marshal, record::RecordParams};
369    use pretty_assertions::assert_eq;
370    use std::env;
371    use std::fs::{self, File};
372    use std::io::prelude::*;
373    use std::path::PathBuf;
374
375    use super::*;
376
377    const TEST_ARCHIVE_PATH: &str = "test_data/example.bincode";
378    const WORKSPACE_DIR_KEY: &str = "CARGO_WORKSPACE_DIR";
379
380    /// Gets the path of the test page ("example.html")
381    fn get_test_path(path: &str) -> Result<PathBuf> {
382        let (_, workspace_dir_path) = env::vars()
383            .find(|(k, _)| k == WORKSPACE_DIR_KEY)
384            .ok_or_else(|| anyhow::anyhow!("{WORKSPACE_DIR_KEY} is not set"))?;
385
386        let workspace_dir_path = fs::canonicalize(workspace_dir_path)?;
387        Ok(workspace_dir_path.join(path))
388    }
389
390    fn load_test_archive_holder() -> Result<Document> {
391        let path = get_test_path(TEST_ARCHIVE_PATH)?;
392        let mut file = File::open(&path)?;
393        let mut buf = vec![];
394        file.read_to_end(&mut buf)?;
395
396        let archive_holder = Document::from_bytes(&buf)?;
397        Ok(archive_holder)
398    }
399
400    pub(crate) fn test_record_params() -> Result<RecordParams> {
401        let document = load_test_archive_holder()?;
402
403        let url = "https://example.org/".to_string();
404        let title = "Example Domain";
405        let client_datetime = "2024-10-11T13:49:46-05:00";
406
407        #[allow(clippy::unreadable_literal)]
408        let timestamp_flora = Some(1728695243i64);
409        let language = Some("en".to_string());
410
411        Ok(RecordParams {
412            id: None,
413            url,
414            title: title.to_string(),
415            client_datetime: client_datetime.to_string(),
416            timestamp_flora,
417            language,
418            document,
419        })
420    }
421
422    #[allow(clippy::expect_used)]
423    #[test]
424    fn can_convert_record_to_db_record() -> Result<()> {
425        let record = Record::new(test_record_params()?);
426        let same_record = Record::new(test_record_params()?);
427        let primary_language_tag = same_record.language().map(|lang| {
428            lang.primary()
429                .expect("Couldn't compute the primary language tag")
430        });
431
432        let db_record: DbRecord = record.try_into()?;
433        assert_eq!(db_record.title, same_record.title());
434        assert_eq!(db_record.language, primary_language_tag);
435        Ok(())
436    }
437
438    #[test]
439    fn can_get_schema() {
440        let schema = DbRecord::schema();
441        insta::assert_snapshot!(&schema);
442    }
443
444    #[test]
445    fn same_record_is_equal() -> anyhow::Result<()> {
446        let record: DbRecord = Record::new(test_record_params()?).try_into()?;
447        assert_eq!(record, record);
448        Ok(())
449    }
450
451    #[test]
452    fn different_records_are_not_equal() -> Result<()> {
453        let record = Record::new(test_record_params()?);
454        let mut other_record_params = test_record_params()?;
455        other_record_params.url = "https://example.com".into();
456        let other_record = Record::new(other_record_params);
457
458        let record: DbRecord = record.try_into()?;
459        let other_record = other_record.try_into()?;
460        assert_ne!(record, other_record);
461        Ok(())
462    }
463
464    #[test]
465    fn record_can_roundtrip_through_db_record() -> Result<()> {
466        let record = Record::new(test_record_params()?);
467        let db_record: DbRecord = record.try_into()?;
468        let roundtripped_record = db_record.to_record()?;
469
470        let same_record = Record::new(test_record_params()?);
471        assert_eq!(same_record, roundtripped_record);
472        Ok(())
473    }
474
475    #[test]
476    fn can_get_table_name_from_schema() {
477        let schema = DbRecord::schema();
478        insta::assert_snapshot!(schema.table_name());
479    }
480}
omry_db/model.rs

omry_db/
model.rs
