omry_db/

model.rs

//! Data structures for flora's SQLite database.
use chrono::{DateTime, FixedOffset};
use omry_archiving::record::LanguageRecord;
use omry_archiving::{
    ClientDateTimeRecord, Document, Marshal, Record, RecordMeta, RecordParams, ToRecord,
};
use std::fmt;
use thiserror::Error;

/// Data types for the database columns.
#[allow(dead_code)]
#[allow(clippy::missing_docs_in_private_items)]
#[derive(Debug)]
enum ColumnType {
    Boolean,
    Integer,
    Real,
    Text,
    Blob,
}

impl fmt::Display for ColumnType {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        let s = match self {
            Self::Boolean => "BOOLEAN",
            Self::Integer => "INTEGER",
            Self::Real => "REAL",
            Self::Text => "TEXT",
            Self::Blob => "BLOB",
        };
        write!(fmt, "{s}")
    }
}

/// A builder for [`ColumnDefinition`].
struct ColumnDefinitionBuilder {
    /// The column name.
    name: String,

    /// The data type of the column.
    column_type: ColumnType,

    /// Whether the column can be nullable.
    not_null: Option<bool>,

    /// Whether the column should be used as a SQLite primary key.
    primary_key: Option<bool>,
}

impl ColumnDefinitionBuilder {
    /// Builds the [`ColumnDefinition`] with the parameters set in this builder's fields.
    fn build(self) -> ColumnDefinition {
        ColumnDefinition {
            name: self.name,
            column_type: self.column_type,
            not_null: self.not_null.unwrap_or_default(),
            primary_key: self.primary_key.unwrap_or_default(),
        }
    }

    /// Sets the value [`ColumnDefinitionBuilder.not_null`] to the given value.
    fn not_null(mut self, not_null: bool) -> Self {
        self.not_null = Some(not_null);
        self
    }

    /// Sets the value [`ColumnDefinitionBuilder.not_null`] to the given value.
    fn primary_key(mut self, primary_key: bool) -> Self {
        self.primary_key = Some(primary_key);
        self
    }
}

/// Structure for programmatically describing a SQLite column.
#[derive(Debug)]
struct ColumnDefinition {
    /// The column name.
    name: String,

    /// The data type of the column.
    column_type: ColumnType,

    /// Whether the column can be nullable.
    not_null: bool,

    /// Whether the column should be used as a SQLite primary key.
    primary_key: bool,
}

impl ColumnDefinition {
    /// Returns the builder for defining a column programmatically.
    fn builder(name: String, column_type: ColumnType) -> ColumnDefinitionBuilder {
        ColumnDefinitionBuilder {
            name,
            column_type,
            not_null: None,
            primary_key: None,
        }
    }
}

/// Definition of a SQLite table schema.
#[derive(Debug)]
pub(crate) struct Schema {
    /// The name of the table that this schema describes.
    table_name: String,

    /// The definitions of the columns in this table.
    columns: Vec<ColumnDefinition>,
}

impl Schema {
    /// Returns the name of this schema's table.
    pub(crate) fn table_name(&self) -> &str {
        &self.table_name
    }
}

impl fmt::Display for Schema {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        writeln!(f, "{} (", self.table_name)?;

        let mut parts = vec![];
        let mut s = String::new();

        for column in &self.columns {
            s.push_str(format!("\t{} {}", column.name, column.column_type).as_str());

            if column.primary_key {
                s.push_str(" PRIMARY KEY");
            }

            // "According to the SQL standard,
            // PRIMARY KEY should always imply NOT NULL.
            // Unfortunately, due to a bug in some early versions, this is not the case in SQLite."
            // https://www.sqlite.org/lang_createtable.html section 3.5
            if column.primary_key || column.not_null {
                s.push_str(" NOT NULL");
            }

            parts.push(s.clone());
            s.clear();
        }

        let s = parts.join(",\n");
        write!(f, "{s}\n)")
    }
}

/// Public wrappere for database API errors.
#[derive(Debug, Error)]
#[error(transparent)]
pub struct DbRecordError(#[from] DbRecordErrorRepr);

/// Private representation of database API errors.
#[derive(Debug, Error)]
enum DbRecordErrorRepr {
    /// Errors encountered when deserializing the web page.
    #[error("Error (de)serializing the web page: {0}")]
    WebArchive(String),
}

/// SQLite representation of a flora `Record`.
#[derive(Debug)]
pub struct DbRecord {
    /// The SQLite row ID of this record.
    pub(crate) id: Option<i64>,

    /// The original URL that corresponds to this record.
    pub(crate) url: String,

    /// The client date and time, when the client saved the content to be archived.
    pub(crate) client_datetime: DateTime<FixedOffset>,

    /// Raw date time, in case we couldn't parse it.
    pub(crate) client_datetime_raw: Option<String>,

    /// The Omry server timestamp, when Omry first imported this record.
    pub(crate) timestamp_flora: i64,

    /// The page title that the client provided for this record.
    pub(crate) title: String,

    /// The language we extracted from this record's metadata, if any.
    pub(crate) language: Option<String>,

    /// The binary serialized web content in this record.
    pub(crate) web_archive_serialized: Vec<u8>,
}

impl DbRecord {
    /// The database schema.
    pub(crate) fn schema() -> Schema {
        let columns = vec![
            ColumnDefinition::builder("id".to_string(), ColumnType::Integer)
                .primary_key(true)
                .build(),
            ColumnDefinition::builder("url".to_string(), ColumnType::Text)
                .not_null(true)
                .build(),
            ColumnDefinition::builder("client_datetime".to_string(), ColumnType::Text)
                .not_null(true)
                .build(),
            ColumnDefinition::builder("client_datetime_raw".to_string(), ColumnType::Text).build(),
            ColumnDefinition::builder("timestamp_flora".to_string(), ColumnType::Integer)
                .not_null(true)
                .build(),
            ColumnDefinition::builder("title".to_string(), ColumnType::Text)
                .not_null(true)
                .build(),
            ColumnDefinition::builder("language".to_string(), ColumnType::Text).build(),
            ColumnDefinition::builder("web_archive_serialized".to_string(), ColumnType::Blob)
                .not_null(true)
                .build(),
        ];
        Schema {
            table_name: "records".to_string(),
            columns,
        }
    }

    /// The SQLite row ID of this record.
    ///
    /// This should only return `None` if the record
    /// hasn't yet been inserted into the database.
    #[must_use]
    pub fn id(&self) -> Option<i64> {
        self.id
    }
}

impl TryFrom<Record> for DbRecord {
    type Error = DbRecordError;

    fn try_from(value: Record) -> Result<Self, Self::Error> {
        let id = value.id();
        let url = value.url().to_string();
        let title = value.title().to_string();

        let (client_datetime, client_datetime_raw) = match value.client_datetime() {
            ClientDateTimeRecord::Parsed(datetime) => (*datetime, None),
            ClientDateTimeRecord::Interpolated { value, raw } => (*value, Some(raw.clone())),
        };

        let timestamp_flora = value.timestamp_archived();

        let language = value.language().map(|lang| {
            lang.primary()
                .unwrap_or_else(|error| error.raw_data().to_string())
        });

        let web_archive_serialized = value.document().to_byte_vec().map_err(|error| {
            DbRecordErrorRepr::WebArchive(format!("Couldn't serialize the archive: {error}"))
        })?;

        Ok(Self {
            id,
            url,
            client_datetime,
            client_datetime_raw,
            timestamp_flora,
            title,
            language,
            web_archive_serialized,
        })
    }
}

impl ToRecord for DbRecord {
    type Error = DbRecordError;

    fn to_record(&self) -> Result<Record, Self::Error> {
        let document = Document::from_bytes(&self.web_archive_serialized)
            .map_err(|error| DbRecordErrorRepr::WebArchive(error.to_string()))?;

        let params = RecordParams {
            id: self.id,
            url: self.url.clone(),
            title: self.title.clone(),
            client_datetime: self.client_datetime.to_rfc3339(),
            timestamp_flora: Some(self.timestamp_flora),
            language: self.language.clone(),
            document,
        };

        Ok(Record::new(params))
    }
}

/// Two instances of `DbRecord` are partially equal if all their fields except `id` are equal,
/// **and** either:
/// - Both of their `id` fields are equal (including if both are `None`), or
/// - One of the records' `id` fields is `None`.
///
/// We consider records where one `DbRecord` is `Some` and the other is `None` to be
/// partially equal to account for a situation where a new record was just inserted into
/// the database, and we are comparing it to the same record that we got back from the database.
impl PartialEq for DbRecord {
    fn eq(&self, other: &Self) -> bool {
        // Short-circuit if the `id` fields are actually different.
        let ids_considered_equal = if let (Some(this_id), Some(other_id)) = (self.id, other.id) {
            this_id == other_id
        } else {
            // One or both `id`s are `None`
            true
        };

        self.url == other.url
            && self.client_datetime == other.client_datetime
            && self.client_datetime_raw == other.client_datetime_raw
            && self.timestamp_flora == other.timestamp_flora
            && self.title == other.title
            && self.language == other.language
            && self.web_archive_serialized == other.web_archive_serialized
            && ids_considered_equal
    }
}

/// Counterpart to [`RecordMeta`] for data as it is stored in SQLite.
#[derive(Debug, PartialEq)]
pub struct DbRecordMeta {
    /// The SQLite row ID of this record.
    pub(crate) id: i64,

    /// The original URL that corresponds to this record.
    pub(crate) url: String,

    /// The client date and time, when the client saved the content to be archived.
    pub(crate) client_datetime: DateTime<FixedOffset>,

    /// The Omry server timestamp, when Omry first imported this record.
    pub(crate) timestamp_flora: i64,

    /// The page title that the client provided for this record.
    pub(crate) title: String,

    /// The language we extracted from this record's metadata, if any.
    pub(crate) language: Option<String>,
}

impl From<DbRecordMeta> for RecordMeta {
    fn from(record: DbRecordMeta) -> Self {
        let DbRecordMeta {
            id,
            url,
            client_datetime,
            timestamp_flora,
            title,
            language,
        } = record;

        let client_datetime = ClientDateTimeRecord::new(&client_datetime.to_rfc3339());

        RecordMeta {
            id,
            url,
            title,
            client_datetime,
            timestamp_flora,
            language: language.map(LanguageRecord::new),
        }
    }
}

#[cfg(test)]
pub(crate) mod tests {
    use anyhow::Result;
    use omry_archiving::{Document, Marshal, record::RecordParams};
    use pretty_assertions::assert_eq;
    use std::env;
    use std::fs::{self, File};
    use std::io::prelude::*;
    use std::path::PathBuf;

    use super::*;

    const TEST_ARCHIVE_PATH: &str = "test_data/example.postcard";
    const WORKSPACE_DIR_KEY: &str = "CARGO_WORKSPACE_DIR";

    /// Gets the path of the test page ("example.html")
    fn get_test_path(path: &str) -> Result<PathBuf> {
        let (_, workspace_dir_path) = env::vars()
            .find(|(k, _)| k == WORKSPACE_DIR_KEY)
            .ok_or_else(|| anyhow::anyhow!("{WORKSPACE_DIR_KEY} is not set"))?;

        let workspace_dir_path = fs::canonicalize(workspace_dir_path)?;
        Ok(workspace_dir_path.join(path))
    }

    fn load_test_archive_holder() -> Result<Document> {
        let path = get_test_path(TEST_ARCHIVE_PATH)?;
        let mut file = File::open(&path)?;
        let mut buf = vec![];
        file.read_to_end(&mut buf)?;

        let archive_holder = Document::from_bytes(&buf)?;
        Ok(archive_holder)
    }

    pub(crate) fn test_record_params() -> Result<RecordParams> {
        let document = load_test_archive_holder()?;

        let url = "https://example.org/".to_string();
        let title = "Example Domain";
        let client_datetime = "2024-10-11T13:49:46-05:00";

        #[allow(clippy::unreadable_literal)]
        let timestamp_flora = Some(1728695243i64);
        let language = Some("en".to_string());

        Ok(RecordParams {
            id: None,
            url,
            title: title.to_string(),
            client_datetime: client_datetime.to_string(),
            timestamp_flora,
            language,
            document,
        })
    }

    #[allow(clippy::expect_used)]
    #[test]
    fn can_convert_record_to_db_record() -> Result<()> {
        let record = Record::new(test_record_params()?);
        let same_record = Record::new(test_record_params()?);
        let primary_language_tag = same_record.language().map(|lang| {
            lang.primary()
                .expect("Couldn't compute the primary language tag")
        });

        let db_record: DbRecord = record.try_into()?;
        assert_eq!(db_record.title, same_record.title());
        assert_eq!(db_record.language, primary_language_tag);
        Ok(())
    }

    #[test]
    fn can_get_schema() {
        let schema = DbRecord::schema();
        insta::assert_snapshot!(&schema);
    }

    #[test]
    fn same_record_is_equal() -> anyhow::Result<()> {
        let record: DbRecord = Record::new(test_record_params()?).try_into()?;
        assert_eq!(record, record);
        Ok(())
    }

    #[test]
    fn different_records_are_not_equal() -> Result<()> {
        let record = Record::new(test_record_params()?);
        let mut other_record_params = test_record_params()?;
        other_record_params.url = "https://example.com".into();
        let other_record = Record::new(other_record_params);

        let record: DbRecord = record.try_into()?;
        let other_record = other_record.try_into()?;
        assert_ne!(record, other_record);
        Ok(())
    }

    #[test]
    fn record_can_roundtrip_through_db_record() -> Result<()> {
        let record = Record::new(test_record_params()?);
        let db_record: DbRecord = record.try_into()?;
        let roundtripped_record = db_record.to_record()?;

        let same_record = Record::new(test_record_params()?);
        assert_eq!(same_record, roundtripped_record);
        Ok(())
    }

    #[test]
    fn can_get_table_name_from_schema() {
        let schema = DbRecord::schema();
        insta::assert_snapshot!(schema.table_name());
    }
}