#![cfg_attr(not(test), deny(clippy::disallowed_methods))]
mod archive;
#[cfg(feature = "storage")]
mod blobs;
mod model;
#[cfg(all(feature = "db", not(feature = "sqlite")))]
mod pg;
mod root;
mod service;
mod store;
mod viewer;
use std::collections::{BTreeMap, BTreeSet};
use bytes::Bytes;
pub use archive::{CapsuleSigner, VerifyReport, verify_dir};
#[cfg(feature = "storage")]
pub use blobs::{collect_blobs, rebind_blobs, restore_blobs};
pub use model::{
BlobEntry, CapsuleManifest, CapsuleModel, DATA_CAPSULE_FORMAT, DATA_CAPSULE_FORMAT_VERSION,
DataCapsuleError, FieldSpec, ModelManifest, Record, Relationship,
};
#[cfg(all(feature = "db", not(feature = "sqlite")))]
pub use pg::PgCapsuleStore;
pub use service::{CapsuleDirectory, CapsuleService, ExportReport};
pub use store::{CapsuleFuture, CapsuleStore, ImportBatch, MemoryCapsuleStore, ModelData};
use model::{check_model_names, record_file, value_key};
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct DataCapsule {
pub manifest: CapsuleManifest,
pub records: BTreeMap<String, Vec<Record>>,
pub blobs: BTreeMap<String, Bytes>,
}
impl DataCapsule {
#[must_use]
pub fn records(&self, table: &str) -> &[Record] {
self.records.get(table).map_or(&[], Vec::as_slice)
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
#[non_exhaustive]
pub struct ImportSummary {
pub tables: Vec<TableCount>,
pub records: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
#[non_exhaustive]
pub struct TableCount {
pub table: String,
pub records: u64,
}
fn check_models(models: &[CapsuleModel]) -> Result<(), DataCapsuleError> {
let mut seen = BTreeSet::new();
for m in models {
check_model_names(
&m.table,
&m.primary_key,
&m.subject_column,
&m.relationships,
&m.blob_columns,
)?;
for column in &m.excluded {
model::check_ident(column)?;
if *column == m.primary_key || *column == m.subject_column {
return Err(DataCapsuleError::InvalidInput(format!(
"{}.{column}: export cannot leave out the key or subject column",
m.table
)));
}
}
if !seen.insert(m.table.as_str()) {
return Err(DataCapsuleError::InvalidInput(format!(
"{} is registered two times",
m.table
)));
}
}
Ok(())
}
pub async fn export_subject(
models: &[CapsuleModel],
store: &dyn CapsuleStore,
subject: &str,
) -> Result<DataCapsule, DataCapsuleError> {
if subject.is_empty() {
return Err(DataCapsuleError::InvalidInput(
"the subject id is empty".to_owned(),
));
}
check_models(models)?;
let data = store.fetch_subject(models, subject).await?;
if data.len() != models.len() {
return Err(DataCapsuleError::Store(format!(
"the store gave data for {} of {} models",
data.len(),
models.len()
)));
}
let mut manifest = CapsuleManifest::new(subject);
let mut records = BTreeMap::new();
let mut described: BTreeMap<&str, Vec<String>> = BTreeMap::new();
let mut held: BTreeMap<&str, Vec<String>> = BTreeMap::new();
for (model, (mut fields, mut rows)) in models.iter().zip(data) {
let configured = [&model.subject_column, &model.primary_key]
.into_iter()
.chain(&model.excluded)
.chain(&model.blob_columns)
.chain(model.relationships.iter().map(|r| &r.column));
for name in configured {
if !fields.iter().any(|f| &f.name == name) {
return Err(DataCapsuleError::InvalidInput(format!(
"{} names column {name:?}, but the table has no such column",
model.table
)));
}
}
for row in &rows {
check_exported_row(model, &fields, row)?;
}
described.insert(
model.table.as_str(),
fields.iter().map(|f| f.name.clone()).collect(),
);
fields.retain(|f| !model.excluded.contains(&f.name));
model::check_field_names(&model.table, &fields)?;
held.insert(
model.table.as_str(),
fields.iter().map(|f| f.name.clone()).collect(),
);
for row in &mut rows {
row.retain(|column, _| !model.excluded.contains(column));
}
manifest.models.push(ModelManifest {
table: model.table.clone(),
primary_key: model.primary_key.clone(),
subject_column: model.subject_column.clone(),
fields,
relationships: model.relationships.clone(),
blob_columns: model
.blob_columns
.iter()
.filter(|c| !model.excluded.contains(c))
.cloned()
.collect(),
record_count: rows.len() as u64,
file: record_file(&model.table),
});
records.insert(model.table.clone(), rows);
}
for model in models {
for rel in &model.relationships {
if let Some(columns) = described.get(rel.target.as_str())
&& !columns.contains(&rel.target_column)
{
return Err(DataCapsuleError::InvalidInput(format!(
"{}.{} links to {}.{:?}, but {} has no such column",
model.table, rel.column, rel.target, rel.target_column, rel.target
)));
}
if let Some(columns) = held.get(rel.target.as_str())
&& !columns.contains(&rel.target_column)
{
return Err(DataCapsuleError::InvalidInput(format!(
"{}.{} links to {}.{:?}, which export leaves out",
model.table, rel.column, rel.target, rel.target_column
)));
}
}
}
Ok(DataCapsule {
manifest,
records,
blobs: BTreeMap::new(),
})
}
fn check_exported_row(
model: &CapsuleModel,
fields: &[FieldSpec],
row: &Record,
) -> Result<(), DataCapsuleError> {
if let Some(column) = row.keys().find(|k| !fields.iter().any(|f| &f.name == *k)) {
return Err(DataCapsuleError::InvalidInput(format!(
"the store gave column {column:?} of {}, which its description of the table \
does not have",
model.table
)));
}
if let Some(field) = fields.iter().find(|f| {
!f.nullable
&& !f.generated
&& !model.excluded.contains(&f.name)
&& row.get(&f.name).is_none_or(serde_json::Value::is_null)
}) {
return Err(DataCapsuleError::InvalidInput(format!(
"the store gave a row of {} without a value in the NOT NULL column {:?}",
model.table, field.name
)));
}
for column in [&model.primary_key, &model.subject_column] {
if row.get(column).and_then(value_key).is_none() {
return Err(DataCapsuleError::InvalidInput(format!(
"the store gave a row of {} without a value in {column:?}",
model.table
)));
}
}
Ok(())
}
pub async fn import_capsule(
capsule: &DataCapsule,
models: &[CapsuleModel],
store: &dyn CapsuleStore,
) -> Result<ImportSummary, DataCapsuleError> {
let order = check_importable(capsule, models)?;
check_target(capsule, models, store).await?;
let batches: Vec<ImportBatch<'_>> = order
.iter()
.map(|model| ImportBatch::new(model, capsule.records(&model.table)))
.collect();
store.insert_all(&batches).await?;
let tables: Vec<TableCount> = batches
.iter()
.map(|b| TableCount {
table: b.model.table.clone(),
records: b.records.len() as u64,
})
.collect();
let records = tables.iter().map(|t| t.records).sum();
Ok(ImportSummary { tables, records })
}
pub(super) fn check_importable<'c>(
capsule: &'c DataCapsule,
models: &[CapsuleModel],
) -> Result<Vec<&'c ModelManifest>, DataCapsuleError> {
let manifest = &capsule.manifest;
archive::check_format(manifest)?;
check_models(models)?;
for model in &manifest.models {
check_model_names(
&model.table,
&model.primary_key,
&model.subject_column,
&model.relationships,
&model.blob_columns,
)?;
model::check_field_names(&model.table, &model.fields)?;
let current = models
.iter()
.find(|m| m.table == model.table)
.ok_or_else(|| DataCapsuleError::UnknownTable(model.table.clone()))?;
if model.subject_column != current.subject_column
|| model.primary_key != current.primary_key
{
return Err(DataCapsuleError::InvalidInput(format!(
"{} in the capsule has subject column {:?} and key {:?}, but the model now \
has {:?} and {:?}",
model.table,
model.subject_column,
model.primary_key,
current.subject_column,
current.primary_key
)));
}
let held = model
.fields
.iter()
.map(|f| f.name.as_str())
.chain(model.blob_columns.iter().map(String::as_str))
.chain(
capsule
.records(&model.table)
.iter()
.flat_map(|row| row.keys().map(String::as_str)),
);
for column in held {
if current.excluded.iter().any(|e| e == column) {
return Err(DataCapsuleError::InvalidInput(format!(
"{}.{column} is excluded from capsules, but the capsule holds it",
model.table
)));
}
}
for row in capsule.records(&model.table) {
if let Some(column) = row
.keys()
.find(|k| !model.fields.iter().any(|f| &f.name == *k))
{
return Err(DataCapsuleError::InvalidInput(format!(
"a row of {} holds column {column:?}, which the manifest does not describe",
model.table
)));
}
if let Some(field) = model.fields.iter().find(|f| {
!f.nullable
&& !f.generated
&& row.get(&f.name).is_none_or(serde_json::Value::is_null)
}) {
return Err(DataCapsuleError::InvalidInput(format!(
"a row of {} has no value in the NOT NULL column {:?}",
model.table, field.name
)));
}
for column in [&model.primary_key, &model.subject_column] {
if row.get(column).and_then(value_key).is_none() {
return Err(DataCapsuleError::InvalidInput(format!(
"a row of {} has no value in {column:?}",
model.table
)));
}
}
}
for row in capsule.records(&model.table) {
for column in blob_columns(model, current) {
if let Some(key) = row.get(column).and_then(viewer::blob_key)
&& manifest.blob(key).is_none()
{
return Err(DataCapsuleError::InvalidInput(format!(
"{}.{column} refers to blob {key:?}, which the capsule does not hold",
model.table
)));
}
}
}
}
import_order(&manifest.models, models)
}
pub(super) async fn check_target(
capsule: &DataCapsule,
models: &[CapsuleModel],
store: &dyn CapsuleStore,
) -> Result<(), DataCapsuleError> {
for model in &capsule.manifest.models {
let Some(current) = models.iter().find(|m| m.table == model.table) else {
return Err(DataCapsuleError::UnknownTable(model.table.clone()));
};
let described = store.describe(current).await?;
for field in &model.fields {
check_target_column(
&model.table,
field,
described.iter().find(|d| d.name == field.name),
)?;
}
let required: Vec<&str> = model
.fields
.iter()
.filter(|f| !f.generated)
.filter(|f| described.iter().any(|d| d.name == f.name && !d.nullable))
.map(|f| f.name.as_str())
.collect();
for row in capsule.records(&model.table) {
if let Some(column) = required
.iter()
.find(|c| row.get(**c).is_none_or(serde_json::Value::is_null))
{
return Err(DataCapsuleError::InvalidInput(format!(
"{}.{column} is NOT NULL in the target table, but a row of the capsule \
has no value in it",
model.table
)));
}
}
if !capsule.records(&model.table).is_empty()
&& let Some(added) = described.iter().find(|d| {
!d.nullable
&& !d.generated
&& !d.has_default
&& !model.fields.iter().any(|f| f.name == d.name)
})
{
return Err(DataCapsuleError::InvalidInput(format!(
"{}.{} is NOT NULL without a default in the target table, but the capsule \
has no value for it",
model.table, added.name
)));
}
}
Ok(())
}
fn check_target_column(
table: &str,
field: &FieldSpec,
target: Option<&FieldSpec>,
) -> Result<(), DataCapsuleError> {
let refuse = |why: String| {
Err(DataCapsuleError::InvalidInput(format!(
"{table}.{} is in the capsule, but {why}",
field.name
)))
};
let Some(target) = target else {
if field.generated {
return Ok(());
}
return refuse("the target table has no such column".to_owned());
};
if target.generated != field.generated {
let now = if target.generated {
"generates"
} else {
"no longer generates"
};
return refuse(format!("the target table {now} it"));
}
if field.generated {
return Ok(());
}
if target.data_type != field.data_type || target.base_type != field.base_type {
let shown = |f: &FieldSpec| {
f.base_type.as_ref().map_or_else(
|| f.data_type.clone(),
|base| format!("{} (over {base})", f.data_type),
)
};
return refuse(format!(
"it is {} there and {} in the target table",
shown(field),
shown(target)
));
}
Ok(())
}
fn blob_columns<'a>(model: &'a ModelManifest, current: &'a CapsuleModel) -> Vec<&'a String> {
let mut columns: Vec<&String> = model.blob_columns.iter().collect();
for column in ¤t.blob_columns {
if !current.excluded.contains(column) && !columns.contains(&column) {
columns.push(column);
}
}
columns
}
#[cfg(feature = "storage")]
pub(super) fn adopt_blob_columns(capsule: &mut DataCapsule, models: &[CapsuleModel]) {
for model in &mut capsule.manifest.models {
if let Some(current) = models.iter().find(|m| m.table == model.table) {
let columns: Vec<String> = blob_columns(model, current).into_iter().cloned().collect();
model.blob_columns = columns;
}
}
}
fn import_order<'m>(
models: &'m [ModelManifest],
current: &[CapsuleModel],
) -> Result<Vec<&'m ModelManifest>, DataCapsuleError> {
let tables: BTreeSet<&str> = models.iter().map(|m| m.table.as_str()).collect();
let mut placed: BTreeSet<&str> = BTreeSet::new();
let mut order = Vec::with_capacity(models.len());
while order.len() < models.len() {
let next = models.iter().find(|m| {
let mut targets = m.relationships.iter().map(|r| r.target.as_str()).chain(
current
.iter()
.filter(|c| c.table == m.table)
.flat_map(|c| c.relationships.iter().map(|r| r.target.as_str())),
);
!placed.contains(m.table.as_str())
&& targets.all(|t| t == m.table || !tables.contains(t) || placed.contains(t))
});
let Some(next) = next else {
let rest: Vec<&str> = models
.iter()
.map(|m| m.table.as_str())
.filter(|t| !placed.contains(t))
.collect();
return Err(DataCapsuleError::RelationshipCycle(rest.join(", ")));
};
placed.insert(next.table.as_str());
order.push(next);
}
Ok(order)
}
#[cfg(test)]
mod tests {
use super::*;
struct FirstOnly(MemoryCapsuleStore);
impl CapsuleStore for FirstOnly {
fn describe<'a>(&'a self, model: &'a CapsuleModel) -> CapsuleFuture<'a, Vec<FieldSpec>> {
self.0.describe(model)
}
fn fetch<'a>(
&'a self,
model: &'a CapsuleModel,
subject: &'a str,
) -> CapsuleFuture<'a, Vec<Record>> {
self.0.fetch(model, subject)
}
fn fetch_subject<'a>(
&'a self,
models: &'a [CapsuleModel],
subject: &'a str,
) -> CapsuleFuture<'a, Vec<ModelData>> {
Box::pin(async move {
let mut data = self.0.fetch_subject(models, subject).await?;
data.truncate(1);
Ok(data)
})
}
fn insert_all<'a>(&'a self, batches: &'a [ImportBatch<'a>]) -> CapsuleFuture<'a, ()> {
self.0.insert_all(batches)
}
}
#[tokio::test]
async fn export_refuses_a_store_that_skips_a_model() {
let fields = || vec![FieldSpec::new("id", "bigint")];
let store = FirstOnly(
MemoryCapsuleStore::new()
.table("users", fields())
.table("posts", fields()),
);
let models = [
CapsuleModel::new("users", "id"),
CapsuleModel::new("posts", "id"),
];
let err = export_subject(&models, &store, "1")
.await
.expect_err("posts is missing");
assert!(matches!(err, DataCapsuleError::Store(_)), "{err:?}");
}
fn manifest(table: &str, targets: &[&str]) -> ModelManifest {
let mut model = CapsuleModel::new(table, "id");
for t in targets {
model = model.belongs_to(format!("{t}_id"), *t);
}
ModelManifest {
table: model.table,
primary_key: model.primary_key,
subject_column: model.subject_column,
fields: Vec::new(),
relationships: model.relationships,
blob_columns: Vec::new(),
record_count: 0,
file: record_file(table),
}
}
fn order(models: &[ModelManifest]) -> Vec<&str> {
import_order(models, &[])
.unwrap()
.iter()
.map(|m| m.table.as_str())
.collect()
}
#[test]
fn order_puts_targets_first() {
let models = [
manifest("comments", &["posts", "users"]),
manifest("posts", &["users"]),
manifest("users", &[]),
];
assert_eq!(order(&models), ["users", "posts", "comments"]);
}
#[test]
fn order_ignores_self_links_and_outside_tables() {
let models = [manifest("nodes", &["nodes", "orgs"]), manifest("a", &[])];
assert_eq!(order(&models), ["nodes", "a"]);
}
#[test]
fn order_reports_a_cycle() {
let models = [
manifest("a", &["b"]),
manifest("b", &["a"]),
manifest("c", &[]),
];
let err = import_order(&models, &[]).unwrap_err();
assert!(
matches!(err, DataCapsuleError::RelationshipCycle(ref t) if t == "a, b"),
"{err}"
);
}
#[test]
fn duplicate_model_is_rejected() {
let models = [CapsuleModel::new("a", "id"), CapsuleModel::new("a", "id")];
assert!(matches!(
check_models(&models),
Err(DataCapsuleError::InvalidInput(_))
));
}
#[test]
fn the_key_and_subject_columns_cannot_be_excluded() {
for column in ["id", "owner_id"] {
let models = [CapsuleModel::new("a", "owner_id").exclude(column)];
assert!(
matches!(
check_models(&models),
Err(DataCapsuleError::InvalidInput(_))
),
"{column}"
);
}
}
}