Skip to main content

cozo/runtime/db/
import_export.rs

1/*
2 * Copyright 2022, The Cozo Project Authors.
3 *
4 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
5 * If a copy of the MPL was not distributed with this file,
6 * You can obtain one at https://mozilla.org/MPL/2.0/.
7 */
8
9use std::collections::BTreeMap;
10
11use itertools::Itertools;
12use miette::{bail, miette, Diagnostic, Result};
13use smartstring::SmartString;
14use thiserror::Error;
15
16use crate::data::functions::current_validity;
17use crate::data::json::JsonValue;
18use crate::data::relation::ColumnDef;
19use crate::data::tuple::{Tuple, TupleT};
20use crate::data::value::DataValue;
21use crate::runtime::relation::{extend_tuple_from_v, AccessLevel, InsufficientAccessLevel};
22use crate::storage::Storage;
23use crate::decode_tuple_from_kv;
24
25use super::{Db, ImportIntoIndex, NamedRows};
26
27impl<'s, S: Storage<'s>> Db<S> {
28    /// Export relations to JSON data.
29    ///
30    /// `relations` contains names of the stored relations to export.
31    pub fn export_relations<I, T>(&'s self, relations: I) -> Result<BTreeMap<String, NamedRows>>
32    where
33        T: AsRef<str>,
34        I: Iterator<Item = T>,
35    {
36        let tx = self.transact()?;
37        let mut ret: BTreeMap<String, NamedRows> = BTreeMap::new();
38        for rel in relations {
39            let handle = tx.get_relation(rel.as_ref(), false)?;
40            let size_hint = handle.metadata.keys.len() + handle.metadata.non_keys.len();
41
42            if handle.access_level < AccessLevel::ReadOnly {
43                bail!(InsufficientAccessLevel(
44                    handle.name.to_string(),
45                    "data export".to_string(),
46                    handle.access_level
47                ));
48            }
49
50            let mut cols = handle
51                .metadata
52                .keys
53                .iter()
54                .map(|col| col.name.clone())
55                .collect_vec();
56            cols.extend(
57                handle
58                    .metadata
59                    .non_keys
60                    .iter()
61                    .map(|col| col.name.clone())
62                    .collect_vec(),
63            );
64
65            let start = Tuple::default().encode_as_key(handle.id);
66            let end = Tuple::default().encode_as_key(handle.id.next());
67
68            let mut rows = vec![];
69            for data in tx.store_tx.range_scan(&start, &end) {
70                let (k, v) = data?;
71                let tuple = decode_tuple_from_kv(&k, &v, Some(size_hint));
72                rows.push(tuple);
73            }
74            let headers = cols.iter().map(|col| col.to_string()).collect_vec();
75            ret.insert(rel.as_ref().to_string(), NamedRows::new(headers, rows));
76        }
77        Ok(ret)
78    }
79
80    /// Import relations. The argument `data` accepts data in the shape of
81    /// what was returned by [Self::export_relations].
82    /// The target stored relations must already exist in the database.
83    /// Any associated indices will be updated.
84    ///
85    /// Note that triggers and callbacks are _not_ run for the relations, if any exists.
86    /// If you need to activate triggers or callbacks, use queries with parameters.
87    pub fn import_relations(&'s self, data: BTreeMap<String, NamedRows>) -> Result<()> {
88        #[derive(Debug, Diagnostic, Error)]
89        #[error("cannot import data for relation '{0}': {1}")]
90        #[diagnostic(code(import::bad_data))]
91        struct BadDataForRelation(String, JsonValue);
92
93        let rel_names = data.keys().map(SmartString::from).collect_vec();
94        let locks = self.obtain_relation_locks(rel_names.iter());
95        let _guards = locks.iter().map(|l| l.read().unwrap()).collect_vec();
96
97        let cur_vld = current_validity();
98
99        let mut tx = self.transact_write()?;
100
101        for (relation_op, in_data) in data {
102            let is_delete;
103            let relation: &str = match relation_op.strip_prefix('-') {
104                None => {
105                    is_delete = false;
106                    &relation_op
107                }
108                Some(s) => {
109                    is_delete = true;
110                    s
111                }
112            };
113            if relation.contains(':') {
114                bail!(ImportIntoIndex(relation.to_string()))
115            }
116            let handle = tx.get_relation(relation, false)?;
117            let has_indices = !handle.indices.is_empty();
118
119            if handle.access_level < AccessLevel::Protected {
120                bail!(InsufficientAccessLevel(
121                    handle.name.to_string(),
122                    "data import".to_string(),
123                    handle.access_level
124                ));
125            }
126
127            let header2idx: BTreeMap<_, _> = in_data
128                .headers
129                .iter()
130                .enumerate()
131                .map(|(i, k)| -> Result<(&str, usize)> { Ok((k as &str, i)) })
132                .try_collect()?;
133
134            let key_indices: Vec<_> = handle
135                .metadata
136                .keys
137                .iter()
138                .map(|col| -> Result<(usize, &ColumnDef)> {
139                    let idx = header2idx.get(&col.name as &str).ok_or_else(|| {
140                        miette!(
141                            "required header {} not found for relation {}",
142                            col.name,
143                            relation
144                        )
145                    })?;
146                    Ok((*idx, col))
147                })
148                .try_collect()?;
149
150            let val_indices: Vec<_> = if is_delete {
151                vec![]
152            } else {
153                handle
154                    .metadata
155                    .non_keys
156                    .iter()
157                    .map(|col| -> Result<(usize, &ColumnDef)> {
158                        let idx = header2idx.get(&col.name as &str).ok_or_else(|| {
159                            miette!(
160                                "required header {} not found for relation {}",
161                                col.name,
162                                relation
163                            )
164                        })?;
165                        Ok((*idx, col))
166                    })
167                    .try_collect()?
168            };
169
170            for row in in_data.rows {
171                let keys: Vec<_> = key_indices
172                    .iter()
173                    .map(|(i, col)| -> Result<DataValue> {
174                        let v = row
175                            .get(*i)
176                            .ok_or_else(|| miette!("row too short: {:?}", row))?;
177                        col.typing.coerce(v.clone(), cur_vld)
178                    })
179                    .try_collect()?;
180                let k_store = handle.encode_key_for_store(&keys, Default::default())?;
181                if has_indices {
182                    if let Some(existing) = tx.store_tx.get(&k_store, false)? {
183                        let mut old = keys.clone();
184                        extend_tuple_from_v(&mut old, &existing);
185                        if is_delete || old != row {
186                            for (idx_rel, extractor) in handle.indices.values() {
187                                let idx_tup =
188                                    extractor.iter().map(|i| old[*i].clone()).collect_vec();
189                                let encoded =
190                                    idx_rel.encode_key_for_store(&idx_tup, Default::default())?;
191                                tx.store_tx.del(&encoded)?;
192                            }
193                        }
194                    }
195                }
196                if is_delete {
197                    tx.store_tx.del(&k_store)?;
198                } else {
199                    let vals: Vec<_> = val_indices
200                        .iter()
201                        .map(|(i, col)| -> Result<DataValue> {
202                            let v = row
203                                .get(*i)
204                                .ok_or_else(|| miette!("row too short: {:?}", row))?;
205                            col.typing.coerce(v.clone(), cur_vld)
206                        })
207                        .try_collect()?;
208                    let v_store = handle.encode_val_only_for_store(&vals, Default::default())?;
209                    tx.store_tx.put(&k_store, &v_store)?;
210                    if has_indices {
211                        let mut kv = keys;
212                        kv.extend(vals);
213                        for (idx_rel, extractor) in handle.indices.values() {
214                            let idx_tup = extractor.iter().map(|i| kv[*i].clone()).collect_vec();
215                            let encoded =
216                                idx_rel.encode_key_for_store(&idx_tup, Default::default())?;
217                            tx.store_tx.put(&encoded, &[])?;
218                        }
219                    }
220                }
221            }
222        }
223        tx.commit_tx()?;
224        Ok(())
225    }
226}