hidane_core/export/
mod.rs

1//! The official emulator's export format (Cloud Datastore's managed export), byte for byte.
2//!
3//! An export named `N` is a directory `N/` holding:
4//!
5//! - `N.overall_export_metadata`: a [`log`] of two records, the version byte `0x33` and an
6//!   `OverallExportMetadata` message pointing at each kind directory's metadata, with its
7//!   entity count and output size; an empty export has no pointer and no kind directory;
8//! - `all_namespaces/all_kinds/all_namespaces_all_kinds.export_metadata`: a plain message
9//!   with the export's name, start and end time (microseconds), and its output files;
10//! - `all_namespaces/all_kinds/output-0`: a [`log`] with one [`entity`] per document.
11//!
12//! `docs/export-format.md` describes every field; `tests/fixtures/export_import.json` in the
13//! `hidane` crate holds exports written by the official emulator.
14
15pub mod entity;
16pub mod log;
17mod wire;
18
19use hidane_proto::google::firestore::v1::{Value, value::ValueType};
20use hidane_proto::ordered::Fields;
21
22pub use wire::Malformed;
23
24use crate::path::ResourcePath;
25use wire::{Reader, Writer};
26
27/// The first record of an overall metadata file.
28pub const VERSION: u8 = 0x33;
29
30/// Where the one kind directory's metadata sits, relative to the export directory.
31pub const KIND_METADATA: &str = "all_namespaces/all_kinds/all_namespaces_all_kinds.export_metadata";
32
33/// The one output file, relative to the kind directory.
34pub const OUTPUT: &str = "output-0";
35
36/// A document as an export holds it.
37#[derive(Debug, Clone, PartialEq)]
38pub struct ExportedDocument {
39    /// The database the key names, `(default)` when it names none.
40    pub database_id: String,
41    pub path: ResourcePath,
42    /// References relative to the importing project have an empty project
43    /// (`projects//databases/…`); see [`Self::fields_in`].
44    pub fields: Fields,
45}
46
47impl ExportedDocument {
48    /// The fields as imported into `project`: references relative to the importing project
49    /// (top-level indexed properties, see [`entity`]) point into it.
50    pub fn fields_in(&self, project: &str) -> Fields {
51        fn resolve(value: &mut Value, project: &str) {
52            match &mut value.value_type {
53                Some(ValueType::ReferenceValue(name)) => {
54                    if let Some(rest) = name.strip_prefix("projects//") {
55                        *name = format!("projects/{project}/{rest}");
56                    }
57                }
58                Some(ValueType::ArrayValue(array)) => {
59                    array.values.iter_mut().for_each(|v| resolve(v, project));
60                }
61                _ => {}
62            }
63        }
64        let mut fields = self.fields.clone();
65        fields.values_mut().for_each(|v| resolve(v, project));
66        fields
67    }
68}
69
70/// An `OverallExportMetadata` pointer to a kind directory's metadata.
71#[derive(Debug, Clone, PartialEq, Eq)]
72pub struct Pointer {
73    /// Relative to the export directory.
74    pub path: String,
75    pub entities: u64,
76    /// The size of the kind's output files.
77    pub bytes: u64,
78}
79
80/// The contents of `{name}.overall_export_metadata`.
81pub fn overall_metadata(pointer: Option<&Pointer>) -> Vec<u8> {
82    let mut w = Writer::default();
83    if let Some(pointer) = pointer {
84        w.message(1, |p| {
85            // Constant in every export the official emulator writes.
86            p.message(1, |format| {
87                format.uint(1, 2);
88                format.uint(3, 3);
89            });
90            p.bytes(2, pointer.path.as_bytes());
91            p.uint(3, pointer.entities);
92            p.uint(4, pointer.bytes);
93        });
94    }
95    log::write_all([&[VERSION][..], &w.into_bytes()])
96}
97
98/// Why an overall metadata file was refused.
99#[derive(Debug, Clone, Copy, PartialEq, Eq)]
100pub enum OverallError {
101    /// The first record is missing or is not [`VERSION`].
102    Version,
103    Malformed,
104}
105
106/// The kind metadata paths an overall metadata file points at, relative to its directory.
107pub fn parse_overall_metadata(data: &[u8]) -> Result<Vec<String>, OverallError> {
108    let mut records = log::records(data);
109    match records.next() {
110        Some(Ok(version)) if *version == [VERSION] => {}
111        _ => return Err(OverallError::Version),
112    }
113    let Some(Ok(metadata)) = records.next() else {
114        return Err(OverallError::Malformed);
115    };
116    let mut paths = Vec::new();
117    for field in Reader::new(&metadata) {
118        let (number, pointer) = field.map_err(|_| OverallError::Malformed)?;
119        if number != 1 {
120            continue;
121        }
122        for field in pointer.fields().ok_or(OverallError::Malformed)? {
123            let (number, value) = field.map_err(|_| OverallError::Malformed)?;
124            if number == 2 {
125                let path = value.as_bytes().ok_or(OverallError::Malformed)?;
126                paths.push(String::from_utf8(path.to_vec()).map_err(|_| OverallError::Malformed)?);
127            }
128        }
129    }
130    Ok(paths)
131}
132
133/// The contents of a kind directory's `.export_metadata`.
134pub fn export_metadata(name: &str, start_micros: i64, end_micros: i64) -> Vec<u8> {
135    let mut w = Writer::default();
136    w.message(1, |m| {
137        m.bytes(1, name.as_bytes());
138        m.int(2, start_micros);
139        m.int(3, end_micros);
140    });
141    w.message(2, |files| {
142        files.bytes(1, b"");
143        files.bytes(2, OUTPUT.as_bytes());
144    });
145    w.into_bytes()
146}
147
148/// The output files a kind's metadata lists, relative to its directory.
149pub fn parse_export_metadata(data: &[u8]) -> Result<Vec<String>, Malformed> {
150    let mut outputs = Vec::new();
151    for field in Reader::new(data) {
152        let (number, files) = field?;
153        if number != 2 {
154            continue;
155        }
156        let mut prefix = String::new();
157        for field in files.fields().ok_or(Malformed)? {
158            let (number, value) = field?;
159            let text = String::from_utf8(value.as_bytes().ok_or(Malformed)?.to_vec())
160                .map_err(|_| Malformed)?;
161            match number {
162                1 => prefix = text,
163                2 if prefix.is_empty() => outputs.push(text),
164                2 => outputs.push(format!("{}/{text}", prefix.trim_end_matches('/'))),
165                _ => {}
166            }
167        }
168    }
169    Ok(outputs)
170}
171
172#[cfg(test)]
173mod tests {
174    use super::*;
175
176    #[test]
177    fn an_empty_export_points_at_nothing() {
178        let data = overall_metadata(None);
179        assert_eq!(parse_overall_metadata(&data), Ok(vec![]));
180        assert_eq!(data.len(), 15);
181    }
182
183    #[test]
184    fn metadata_round_trips() {
185        let pointer = Pointer {
186            path: KIND_METADATA.to_owned(),
187            entities: 9,
188            bytes: 4928,
189        };
190        assert_eq!(
191            parse_overall_metadata(&overall_metadata(Some(&pointer))),
192            Ok(vec![KIND_METADATA.to_owned()])
193        );
194        assert_eq!(
195            parse_export_metadata(&export_metadata("n", 1, 2)),
196            Ok(vec![OUTPUT.to_owned()])
197        );
198    }
199
200    #[test]
201    fn another_version_is_refused() {
202        assert_eq!(
203            parse_overall_metadata(&log::write_all([&[0x34u8][..], &[]])),
204            Err(OverallError::Version)
205        );
206        assert_eq!(
207            parse_overall_metadata(b"garbage"),
208            Err(OverallError::Version)
209        );
210    }
211}
hidane_core/export/mod.rs

hidane_core/export/
mod.rs
