hidane_core/export/
mod.rs

1//! The official emulator's export format (Cloud Datastore's managed export), byte for byte.
2//!
3//! An export named `N` is a directory `N/` holding:
4//!
5//! - `N.overall_export_metadata`: a [`log`] of two records, the version byte `0x33` and an
6//!   `OverallExportMetadata` message pointing at each kind directory's metadata, with its
7//!   entity count and output size; an empty export has no pointer and no kind directory;
8//! - `all_namespaces/all_kinds/all_namespaces_all_kinds.export_metadata`: a plain message
9//!   with the export's name, start and end time (microseconds), and its output files;
10//! - `all_namespaces/all_kinds/output-0`: a [`log`] with one [`entity`] per document.
11//!
12//! `docs/export-format.md` describes every field; `tests/fixtures/export_import.json` in the
13//! `hidane` crate holds exports written by the official emulator.
14
15pub mod datastore;
16pub mod entity;
17pub mod log;
18mod wire;
19
20use hidane_proto::google::firestore::v1::{Value, value::ValueType};
21use hidane_proto::ordered::Fields;
22
23pub use wire::Malformed;
24
25use crate::path::ResourcePath;
26use wire::{Reader, Writer};
27
28/// The first record of an overall metadata file.
29pub const VERSION: u8 = 0x33;
30
31/// Where the one kind directory's metadata sits, relative to the export directory.
32pub const KIND_METADATA: &str = "all_namespaces/all_kinds/all_namespaces_all_kinds.export_metadata";
33
34/// The one output file, relative to the kind directory.
35pub const OUTPUT: &str = "output-0";
36
37/// A document as an export holds it.
38#[derive(Debug, Clone, PartialEq)]
39pub struct ExportedDocument {
40    /// The database the key names, `(default)` when it names none.
41    pub database_id: String,
42    pub path: ResourcePath,
43    /// References relative to the importing project have an empty project
44    /// (`projects//databases/…`); see [`Self::fields_in`].
45    pub fields: Fields,
46}
47
48impl ExportedDocument {
49    /// The fields as imported into `project`: references relative to the importing project
50    /// (top-level indexed properties, see [`entity`]) point into it.
51    pub fn fields_in(&self, project: &str) -> Fields {
52        fn resolve(value: &mut Value, project: &str) {
53            match &mut value.value_type {
54                Some(ValueType::ReferenceValue(name)) => {
55                    if let Some(rest) = name.strip_prefix("projects//") {
56                        *name = format!("projects/{project}/{rest}");
57                    }
58                }
59                Some(ValueType::ArrayValue(array)) => {
60                    array.values.iter_mut().for_each(|v| resolve(v, project));
61                }
62                _ => {}
63            }
64        }
65        let mut fields = self.fields.clone();
66        fields.values_mut().for_each(|v| resolve(v, project));
67        fields
68    }
69}
70
71/// An `OverallExportMetadata` pointer to a kind directory's metadata.
72#[derive(Debug, Clone, PartialEq, Eq)]
73pub struct Pointer {
74    /// Relative to the export directory.
75    pub path: String,
76    pub entities: u64,
77    /// The size of the kind's output files.
78    pub bytes: u64,
79}
80
81/// The contents of `{name}.overall_export_metadata`.
82pub fn overall_metadata(pointer: Option<&Pointer>) -> Vec<u8> {
83    let mut w = Writer::default();
84    if let Some(pointer) = pointer {
85        w.message(1, |p| {
86            // Constant in every export the official emulator writes.
87            p.message(1, |format| {
88                format.uint(1, 2);
89                format.uint(3, 3);
90            });
91            p.bytes(2, pointer.path.as_bytes());
92            p.uint(3, pointer.entities);
93            p.uint(4, pointer.bytes);
94        });
95    }
96    log::write_all([&[VERSION][..], &w.into_bytes()])
97}
98
99/// Why an overall metadata file was refused.
100#[derive(Debug, Clone, Copy, PartialEq, Eq)]
101pub enum OverallError {
102    /// The first record is missing or is not [`VERSION`].
103    Version,
104    Malformed,
105}
106
107/// The kind metadata paths an overall metadata file points at, relative to its directory.
108pub fn parse_overall_metadata(data: &[u8]) -> Result<Vec<String>, OverallError> {
109    let mut records = log::records(data);
110    match records.next() {
111        Some(Ok(version)) if *version == [VERSION] => {}
112        _ => return Err(OverallError::Version),
113    }
114    let Some(Ok(metadata)) = records.next() else {
115        return Err(OverallError::Malformed);
116    };
117    let mut paths = Vec::new();
118    for field in Reader::new(&metadata) {
119        let (number, pointer) = field.map_err(|_| OverallError::Malformed)?;
120        if number != 1 {
121            continue;
122        }
123        for field in pointer.fields().ok_or(OverallError::Malformed)? {
124            let (number, value) = field.map_err(|_| OverallError::Malformed)?;
125            if number == 2 {
126                let path = value.as_bytes().ok_or(OverallError::Malformed)?;
127                paths.push(String::from_utf8(path.to_vec()).map_err(|_| OverallError::Malformed)?);
128            }
129        }
130    }
131    Ok(paths)
132}
133
134/// The contents of a kind directory's `.export_metadata`.
135pub fn export_metadata(name: &str, start_micros: i64, end_micros: i64) -> Vec<u8> {
136    let mut w = Writer::default();
137    w.message(1, |m| {
138        m.bytes(1, name.as_bytes());
139        m.int(2, start_micros);
140        m.int(3, end_micros);
141    });
142    w.message(2, |files| {
143        files.bytes(1, b"");
144        files.bytes(2, OUTPUT.as_bytes());
145    });
146    w.into_bytes()
147}
148
149/// The output files a kind's metadata lists, relative to its directory.
150pub fn parse_export_metadata(data: &[u8]) -> Result<Vec<String>, Malformed> {
151    let mut outputs = Vec::new();
152    for field in Reader::new(data) {
153        let (number, files) = field?;
154        if number != 2 {
155            continue;
156        }
157        let mut prefix = String::new();
158        for field in files.fields().ok_or(Malformed)? {
159            let (number, value) = field?;
160            let text = String::from_utf8(value.as_bytes().ok_or(Malformed)?.to_vec())
161                .map_err(|_| Malformed)?;
162            match number {
163                1 => prefix = text,
164                2 if prefix.is_empty() => outputs.push(text),
165                2 => outputs.push(format!("{}/{text}", prefix.trim_end_matches('/'))),
166                _ => {}
167            }
168        }
169    }
170    Ok(outputs)
171}
172
173#[cfg(test)]
174mod tests {
175    use super::*;
176
177    #[test]
178    fn an_empty_export_points_at_nothing() {
179        let data = overall_metadata(None);
180        assert_eq!(parse_overall_metadata(&data), Ok(vec![]));
181        assert_eq!(data.len(), 15);
182    }
183
184    #[test]
185    fn metadata_round_trips() {
186        let pointer = Pointer {
187            path: KIND_METADATA.to_owned(),
188            entities: 9,
189            bytes: 4928,
190        };
191        assert_eq!(
192            parse_overall_metadata(&overall_metadata(Some(&pointer))),
193            Ok(vec![KIND_METADATA.to_owned()])
194        );
195        assert_eq!(
196            parse_export_metadata(&export_metadata("n", 1, 2)),
197            Ok(vec![OUTPUT.to_owned()])
198        );
199    }
200
201    #[test]
202    fn another_version_is_refused() {
203        assert_eq!(
204            parse_overall_metadata(&log::write_all([&[0x34u8][..], &[]])),
205            Err(OverallError::Version)
206        );
207        assert_eq!(
208            parse_overall_metadata(b"garbage"),
209            Err(OverallError::Version)
210        );
211    }
212}
hidane_core/export/mod.rs

hidane_core/export/
mod.rs
