glean_core/storage/
mod.rs

1// This Source Code Form is subject to the terms of the Mozilla Public
2// License, v. 2.0. If a copy of the MPL was not distributed with this
3// file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
5#![allow(non_upper_case_globals)]
6
7//! Storage snapshotting.
8
9use std::collections::HashMap;
10
11use serde_json::{json, Value as JsonValue};
12
13use crate::coverage::record_coverage;
14use crate::database::Database;
15use crate::metrics::dual_labeled_counter::RECORD_SEPARATOR;
16use crate::metrics::Metric;
17use crate::Lifetime;
18
19// An internal ping name, not to be touched by anything else
20pub(crate) const INTERNAL_STORAGE: &str = "glean_internal_info";
21
22/// Snapshot metrics from the underlying database.
23pub struct StorageManager;
24
25/// Labeled metrics are stored as `<metric id>/<label>`.
26/// They need to go into a nested object in the final snapshot.
27///
28/// We therefore extract the metric id and the label from the key and construct the new object or
29/// add to it.
30fn snapshot_labeled_metrics(
31    snapshot: &mut HashMap<String, HashMap<String, JsonValue>>,
32    metric_id: &str,
33    metric: &Metric,
34) {
35    let ping_section = format!("labeled_{}", metric.ping_section());
36    let map = snapshot.entry(ping_section).or_default();
37
38    // Safe unwrap, the function is only called when the id does contain a '/'
39    let (metric_id, label) = metric_id.split_once('/').unwrap();
40
41    let obj = map.entry(metric_id.into()).or_insert_with(|| json!({}));
42    let obj = obj.as_object_mut().unwrap(); // safe unwrap, we constructed the object above
43    obj.insert(label.into(), metric.as_json());
44}
45
46/// Dual Labeled metrics are stored as `<metric id><\x1e><key><\x1e><category>`.
47/// They need to go into a nested object in the final snapshot.
48///
49/// We therefore extract the metric id and the label from the key and construct the new object or
50/// add to it.
51fn snapshot_dual_labeled_metrics(
52    snapshot: &mut HashMap<String, HashMap<String, JsonValue>>,
53    metric_id: &str,
54    metric: &Metric,
55) {
56    let ping_section = format!("dual_labeled_{}", metric.ping_section());
57    let map = snapshot.entry(ping_section).or_default();
58    let parts = metric_id.split(RECORD_SEPARATOR).collect::<Vec<&str>>();
59
60    let obj = map
61        .entry(parts[0].into())
62        .or_insert_with(|| json!({}))
63        .as_object_mut()
64        .unwrap(); // safe unwrap, we constructed the object above
65    let key_obj = obj.entry(parts[1].to_string()).or_insert_with(|| json!({}));
66    let key_obj = key_obj.as_object_mut().unwrap();
67    key_obj.insert(parts[2].into(), metric.as_json());
68}
69
70impl StorageManager {
71    /// Snapshots the given store and optionally clear it.
72    ///
73    /// # Arguments
74    ///
75    /// * `storage` - the database to read from.
76    /// * `store_name` - the store to snapshot.
77    /// * `clear_store` - whether to clear the data after snapshotting.
78    ///
79    /// # Returns
80    ///
81    /// The stored data in a string encoded as JSON.
82    /// If no data for the store exists, `None` is returned.
83    pub fn snapshot(
84        &self,
85        storage: &Database,
86        store_name: &str,
87        clear_store: bool,
88    ) -> Option<String> {
89        self.snapshot_as_json(storage, store_name, clear_store)
90            .map(|data| ::serde_json::to_string_pretty(&data).unwrap())
91    }
92
93    /// Snapshots the given store and optionally clear it.
94    ///
95    /// # Arguments
96    ///
97    /// * `storage` - the database to read from.
98    /// * `store_name` - the store to snapshot.
99    /// * `clear_store` - whether to clear the data after snapshotting.
100    ///
101    /// # Returns
102    ///
103    /// A JSON representation of the stored data.
104    /// If no data for the store exists, `None` is returned.
105    pub fn snapshot_as_json(
106        &self,
107        storage: &Database,
108        store_name: &str,
109        clear_store: bool,
110    ) -> Option<JsonValue> {
111        let mut snapshot: HashMap<String, HashMap<String, JsonValue>> = HashMap::new();
112
113        let mut snapshotter = |metric_id: &[u8], metric: &Metric| {
114            let metric_id = String::from_utf8_lossy(metric_id).into_owned();
115            if metric_id.contains('/') {
116                snapshot_labeled_metrics(&mut snapshot, &metric_id, metric);
117            } else if metric_id.split(RECORD_SEPARATOR).count() == 3 {
118                snapshot_dual_labeled_metrics(&mut snapshot, &metric_id, metric);
119            } else {
120                let map = snapshot.entry(metric.ping_section().into()).or_default();
121                map.insert(metric_id, metric.as_json());
122            }
123        };
124
125        storage.iter_store_from(Lifetime::Ping, store_name, None, &mut snapshotter);
126        storage.iter_store_from(Lifetime::Application, store_name, None, &mut snapshotter);
127        storage.iter_store_from(Lifetime::User, store_name, None, &mut snapshotter);
128
129        // Add send in all pings client.annotations
130        if store_name != "glean_client_info" {
131            storage.iter_store_from(Lifetime::Application, "all-pings", None, snapshotter);
132        }
133
134        if clear_store {
135            if let Err(e) = storage.clear_ping_lifetime_storage(store_name) {
136                log::warn!("Failed to clear lifetime storage: {:?}", e);
137            }
138        }
139
140        if snapshot.is_empty() {
141            None
142        } else {
143            Some(json!(snapshot))
144        }
145    }
146
147    /// Gets the current value of a single metric identified by name.
148    ///
149    /// # Arguments
150    ///
151    /// * `storage` - The database to get data from.
152    /// * `store_name` - The store name to look into.
153    /// * `metric_id` - The full metric identifier.
154    ///
155    /// # Returns
156    ///
157    /// The decoded metric or `None` if no data is found.
158    pub fn snapshot_metric(
159        &self,
160        storage: &Database,
161        store_name: &str,
162        metric_id: &str,
163        metric_lifetime: Lifetime,
164    ) -> Option<Metric> {
165        let mut snapshot: Option<Metric> = None;
166
167        let mut snapshotter = |id: &[u8], metric: &Metric| {
168            let id = String::from_utf8_lossy(id).into_owned();
169            if id == metric_id {
170                snapshot = Some(metric.clone())
171            }
172        };
173
174        storage.iter_store_from(metric_lifetime, store_name, None, &mut snapshotter);
175
176        snapshot
177    }
178
179    /// Gets the current value of a single metric identified by name.
180    ///
181    /// Use this API, rather than `snapshot_metric` within the testing API, so
182    /// that the usage will be reported in coverage, if enabled.
183    ///
184    /// # Arguments
185    ///
186    /// * `storage` - The database to get data from.
187    /// * `store_name` - The store name to look into.
188    /// * `metric_id` - The full metric identifier.
189    ///
190    /// # Returns
191    ///
192    /// The decoded metric or `None` if no data is found.
193    pub fn snapshot_metric_for_test(
194        &self,
195        storage: &Database,
196        store_name: &str,
197        metric_id: &str,
198        metric_lifetime: Lifetime,
199    ) -> Option<Metric> {
200        record_coverage(metric_id);
201        self.snapshot_metric(storage, store_name, metric_id, metric_lifetime)
202    }
203
204    ///  Snapshots the experiments.
205    ///
206    /// # Arguments
207    ///
208    /// * `storage` - The database to get data from.
209    /// * `store_name` - The store name to look into.
210    ///
211    /// # Returns
212    ///
213    /// A JSON representation of the experiment data, in the following format:
214    ///
215    /// ```json
216    /// {
217    ///  "experiment-id": {
218    ///    "branch": "branch-id",
219    ///    "extra": {
220    ///      "additional": "property",
221    ///      // ...
222    ///    }
223    ///  }
224    /// }
225    /// ```
226    ///
227    /// If no data for the store exists, `None` is returned.
228    pub fn snapshot_experiments_as_json(
229        &self,
230        storage: &Database,
231        store_name: &str,
232    ) -> Option<JsonValue> {
233        let mut snapshot: HashMap<String, JsonValue> = HashMap::new();
234
235        let mut snapshotter = |metric_id: &[u8], metric: &Metric| {
236            let metric_id = String::from_utf8_lossy(metric_id).into_owned();
237            if metric_id.ends_with("#experiment") {
238                let (name, _) = metric_id.split_once('#').unwrap(); // safe unwrap, we ensured there's a `#` in the string
239                snapshot.insert(name.to_string(), metric.as_json());
240            }
241        };
242
243        storage.iter_store_from(Lifetime::Application, store_name, None, &mut snapshotter);
244
245        if snapshot.is_empty() {
246            None
247        } else {
248            Some(json!(snapshot))
249        }
250    }
251}
252
253#[cfg(test)]
254mod test {
255    use super::*;
256    use crate::metrics::ExperimentMetric;
257    use crate::Glean;
258
259    // Experiment's API tests: the next test comes from glean-ac's
260    // ExperimentsStorageEngineTest.kt.
261    #[test]
262    fn test_experiments_json_serialization() {
263        let t = tempfile::tempdir().unwrap();
264        let name = t.path().display().to_string();
265        let glean = Glean::with_options(&name, "org.mozilla.glean", true, true);
266
267        let extra: HashMap<String, String> = [("test-key".into(), "test-value".into())]
268            .iter()
269            .cloned()
270            .collect();
271
272        let metric = ExperimentMetric::new(&glean, "some-experiment".to_string());
273
274        metric.set_active_sync(&glean, "test-branch".to_string(), extra);
275        let snapshot = StorageManager
276            .snapshot_experiments_as_json(glean.storage(), "glean_internal_info")
277            .unwrap();
278        assert_eq!(
279            json!({"some-experiment": {"branch": "test-branch", "extra": {"test-key": "test-value"}}}),
280            snapshot
281        );
282
283        metric.set_inactive_sync(&glean);
284
285        let empty_snapshot =
286            StorageManager.snapshot_experiments_as_json(glean.storage(), "glean_internal_info");
287        assert!(empty_snapshot.is_none());
288    }
289
290    #[test]
291    fn test_experiments_json_serialization_empty() {
292        let t = tempfile::tempdir().unwrap();
293        let name = t.path().display().to_string();
294        let glean = Glean::with_options(&name, "org.mozilla.glean", true, true);
295
296        let metric = ExperimentMetric::new(&glean, "some-experiment".to_string());
297
298        metric.set_active_sync(&glean, "test-branch".to_string(), HashMap::new());
299        let snapshot = StorageManager
300            .snapshot_experiments_as_json(glean.storage(), "glean_internal_info")
301            .unwrap();
302        assert_eq!(
303            json!({"some-experiment": {"branch": "test-branch"}}),
304            snapshot
305        );
306
307        metric.set_inactive_sync(&glean);
308
309        let empty_snapshot =
310            StorageManager.snapshot_experiments_as_json(glean.storage(), "glean_internal_info");
311        assert!(empty_snapshot.is_none());
312    }
313}
glean_core/storage/mod.rs

glean_core/storage/
mod.rs
