Skip to main content

oak_core/sqlite/
tree_inspect.rs

1//! Bounded whole-tree evidence for one pinned commit (fb-519, fb-522).
2//!
3//! Every tree is decoded and hash-verified, every listed file's logical bytes
4//! are streamed through BLAKE3 (Oak identity) and SHA-256 (portable digest),
5//! and budgets stop the listing at an explicit, deterministic truncation
6//! point instead of silently omitting paths. Like file inspection, this
7//! requires `open_read_only`: HEAD, commit, trees and blobs share one read
8//! snapshot, and nothing is written, migrated or hydrated.
9use super::file_inspect::{
10    budget, check_inspectable_schema, database, failure, require_read_only, resolve_revision,
11    validate_revision, verified_commit_root, verified_tree, verify_blob_into, Checked, Failure,
12    FileInspectionProofScope, InspectionStatus, VerifiedBlob,
13};
14use super::SqliteRepository;
15use crate::{hash_bytes, FileMode, Hash, OakError, Result, Tree, TreeEntryKind};
16use rusqlite::{Connection, OptionalExtension};
17use serde::Serialize;
18use std::collections::HashMap;
19
20pub const DEFAULT_TREE_MAX_FILES: u64 = 100_000;
21pub const MAX_TREE_MAX_FILES: u64 = 10_000_000;
22pub const DEFAULT_TREE_MAX_BYTES: u64 = 1024 * 1024 * 1024;
23pub const MAX_TREE_MAX_BYTES: u64 = 1 << 40;
24/// Aggregate decoded canonical tree bytes for one inspection.
25const WHOLE_TREE_BYTES: u64 = 256 * 1024 * 1024;
26const MAX_DEPTH: usize = 128;
27const MAX_PATH_BYTES: usize = 4096;
28
29#[derive(Debug, Clone, Copy)]
30pub struct TreeInspectionLimits {
31    pub max_files: u64,
32    pub max_bytes: u64,
33}
34
35impl Default for TreeInspectionLimits {
36    fn default() -> Self {
37        Self {
38            max_files: DEFAULT_TREE_MAX_FILES,
39            max_bytes: DEFAULT_TREE_MAX_BYTES,
40        }
41    }
42}
43
44#[derive(Debug, Serialize)]
45pub struct TreeInspection {
46    pub requested_revision: String,
47    pub commit: Option<String>,
48    pub commit_verification: &'static str,
49    pub root_tree: Option<String>,
50    /// `verified` only when every listed tree and file verified. A budget
51    /// truncation alone leaves this `verified`; check `complete`.
52    pub status: InspectionStatus,
53    #[serde(skip_serializing_if = "Option::is_none")]
54    pub reason: Option<String>,
55    /// True only when every path under the root was listed and verified.
56    pub complete: bool,
57    pub truncated: bool,
58    #[serde(skip_serializing_if = "Option::is_none")]
59    pub truncation: Option<TreeTruncation>,
60    /// Files listed (verified or not).
61    pub file_count: u64,
62    /// Sum of listed files' verified logical sizes.
63    pub logical_bytes: u64,
64    pub verified_trees: usize,
65    /// Listing order: depth-first by canonical entry name.
66    pub order: &'static str,
67    pub max_files: u64,
68    pub max_bytes: u64,
69    pub proof_scope: FileInspectionProofScope,
70    pub files: Vec<TreeFileEvidence>,
71}
72
73#[derive(Debug, Serialize)]
74pub struct TreeTruncation {
75    /// `max_files` or `max_bytes`.
76    pub limit: &'static str,
77    /// First path not listed.
78    pub next_path: String,
79    #[serde(skip_serializing_if = "Option::is_none")]
80    pub next_declared_size: Option<u64>,
81}
82
83#[derive(Debug, Serialize)]
84pub struct TreeFileEvidence {
85    pub path: String,
86    pub mode: FileMode,
87    pub blob: String,
88    /// Verified logical size (decoded bytes, never codec bytes).
89    #[serde(skip_serializing_if = "Option::is_none")]
90    pub size: Option<u64>,
91    #[serde(skip_serializing_if = "Option::is_none")]
92    pub sha256: Option<String>,
93    pub status: InspectionStatus,
94    #[serde(skip_serializing_if = "Option::is_none")]
95    pub reason: Option<String>,
96}
97
98/// Receives each listed file's logical bytes as they are verified, so a
99/// materializer never holds a whole file in memory (fb-522).
100///
101/// Protocol per file: `begin`, zero or more `write_chunk` calls with
102/// decoded bytes that are NOT yet verified, then `finish` with the file's
103/// evidence. The bytes are trustworthy only when `finish` sees
104/// `status == verified`; otherwise the sink must discard them.
105pub trait TreeFileSink {
106    fn begin(&mut self, path: &str, mode: FileMode) -> Result<()>;
107    fn write_chunk(&mut self, bytes: &[u8]) -> std::io::Result<()>;
108    fn finish(&mut self, file: &TreeFileEvidence) -> Result<()>;
109}
110
111struct SinkWriter<'s>(&'s mut dyn TreeFileSink);
112impl std::io::Write for SinkWriter<'_> {
113    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
114        self.0.write_chunk(buf)?;
115        Ok(buf.len())
116    }
117    fn flush(&mut self) -> std::io::Result<()> {
118        Ok(())
119    }
120}
121
122impl SqliteRepository {
123    /// Verify and list one pinned commit's whole tree within `limits`.
124    pub fn inspect_pinned_tree(
125        &self,
126        revision: &str,
127        limits: TreeInspectionLimits,
128    ) -> Result<TreeInspection> {
129        self.walk_pinned_tree(revision, limits, None)
130    }
131
132    /// [`Self::inspect_pinned_tree`], streaming each file's bytes through
133    /// `visitor` (for materialization) while they are hashed. A `begin` or
134    /// `finish` error aborts the walk and is returned.
135    pub fn visit_pinned_tree(
136        &self,
137        revision: &str,
138        limits: TreeInspectionLimits,
139        visitor: &mut dyn TreeFileSink,
140    ) -> Result<TreeInspection> {
141        self.walk_pinned_tree(revision, limits, Some(visitor))
142    }
143
144    fn walk_pinned_tree(
145        &self,
146        revision: &str,
147        limits: TreeInspectionLimits,
148        visitor: Option<&mut dyn TreeFileSink>,
149    ) -> Result<TreeInspection> {
150        if limits.max_files == 0 || limits.max_files > MAX_TREE_MAX_FILES {
151            return Err(OakError::InvalidArgument(format!(
152                "--max-files must be between 1 and {MAX_TREE_MAX_FILES}"
153            )));
154        }
155        if limits.max_bytes > MAX_TREE_MAX_BYTES {
156            return Err(OakError::InvalidArgument(format!(
157                "--max-bytes must be at most {MAX_TREE_MAX_BYTES}"
158            )));
159        }
160        validate_revision(revision)?;
161        let conn = self.conn.lock().unwrap();
162        require_read_only(&conn, "tree inspection")?;
163        let mut out = TreeInspection {
164            requested_revision: revision.into(),
165            commit: None,
166            commit_verification: "not_verified",
167            root_tree: None,
168            status: InspectionStatus::Verified,
169            reason: None,
170            complete: false,
171            truncated: false,
172            truncation: None,
173            file_count: 0,
174            logical_bytes: 0,
175            verified_trees: 0,
176            order: "depth_first_by_entry_name",
177            max_files: limits.max_files,
178            max_bytes: limits.max_bytes,
179            proof_scope: FileInspectionProofScope {
180                source: "local_sqlite_snapshot",
181                exclusions: ["commit_files", "ancestor_closure", "remote_durability"],
182            },
183            files: Vec::new(),
184        };
185        let mut walk = Walk {
186            conn: &conn,
187            limits,
188            tree_budget: WHOLE_TREE_BYTES,
189            trees: HashMap::new(),
190            blobs: HashMap::new(),
191            visitor,
192            stopped: false,
193        };
194        match walk.run(&mut out) {
195            Ok(()) => {}
196            Err(WalkError::Check(Failure(status, reason))) => {
197                out.status = status;
198                out.reason = Some(reason);
199            }
200            Err(WalkError::Visitor(error)) => return Err(error),
201        }
202        out.complete = out.status == InspectionStatus::Verified && !out.truncated;
203        Ok(out)
204    }
205}
206
207enum WalkError {
208    Check(Failure),
209    Visitor(OakError),
210}
211impl From<Failure> for WalkError {
212    fn from(value: Failure) -> Self {
213        Self::Check(value)
214    }
215}
216
217struct Walk<'c, 'v> {
218    conn: &'c Connection,
219    limits: TreeInspectionLimits,
220    tree_budget: u64,
221    trees: HashMap<Hash, Tree>,
222    blobs: HashMap<Hash, VerifiedBlob>,
223    visitor: Option<&'v mut dyn TreeFileSink>,
224    stopped: bool,
225}
226
227impl Walk<'_, '_> {
228    fn run(&mut self, out: &mut TreeInspection) -> std::result::Result<(), WalkError> {
229        check_inspectable_schema(self.conn)?;
230        let revision = resolve_revision(self.conn, &out.requested_revision)?;
231        let hash = Hash::from_hex(&revision).map_err(super::file_inspect::corrupt)?;
232        out.commit = Some(hash.to_string());
233        let root = verified_commit_root(self.conn, &hash)?;
234        out.commit_verification = "verified_v1_header";
235        out.root_tree = Some(root.to_string());
236        self.dir(out, &root, "", 0)
237    }
238
239    fn tree(&mut self, hash: &Hash, out: &mut TreeInspection) -> Checked<Tree> {
240        if let Some(tree) = self.trees.get(hash) {
241            return Ok(tree.clone());
242        }
243        let tree = verified_tree(self.conn, hash, &mut self.tree_budget)?;
244        out.verified_trees += 1;
245        self.trees.insert(hash.clone(), tree.clone());
246        Ok(tree)
247    }
248
249    fn dir(
250        &mut self,
251        out: &mut TreeInspection,
252        hash: &Hash,
253        prefix: &str,
254        depth: usize,
255    ) -> std::result::Result<(), WalkError> {
256        if depth >= MAX_DEPTH {
257            return Err(budget("tree depth exceeds the 128-level inspection budget").into());
258        }
259        let tree = self.tree(hash, out)?;
260        for entry in &tree.entries {
261            if self.stopped {
262                return Ok(());
263            }
264            let path = if prefix.is_empty() {
265                entry.name.clone()
266            } else {
267                format!("{prefix}/{}", entry.name)
268            };
269            if path.len() > MAX_PATH_BYTES {
270                return Err(budget("a tree path exceeds the 4096-byte inspection budget").into());
271            }
272            match entry.kind {
273                TreeEntryKind::Tree => self.dir(out, &entry.hash, &path, depth + 1)?,
274                TreeEntryKind::Blob => self.file(out, path, entry.mode, &entry.hash)?,
275            }
276        }
277        Ok(())
278    }
279
280    fn file(
281        &mut self,
282        out: &mut TreeInspection,
283        path: String,
284        mode: FileMode,
285        blob: &Hash,
286    ) -> std::result::Result<(), WalkError> {
287        let declared = declared_blob_size(self.conn, blob)?;
288        if out.file_count >= self.limits.max_files {
289            return self.truncate(out, "max_files", path, declared);
290        }
291        if let Some(size) = declared {
292            if out.logical_bytes.saturating_add(size) > self.limits.max_bytes {
293                return self.truncate(out, "max_bytes", path, declared);
294            }
295        }
296        out.file_count += 1;
297        let remaining = self.limits.max_bytes - out.logical_bytes;
298        let mut evidence = TreeFileEvidence {
299            path,
300            mode,
301            blob: blob.to_string(),
302            size: None,
303            sha256: None,
304            status: InspectionStatus::Verified,
305            reason: None,
306        };
307        let cached = if self.visitor.is_none() {
308            self.blobs.get(blob).cloned()
309        } else {
310            None
311        };
312        let result = match cached {
313            Some(verified) => Ok(verified),
314            None => {
315                let mut declared_size = None;
316                let result = if let Some(visitor) = self.visitor.as_deref_mut() {
317                    visitor
318                        .begin(&evidence.path, mode)
319                        .map_err(WalkError::Visitor)?;
320                    verify_blob_into(
321                        self.conn,
322                        blob,
323                        remaining,
324                        &mut declared_size,
325                        &mut SinkWriter(visitor),
326                    )
327                } else {
328                    verify_blob_into(
329                        self.conn,
330                        blob,
331                        remaining,
332                        &mut declared_size,
333                        &mut std::io::sink(),
334                    )
335                };
336                if let Ok(verified) = &result {
337                    if self.visitor.is_none() {
338                        self.blobs.insert(blob.clone(), verified.clone());
339                    }
340                }
341                result
342            }
343        };
344        match result {
345            Ok(verified) => {
346                out.logical_bytes += verified.logical_size;
347                evidence.size = Some(verified.logical_size);
348                evidence.sha256 = Some(verified.sha256);
349            }
350            Err(Failure(status, reason)) => {
351                evidence.status = status;
352                evidence.reason = Some(reason.clone());
353                if out.status == InspectionStatus::Verified {
354                    out.status = status;
355                    out.reason = Some(format!("{}: {reason}", evidence.path));
356                }
357            }
358        }
359        if let Some(visitor) = self.visitor.as_deref_mut() {
360            visitor.finish(&evidence).map_err(WalkError::Visitor)?;
361        }
362        out.files.push(evidence);
363        Ok(())
364    }
365
366    fn truncate(
367        &mut self,
368        out: &mut TreeInspection,
369        limit: &'static str,
370        next_path: String,
371        next_declared_size: Option<u64>,
372    ) -> std::result::Result<(), WalkError> {
373        self.stopped = true;
374        out.truncated = true;
375        out.truncation = Some(TreeTruncation {
376            limit,
377            next_path,
378            next_declared_size,
379        });
380        Ok(())
381    }
382}
383
384/// Declared logical size, or `None` when the blob has no local row (the
385/// canonical empty blob is implicitly zero bytes).
386fn declared_blob_size(conn: &Connection, hash: &Hash) -> Checked<Option<u64>> {
387    let size: Option<i64> = conn
388        .query_row(
389            "SELECT size FROM blobs WHERE hash=?1",
390            [hash.as_str()],
391            |r| r.get(0),
392        )
393        .optional()
394        .map_err(database)?;
395    match size {
396        Some(size) => u64::try_from(size)
397            .map(Some)
398            .map_err(|_| failure(InspectionStatus::Corrupt, "negative blob size")),
399        None if hash == &hash_bytes(&[]) => Ok(Some(0)),
400        None => Ok(None),
401    }
402}