datui_lib/analysis/
quality_memory.rs

1//! What Data Quality keeps for the session: its reports, the rows its sampled runs
2//! read, and local copies of remote sources.
3
4use std::path::PathBuf;
5use std::sync::Arc;
6
7use crate::{analysis::data_quality, analysis::sampling};
8
9/// A Data Quality report, by everything its plan says: the acquisition it measured
10/// and the report's own choices.
11pub(crate) struct QualityCacheEntry {
12    pub(crate) dataset_generation: u64,
13    pub(crate) view_generation: u64,
14    pub(crate) plan: data_quality::DataQualityPlan,
15    pub(crate) results: data_quality::DataQualityResults,
16    pub(crate) bytes: usize,
17}
18
19/// Rows a sampled Data Quality run read, and what decided which rows they were: its
20/// acquisition identity. A run or a drill that names the same rows cuts these instead
21/// of reading.
22///
23/// The dataset and view generations stand for the source: a session snapshot of
24/// the dataset as opened and the view as it was. A file changed on disk since is not
25/// noticed; opening it again starts a new dataset generation, and reads it again.
26#[derive(Debug, Clone)]
27pub struct KeptQualitySample {
28    pub(crate) dataset_generation: u64,
29    pub(crate) view_generation: u64,
30    pub(crate) sample: sampling::Sample,
31    pub(crate) rows: std::sync::Arc<data_quality::QualitySample>,
32    /// The source as the run that read these rows found it, stated when it began: a
33    /// report measured on them later is labeled with this, not with the file as it
34    /// stands then.
35    pub(crate) source: crate::analysis::quality_export::SourceIdentity,
36}
37
38impl KeptQualitySample {
39    pub(crate) fn same_rows(&self, other: &Self) -> bool {
40        self.dataset_generation == other.dataset_generation
41            && self.view_generation == other.view_generation
42            && self.sample == other.sample
43    }
44}
45
46/// Memory Data Quality keeps for the session: the rows its sampled runs read and the
47/// reports they made. Past it, reports that retained rows can remake go first, then
48/// the oldest rows, then the oldest other reports; the newest of each always stays.
49pub const QUALITY_MEMORY_BUDGET: usize = 256 * 1024 * 1024;
50
51/// Where a full scan's passes read a remote source from, decided at Run.
52pub(crate) enum QualityCopyJob {
53    /// The source, in each pass.
54    Source,
55    /// A copy fetched earlier.
56    Kept(Arc<crate::cloud::local_copy::LocalCopy>),
57    /// A copy of `objects` fetched under `root` first.
58    Fetch {
59        objects: Vec<crate::cloud::local_copy::RemoteObject>,
60        root: PathBuf,
61    },
62}
63
64/// A local copy of a dataset's remote objects, kept for later full scans of the
65/// dataset it was fetched for.
66#[derive(Debug, Clone)]
67pub struct RetainedCopy {
68    pub(crate) dataset_generation: u64,
69    pub(crate) copy: Arc<crate::cloud::local_copy::LocalCopy>,
70}
71
72/// Acquisitions released to the budget that Setup still names, so a Run that reads
73/// them again says why.
74pub(crate) const QUALITY_RELEASED_REMEMBERED: usize = 16;
datui_lib/analysis/quality_memory.rs

datui_lib/analysis/
quality_memory.rs
