1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
//! What Data Quality keeps for the session: its reports, the rows its sampled runs
//! read, and local copies of remote sources.
use std::path::PathBuf;
use std::sync::Arc;
use crate::{data_quality, sampling};
/// A Data Quality report, by everything its plan says: the acquisition it measured
/// and the report's own choices.
pub(crate) struct QualityCacheEntry {
pub(crate) dataset_generation: u64,
pub(crate) view_generation: u64,
pub(crate) plan: data_quality::DataQualityPlan,
pub(crate) results: data_quality::DataQualityResults,
pub(crate) bytes: usize,
}
/// Rows a sampled Data Quality run read, and what decided which rows they were: its
/// acquisition identity. A run or a drill that names the same rows cuts these instead
/// of reading.
///
/// The dataset and view generations stand for the source: a session snapshot of
/// the dataset as opened and the view as it was. A file changed on disk since is not
/// noticed; opening it again starts a new dataset generation, and reads it again.
#[derive(Debug, Clone)]
pub struct KeptQualitySample {
pub(crate) dataset_generation: u64,
pub(crate) view_generation: u64,
pub(crate) sample: sampling::Sample,
pub(crate) rows: std::sync::Arc<data_quality::QualitySample>,
/// The source as the run that read these rows found it, stated when it began: a
/// report measured on them later is labeled with this, not with the file as it
/// stands then.
pub(crate) source: crate::quality_export::SourceIdentity,
}
impl KeptQualitySample {
pub(crate) fn same_rows(&self, other: &Self) -> bool {
self.dataset_generation == other.dataset_generation
&& self.view_generation == other.view_generation
&& self.sample == other.sample
}
}
/// Memory Data Quality keeps for the session: the rows its sampled runs read and the
/// reports they made. Past it, reports that retained rows can remake go first, then
/// the oldest rows, then the oldest other reports; the newest of each always stays.
pub const QUALITY_MEMORY_BUDGET: usize = 256 * 1024 * 1024;
/// Where a full scan's passes read a remote source from, decided at Run.
pub(crate) enum QualityCopyJob {
/// The source, in each pass.
Source,
/// A copy fetched earlier.
Kept(Arc<crate::local_copy::LocalCopy>),
/// A copy of `objects` fetched under `root` first.
Fetch {
objects: Vec<crate::local_copy::RemoteObject>,
root: PathBuf,
},
}
/// A local copy of a dataset's remote objects, kept for later full scans of the
/// dataset it was fetched for.
#[derive(Debug, Clone)]
pub struct RetainedCopy {
pub(crate) dataset_generation: u64,
pub(crate) copy: Arc<crate::local_copy::LocalCopy>,
}
/// Acquisitions released to the budget that Setup still names, so a Run that reads
/// them again says why.
pub(crate) const QUALITY_RELEASED_REMEMBERED: usize = 16;