zahirscan 0.3.3

Token-efficient content compression for AI analysis using probabilistic template mining
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

//! Shared literals and numeric limits for structured parsers (CSV, columnar binaries, MTX, NPY).

/// Serialized `encoding` and related labels in columnar / CSV metadata JSON.
pub struct StructuredEncoding;

impl StructuredEncoding {
    pub const TABULAR_BINARY: &'static str = "binary";
    pub const NUMPY: &'static str = "numpy";
    pub const MATLAB: &'static str = "matlab";
    pub const MATRIX_MARKET: &'static str = "matrix-market";
    /// Serialized label for ONNX (`.onnx`) graph reports when shared with other structured types.
    pub const ONNX: &'static str = "onnx";
}

/// [`crate::results::CsvMetadata::encoding`]-style hints for delimiter-separated text.
pub struct CsvEncodingLabel;

impl CsvEncodingLabel {
    pub const UTF8: &'static str = "UTF-8";
    pub const NON_UTF8: &'static str = "Non-UTF-8";
}

/// [`crate::results::ArrowIpcMetadata::container_kind`] values.
pub struct ArrowIpcContainerKind;

impl ArrowIpcContainerKind {
    pub const IPC_FILE: &'static str = "ipc_file";
    pub const IPC_STREAM: &'static str = "ipc_stream";
    pub const FEATHER: &'static str = "feather";
}

/// Matrix Market [`crate::results::MtxMetadata::symmetry`] strings.
pub struct MtxSymmetryLabel;

impl MtxSymmetryLabel {
    pub const GENERAL: &'static str = "general";
    pub const SYMMETRIC: &'static str = "symmetric";
}

/// Numeric caps and thresholds shared across structured parsers.
pub mod limits {
    /// Column scaling numerator: `effective = base * N / max(cols, N)` for CSV and tabular binaries.
    pub const TABULAR_COL_SCALE_NUMERATOR: usize = 4000;
    /// CSV pass-2 and wide MTX string-inference column chunk width (`HashSet` / buffers per chunk).
    pub const TABULAR_COLUMN_CHUNK: usize = 256;
    /// File size (bytes) above which tabular row samples use per-decade retention scaling (`10^5`).
    pub const TABULAR_SAMPLE_BYTE_THRESHOLD: u64 = 100_000;
    /// Floor on retained sample fraction after file-size decade scaling.
    pub const TABULAR_BYTE_SCALE_MIN_RETAIN_FRAC: f64 = 0.03;
    /// Per decade above [`TABULAR_SAMPLE_BYTE_THRESHOLD`], subtract this fraction from 100% retention (e.g. 2% → 98%, 96%, …).
    pub const TABULAR_BYTE_SCALE_PCT_PER_DECADE: f64 = 0.02;

    /// Average bytes per row (`file_bytes / row_count`) at or below this is treated as **skinny** rows: large
    /// logical tables with small on-disk size skip harsh byte-decade shrink when other full-scan conditions hold.
    pub const TABULAR_BPR_SKINNY_MAX_BYTES: u64 = 2048;
    /// When row count is known, allow scanning **all** rows (subject to [`TABULAR_FULL_SCAN_MAX_ROWS`]) only if
    /// the file is at most this many bytes (cheap full read for stats).
    pub const TABULAR_FULL_SCAN_MAX_FILE_BYTES: u64 = 64 * 1024 * 1024;
    /// Max rows for the full-scan path (avoids multi-million-row tabular stats on pathological skinny files).
    pub const TABULAR_FULL_SCAN_MAX_ROWS: usize = 500_000;

    /// Back-compat alias for [`TABULAR_SAMPLE_BYTE_THRESHOLD`].
    pub const CSV_SAMPLE_BYTE_THRESHOLD: u64 = TABULAR_SAMPLE_BYTE_THRESHOLD;
    /// Back-compat alias for [`TABULAR_BYTE_SCALE_MIN_RETAIN_FRAC`].
    pub const CSV_BYTE_SCALE_MIN_RETAIN_FRAC: f64 = TABULAR_BYTE_SCALE_MIN_RETAIN_FRAC;
    /// Back-compat alias for [`TABULAR_BYTE_SCALE_PCT_PER_DECADE`].
    pub const CSV_BYTE_SCALE_PCT_PER_DECADE: f64 = TABULAR_BYTE_SCALE_PCT_PER_DECADE;
    /// Cap on infer-sample `rows × cols` string grid for MTX.
    pub const MAX_MTX_INFERENCE_STRING_CELLS: usize = 256_000;
    /// Max `rows × cols` for full logical sparse materialization in MTX numeric stats.
    pub const MAX_MTX_TABULAR_CELLS: usize = 8_000_000;

    /// Base cap on 3D tensor **plane** summary rows; [`super::tensor3d_max_reported_planes`] may
    /// increase this for large `n_along` (capped at [`TENSOR3D_MAX_REPORTED_PLANES_LOG_CAP`]).
    pub const TENSOR3D_MAX_PLANES: usize = 32;
    /// Upper bound for [`super::tensor3d_max_reported_planes`].
    pub const TENSOR3D_MAX_REPORTED_PLANES_LOG_CAP: usize = 64;
    /// Max linear element visits across the whole 3D tensor (after subsampling stride).
    pub const TENSOR3D_MAX_LINEAR_SAMPLES: usize = 2_000_000;
    /// Max samples taken within one plane (each plane is strided to stay within this).
    pub const TENSOR3D_MAX_PLANE_LINEAR_SAMPLES: usize = 200_000;
}

/// `n_along` = array extent along the chosen 3D stack axis.
/// Returns `32` for `n_along` ≤ 10^3, then +3 for each order of 10 (10^4 → 35, 10^5 → 38, …), capped.
#[must_use]
pub fn tensor3d_max_reported_planes(n_along: usize) -> usize {
    if n_along == 0 {
        return 0;
    }
    let d = n_along.ilog10().saturating_sub(3);
    (limits::TENSOR3D_MAX_PLANES + d as usize * 3).min(limits::TENSOR3D_MAX_REPORTED_PLANES_LOG_CAP)
}