znippy-common 0.9.14

Core logic and data structures for Znippy, a parallel chunked compression system.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

extern crate core;

/// Re-export arrow so plugins implement `schema_fields()` against the exact same arrow
/// version as the core trait, avoiding type-mismatch across crate boundaries.
pub use arrow;

/// Re-export blake3 for the same reason arrow is re-exported: the archive's
/// per-chunk `checksum` column is blake3 over the chunk's bytes, and a writer
/// living in another crate (`znippy-plugin-git`'s generation-0 seal) must fill
/// that column with the **same** implementation the reader checks it against.
/// Pinning it here rather than re-declaring the dependency there is what makes
/// a version skew between the two impossible rather than merely unlikely.
pub use blake3;

/// **Introspection / emit marker** — record one functional-status row for the
/// nornir test matrix. Wraps `nornir_testmatrix::functional_status` behind the
/// `testmatrix` feature (a compiled-out `#[inline]` no-op otherwise, with no
/// nornir dep in the default build). `component` is the reporting surface (e.g.
/// `"znippy-common/decompress"`), `check` what it verified, `ok` the REAL
/// verdict, `detail` a short measured note. The decompress path + the Magazine
/// slot-lifecycle self-tests call this so `nornir test --features testmatrix`
/// SEES each surface's health. Defined only under the feature — every call site
/// is itself `#[cfg(feature = "testmatrix")]` (so the detail-string work is
/// stripped in the default build, per the nornir-testmatrix doctrine), so there
/// is no non-feature caller to no-op for.
#[cfg(feature = "testmatrix")]
#[inline]
pub(crate) fn functional_status(component: &str, check: &str, ok: bool, detail: &str) {
    nornir_testmatrix::functional_status(component, check, ok, detail);
}

pub mod codec;
pub mod common_config;
/// Host memory and CPU facts read straight from `/proc` and `/sys/fs/cgroup`.
/// Replaces the `sysinfo` runtime dependency, which reached this crate's whole
/// consumer set — and, via its `multithread` feature, put `rayon` in it.
pub mod hostinfo;
pub mod index;
/// Whether a payload is already compressed, and so should be stored raw.
/// Resolution order: an explicit caller hint, then the extension table, then a
/// magic-byte probe over the head of the real bytes.
pub mod precompressed;
pub mod archive;
pub mod meta_sink;
pub mod meta_sink_append;
/// Searchable archive metadata: the `META_MODULE` sub-index, its typed key/value
/// model, and the query API — "which entries carry key X" answered from an
/// index, at a cost set by the metadata, not by the payload.
pub mod meta_index;

pub mod slotpool;

pub mod meta;
pub use meta::{BlobMeta, ChunkMeta, FileMeta};

/// Detached, streaming CMS provenance signatures (per-artifact + per-archive).
/// Off by default; default builds and the on-disk format are byte-unchanged.
#[cfg(feature = "sign")]
pub mod sign;

pub mod plugin;
pub mod plugins;
pub mod views;

pub mod decompress;

/// The DYNAMIC half of the archive: append at O(1), seal once. See `hot`.
pub mod hot;

pub use meta_sink_append::{
    compact_archive, read_delta_map, supersede_as_delta, CompactReport, SupersedeOutcome,
    MAX_GENERATION_CHAIN,
};
pub use archive::{
    BaseResolve, ChunkSplitter, DeltaSplitter, Entry, ReconstructCtx, RegionDeltaSplitter,
    Splitter, StoredUnit, UnitPayload, ZnippyArchive, ZnippyReader,
};
pub use views::{
    CondaPackage, CondaView, GemPackage, GemView, MavenPackage, MavenView, NpmPackage, NpmView,
    PythonKind, PythonPackage, PythonView, RustPackage, RustView, CONDA_PKG_TYPE, GEM_PKG_TYPE,
    MAVEN_PKG_TYPE, NPM_PKG_TYPE, PYTHON_PKG_TYPE, RUST_PKG_TYPE,
};
pub use decompress::{decompress_archive, decompress_archive_filtered, get_file};
pub use meta_sink::{
    ArchiveMetaSink, ArrowIpcSink, GroupKey, LookupView, MetaSinkFactory, ReservedPayload,
    ReservedSection, ReservedSectionBuilder,
};
pub use hot::{HotAppendReport, HotArchive, HotSealReport, seal_hot};
pub use meta_sink_append::{
    append_files_with_policy,
    AppendReport, ArrowIpcSinkAppend, append_files, append_files_with_meta, base_batch_from_rows,
    create_archive, create_archive_to_vec, create_archive_with_meta,
};
/// Searchable metadata: seal typed key/value facts with an archive and answer
/// "which entries carry key X" from the index alone. `ArchiveMeta::NoMetadata`
/// is a distinct state from a present-but-empty index — see `meta_index`.
pub use meta_index::{
    ArchiveMeta, MetaEntry, MetaIndex, MetaSearch, MetaTable, MetaValue, read_archive_meta,
};

pub use precompressed::{ContentHint, SkipPolicy, SNIFF_PREFIX_LEN, is_zlib_stream, looks_compressed};

pub use index::{
    ArchiveReader,
    ArtifactMeta, ChunkLoc, IndexFilter, VerifyReport, ZNIPPY_INDEX_SCHEMA,
    MULTI_INDEX_MAGIC, ManifestEntry, IndexFooter,
    META_MODULE, build_arrow_metadata_for_config, build_metadata_batch, data_subindex_schema,
    extract_config_from_arrow_metadata, get_all_files_meta, get_files_meta_with_prefix,
    interpret_footer,
    is_probably_compressed, is_reserved_module, list_archive_contents, locate_file,
    read_manifest_bytes, read_reserved_section_bytes, read_znippy_full_manifest, read_znippy_index,
    read_znippy_index_filtered, read_znippy_manifest,
    should_skip_compression, verify_archive_integrity, write_manifest_bytes,
    znippy_index_schema, SIGN_ARCHIVE_MODULE, SIGN_ARTIFACTS_MODULE,
    GUNNAR_GRAPH_MODULE, GUNNAR_OID_MODULE, GUNNAR_REACH_MODULE, GUNNAR_REFS_MODULE,
    GUNNAR_SECRETS_MODULE, LOOKUP_MODULE,
    RESERVED_MODULES, RESERVED_PKG_TYPE, TRIE_MODULE, lookup_schema,
};

#[derive(Debug)]
pub struct CompressionReport {
    pub total_files: u64,
    pub compressed_files: u64,
    pub uncompressed_files: u64,
    pub total_dirs: u64,
    pub total_bytes_in: u64,
    pub total_bytes_out: u64,
    pub compressed_bytes: u64,
    pub uncompressed_bytes: u64,
    pub compression_ratio: f32,
    pub chunks: u64,
    /// Files that the walk enumerated but whose bytes never reached a committed
    /// chunk (open/read failed and the file was silently dropped). Derived as
    /// `total_files - (compressed_files + uncompressed_files)`. A green compress
    /// surface REQUIRES `files_failed == 0` — see `compress_reporting`.
    pub files_failed: u64,
}