mkit_server/indexed/mod.rs
1//! Indexed ingestion shared by native inline verification and future Worker
2//! scheduling. Verification extracts large objects into the global object
3//! store before a pack's `Verified` state is written (WP-4.10).
4
5pub mod budget;
6pub mod checkpoint;
7pub mod classify;
8pub mod entries;
9mod etag;
10mod extract;
11pub mod geometry;
12pub mod inspection;
13pub mod job;
14pub mod publication;
15pub mod resolve;
16pub mod scheduled;
17mod selection;
18pub mod state;
19pub mod verify;
20
21#[cfg(all(test, feature = "memory"))]
22mod extract_tests;
23#[cfg(all(test, feature = "memory"))]
24mod job_tests;
25#[cfg(all(test, feature = "memory"))]
26mod tests;
27
28use crate::{ErrorDetail, ServerError};
29
30/// Exact advertised pack-size refusal, including cached verification.
31pub(crate) const PACK_CAP_MESSAGE: &str = "pack exceeds indexed max_pack_bytes";
32
33pub(crate) fn check_pack_cap(bytes: u64, cap: u64) -> Result<(), ServerError> {
34 if bytes > cap {
35 return Err(ServerError::invalid_argument(PACK_CAP_MESSAGE));
36 }
37 Ok(())
38}
39
40/// The canonical pending response: one protobuf detail, HTTP 503 and
41/// `Retry-After` rounded up to whole seconds.
42#[must_use]
43pub fn pending(retry_after_ms: u64) -> ServerError {
44 let retry_after_ms = retry_after_ms.max(1_000);
45 let mut value = vec![0x08];
46 let mut n = retry_after_ms;
47 while n >= 0x80 {
48 value.push(u8::try_from(n & 0x7f).unwrap_or(0) | 0x80);
49 n >>= 7;
50 }
51 value.push(u8::try_from(n).unwrap_or(0));
52 ServerError::unavailable("pack verification pending")
53 .with_detail(ErrorDetail {
54 type_name: "mkit.transport.v1.PendingVerification".into(),
55 value: value.into(),
56 })
57 .with_http_status(503)
58 .with_header("Retry-After", retry_after_ms.div_ceil(1_000).to_string())
59}
60
61/// Limits for opt-in indexed mode. A deployment cannot switch an existing
62/// opaque repository to indexed mode: its member packs have no `i` rows.
63/// Native embedders configure it programmatically; Workers enable it through
64/// the Paid Workers launch profile with scheduled verification, permanent retention
65/// and GC disabled. Async inspection and holds remain post-launch features.
66#[derive(Debug, Clone, Copy, PartialEq, Eq)]
67#[non_exhaustive]
68pub struct IndexedConfig {
69 /// Maximum total delta chain depth across in-pack and member bases.
70 pub max_delta_chain_depth: u32,
71 /// Repository membership lag window in milliseconds.
72 pub relay_lag_bound_ms: u64,
73 /// Largest accepted indexed pack.
74 pub max_pack_bytes: u64,
75 /// Whole-pack decoding budget; at least `max_pack_bytes`. Each canonical
76 /// entry and encoded frame also obeys [`geometry`], in either verification mode.
77 pub decode_budget: u64,
78 /// Least Blob size extracted into the global object store, at least 1.
79 /// Never advertised (SPEC-SERVER ยง9.6).
80 pub extract_min_bytes: u64,
81 /// Most content bytes one advance may reassemble into the object store,
82 /// and most member bytes it may resolve to do so. A small manifest over
83 /// many member chunks amplifies into a large reassembly; exceeding this
84 /// is `pack exceeds indexed decode budget`. `None` derives
85 /// `4 * max_pack_bytes` when the pipeline is built; a set value must be
86 /// at least `max_pack_bytes`.
87 pub max_extract_bytes: Option<u64>,
88 /// Where verification runs: inline in the advance, or in checkpointed
89 /// alarm slices (WP-4.8). Programmatic only.
90 pub verification: VerificationMode,
91 /// Most member commits and uncached delta bases one fast-forward check
92 /// may resolve (WP-4.17), from 1 to [`MAX_ANCESTRY_COMMITS_LIMIT`].
93 /// Beyond it the check is unproven and
94 /// the write is denied.
95 pub max_ancestry_commits: u32,
96}
97
98/// Where a ticketed pack is verified.
99#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
100#[non_exhaustive]
101pub enum VerificationMode {
102 /// In the advance, whole-pack and in memory (native).
103 #[default]
104 Inline,
105 /// By kind-7 slices; an advance only checks their result and answers
106 /// `PendingVerification` until they finish (Workers).
107 Scheduled,
108}
109
110impl IndexedConfig {
111 /// Scheduled verification (WP-4.8) for packs up to `max_pack_bytes`, every
112 /// other limit at its default; the decode budget is raised to cover the
113 /// pack cap when that is larger.
114 #[must_use]
115 pub fn scheduled(max_pack_bytes: u64) -> Self {
116 let defaults = Self::default();
117 Self {
118 verification: VerificationMode::Scheduled,
119 max_pack_bytes,
120 decode_budget: defaults.decode_budget.max(max_pack_bytes),
121 max_ancestry_commits: SCHEDULED_MAX_ANCESTRY_COMMITS,
122 ..defaults
123 }
124 }
125
126 /// [`Self::max_extract_bytes`], or `4 * max_pack_bytes` when unset.
127 #[must_use]
128 pub fn effective_max_extract_bytes(&self) -> u64 {
129 self.max_extract_bytes
130 .unwrap_or_else(|| self.max_pack_bytes.saturating_mul(4))
131 }
132}
133
134/// The largest accepted [`IndexedConfig::max_ancestry_commits`].
135pub const MAX_ANCESTRY_COMMITS_LIMIT: u32 = 65_536;
136
137/// [`IndexedConfig::max_ancestry_commits`] under scheduled verification: a
138/// Worker alarm cannot walk more (WP-4.17's Worker cap, R-171).
139pub const SCHEDULED_MAX_ANCESTRY_COMMITS: u32 = 64;
140
141/// The default [`IndexedConfig::max_ancestry_commits`].
142pub const DEFAULT_MAX_ANCESTRY_COMMITS: u32 = 256;
143
144/// The default [`IndexedConfig::extract_min_bytes`]: 64 KiB.
145pub const DEFAULT_EXTRACT_MIN_BYTES: u64 = 64 * 1024;
146
147impl Default for IndexedConfig {
148 fn default() -> Self {
149 Self {
150 max_delta_chain_depth: 50,
151 relay_lag_bound_ms: crate::relay::RELAY_LAG_BOUND_MS,
152 max_pack_bytes: 2 << 30,
153 decode_budget: 2 << 30,
154 extract_min_bytes: DEFAULT_EXTRACT_MIN_BYTES,
155 max_extract_bytes: None,
156 verification: VerificationMode::Inline,
157 max_ancestry_commits: DEFAULT_MAX_ANCESTRY_COMMITS,
158 }
159 }
160}