mini_app_core/
snapshot_upload.rs

1/// S3-compatible snapshot upload for the `data_snapshot` MCP tool.
2///
3/// This module implements **S3 Protocol support, not AWS-specific support**:
4/// the endpoint is injected via environment variables, so AWS S3, Backblaze B2
5/// (S3-Compatible API), Cloudflare R2, and MinIO all work through the same
6/// code path.
7///
8/// # Feature gating
9///
10/// The network `put` path (and the `object_store` dependency) is compiled only
11/// with the `s3-upload` cargo feature. **Configuration resolution is compiled
12/// unconditionally** so that callers can produce a precise
13/// `UPLOAD_NOT_CONFIGURED` error (missing env vs disabled feature) in every
14/// build flavour, and so the pure resolution logic stays unit-testable without
15/// the heavy dependency.
16///
17/// # Configuration (environment only)
18///
19/// Credentials and endpoints are read from `MINI_APP_S3_*` environment
20/// variables (never from tool-call arguments) so that secrets never travel
21/// through the LLM-visible MCP layer. The variables ride the existing
22/// `.mini-app-mcp.env` (dotenvy) loading path.
23///
24/// | env | required | meaning |
25/// |---|---|---|
26/// | `MINI_APP_S3_ENDPOINT` | yes | S3-compatible endpoint URL |
27/// | `MINI_APP_S3_BUCKET` | yes | bucket name |
28/// | `MINI_APP_S3_ACCESS_KEY_ID` | yes | access key id |
29/// | `MINI_APP_S3_SECRET_ACCESS_KEY` | yes | secret access key |
30/// | `MINI_APP_S3_PREFIX` | no | key prefix (default `mini-app-snapshots/`) |
31/// | `MINI_APP_S3_REGION` | no | signing region. Unset = derived from `s3.<region>.<domain>` endpoints (B2 / AWS regional); underivable hosts fall back to `us-east-1` |
32/// | `MINI_APP_S3_VIRTUAL_HOSTED_STYLE` | no | `true` = virtual-hosted addressing; default `false` = path style (MinIO-compatible) |
33/// | `MINI_APP_S3_CHECKSUM` | no | `sha256` = send `x-amz-checksum-sha256` on put; default `none` (some S3-compatible providers reject checksum headers) |
34///
35/// # Failure semantics
36///
37/// - Missing configuration is detected **before** any snapshot is written
38///   (the caller checks first) — [`MiniAppError::UploadNotConfigured`].
39/// - A failed upload of an individual snapshot file is **non-fatal** for the
40///   `data_snapshot` call: the local snapshot and purge results stand, and the
41///   error is reported in the `upload_errors[]` response field.
42/// - Remote retention is intentionally out of scope: old objects are expected
43///   to be expired by bucket lifecycle rules (KNOWN LIMITATION).
44use std::collections::HashMap;
45
46use crate::error::MiniAppError;
47
48/// Env var: S3-compatible endpoint URL (required).
49pub const ENV_ENDPOINT: &str = "MINI_APP_S3_ENDPOINT";
50/// Env var: bucket name (required).
51pub const ENV_BUCKET: &str = "MINI_APP_S3_BUCKET";
52/// Env var: access key id (required).
53pub const ENV_ACCESS_KEY_ID: &str = "MINI_APP_S3_ACCESS_KEY_ID";
54/// Env var: secret access key (required).
55pub const ENV_SECRET_ACCESS_KEY: &str = "MINI_APP_S3_SECRET_ACCESS_KEY";
56/// Env var: key prefix (optional).
57pub const ENV_PREFIX: &str = "MINI_APP_S3_PREFIX";
58/// Env var: signing region (optional).
59pub const ENV_REGION: &str = "MINI_APP_S3_REGION";
60/// Env var: addressing style (optional, `true`/`false`).
61pub const ENV_VIRTUAL_HOSTED_STYLE: &str = "MINI_APP_S3_VIRTUAL_HOSTED_STYLE";
62/// Env var: upload checksum algorithm (optional, `none`/`sha256`).
63pub const ENV_CHECKSUM: &str = "MINI_APP_S3_CHECKSUM";
64
65/// Default key prefix used when `MINI_APP_S3_PREFIX` is not set.
66pub const DEFAULT_PREFIX: &str = "mini-app-snapshots/";
67
68/// Dummy signing region used when `MINI_APP_S3_REGION` is not set **and** no
69/// region can be derived from the endpoint host.
70///
71/// Several S3-compatible providers accept any region string but the SigV4
72/// signer requires one. Endpoints of the form `s3.<region>.<domain>` (B2,
73/// AWS regional) have their region derived automatically — see
74/// [`derive_region_from_endpoint`] — so this fallback only applies to hosts
75/// without an embedded region (MinIO, R2, `s3.amazonaws.com`).
76pub const DEFAULT_REGION: &str = "us-east-1";
77
78/// Derives the signing region from an endpoint of the form
79/// `https://s3.<region>.<domain>...` (Backblaze B2 and AWS regional
80/// endpoints embed the region as the second host label).
81///
82/// Returns `None` when the host does not match that shape — e.g.
83/// `s3.amazonaws.com` (3 labels, no region), MinIO hosts, or R2's
84/// `<account>.r2.cloudflarestorage.com`.
85pub fn derive_region_from_endpoint(endpoint: &str) -> Option<String> {
86    let host = endpoint
87        .trim_start_matches("https://")
88        .trim_start_matches("http://")
89        .split(['/', ':'])
90        .next()?;
91    let labels: Vec<&str> = host.split('.').collect();
92    // `s3.<region>.<domain>.<tld>` needs at least 4 labels so that the second
93    // label is a region and not the registrable domain itself.
94    if labels.len() >= 4 && labels[0] == "s3" && !labels[1].is_empty() {
95        Some(labels[1].to_string())
96    } else {
97        None
98    }
99}
100
101/// Resolved upload configuration for an S3-compatible destination.
102#[derive(Debug, Clone)]
103pub struct S3UploadConfig {
104    /// S3-compatible endpoint URL (e.g. `https://s3.us-west-004.backblazeb2.com`).
105    pub endpoint: String,
106    /// Bucket name.
107    pub bucket: String,
108    /// Access key id.
109    pub access_key_id: String,
110    /// Secret access key.
111    pub secret_access_key: String,
112    /// Key prefix; joined with the snapshot file name by [`S3UploadConfig::key_for`].
113    pub prefix: String,
114    /// Optional signing region ([`DEFAULT_REGION`] is used when absent).
115    pub region: Option<String>,
116    /// Addressing style: `true` = virtual-hosted (`bucket.endpoint/key`),
117    /// `false` = path style (`endpoint/bucket/key`, default — required by
118    /// MinIO and accepted by AWS / B2 / R2).
119    pub virtual_hosted_style: bool,
120    /// When `true`, puts carry an `x-amz-checksum-sha256` header (the only
121    /// algorithm `object_store` supports). Default `false` — some
122    /// S3-compatible providers reject checksum headers with
123    /// `400 InvalidArgument: Unsupported header`.
124    pub checksum_sha256: bool,
125}
126
127impl S3UploadConfig {
128    /// Resolves the configuration from an arbitrary variable map.
129    ///
130    /// Pure function over `vars` — no process-environment access — so unit
131    /// tests can exercise every branch without mutating `std::env` (which is
132    /// unsafe under parallel test execution).
133    ///
134    /// Empty-string values are treated as missing.
135    ///
136    /// # Errors
137    /// - [`MiniAppError::UploadNotConfigured`] listing every missing required
138    ///   variable by name.
139    pub fn from_vars(vars: &HashMap<String, String>) -> Result<Self, MiniAppError> {
140        let get = |key: &str| -> Option<String> {
141            vars.get(key)
142                .map(|v| v.trim().to_string())
143                .filter(|v| !v.is_empty())
144        };
145
146        let mut missing: Vec<&str> = Vec::new();
147        let endpoint = get(ENV_ENDPOINT);
148        if endpoint.is_none() {
149            missing.push(ENV_ENDPOINT);
150        }
151        let bucket = get(ENV_BUCKET);
152        if bucket.is_none() {
153            missing.push(ENV_BUCKET);
154        }
155        let access_key_id = get(ENV_ACCESS_KEY_ID);
156        if access_key_id.is_none() {
157            missing.push(ENV_ACCESS_KEY_ID);
158        }
159        let secret_access_key = get(ENV_SECRET_ACCESS_KEY);
160        if secret_access_key.is_none() {
161            missing.push(ENV_SECRET_ACCESS_KEY);
162        }
163
164        if !missing.is_empty() {
165            return Err(MiniAppError::UploadNotConfigured(format!(
166                "missing env: {}",
167                missing.join(", ")
168            )));
169        }
170
171        let virtual_hosted_style = match get(ENV_VIRTUAL_HOSTED_STYLE).as_deref() {
172            None => false,
173            Some(v) if v.eq_ignore_ascii_case("true") || v == "1" => true,
174            Some(v) if v.eq_ignore_ascii_case("false") || v == "0" => false,
175            Some(other) => {
176                return Err(MiniAppError::UploadNotConfigured(format!(
177                    "{ENV_VIRTUAL_HOSTED_STYLE} must be true/false, got '{other}'"
178                )));
179            }
180        };
181
182        let checksum_sha256 = match get(ENV_CHECKSUM).as_deref() {
183            None => false,
184            Some(v) if v.eq_ignore_ascii_case("none") => false,
185            Some(v) if v.eq_ignore_ascii_case("sha256") => true,
186            Some(other) => {
187                return Err(MiniAppError::UploadNotConfigured(format!(
188                    "{ENV_CHECKSUM} must be none/sha256, got '{other}'"
189                )));
190            }
191        };
192
193        // SAFETY of unwraps: all four options were verified Some above
194        // (missing.is_empty() implies each individual check passed).
195        let endpoint = endpoint.unwrap();
196        // Explicit env wins; otherwise derive from `s3.<region>.<domain>`
197        // endpoints (B2 / AWS regional) so users don't repeat themselves.
198        let region = get(ENV_REGION).or_else(|| derive_region_from_endpoint(&endpoint));
199        Ok(S3UploadConfig {
200            endpoint,
201            bucket: bucket.unwrap(),
202            access_key_id: access_key_id.unwrap(),
203            secret_access_key: secret_access_key.unwrap(),
204            prefix: get(ENV_PREFIX).unwrap_or_else(|| DEFAULT_PREFIX.to_string()),
205            region,
206            virtual_hosted_style,
207            checksum_sha256,
208        })
209    }
210
211    /// Resolves the configuration from the process environment.
212    ///
213    /// # Errors
214    /// - [`MiniAppError::UploadNotConfigured`] when required variables are
215    ///   missing (see [`S3UploadConfig::from_vars`]).
216    pub fn from_env() -> Result<Self, MiniAppError> {
217        let vars: HashMap<String, String> = std::env::vars().collect();
218        Self::from_vars(&vars)
219    }
220
221    /// Builds the object key for a snapshot file name by joining `prefix` and
222    /// `file_name` with exactly one `/` (an empty prefix yields the bare name).
223    pub fn key_for(&self, file_name: &str) -> String {
224        let trimmed = self.prefix.trim_end_matches('/');
225        if trimmed.is_empty() {
226            file_name.to_string()
227        } else {
228            format!("{}/{}", trimmed, file_name)
229        }
230    }
231}
232
233/// Whether this binary was built with the `s3-upload` feature.
234///
235/// Callers use this to distinguish "feature disabled" from "env incomplete"
236/// in `UPLOAD_NOT_CONFIGURED` messages.
237pub const fn upload_feature_enabled() -> bool {
238    cfg!(feature = "s3-upload")
239}
240
241/// Uploads one local snapshot file to `{bucket}/{key}` on the configured
242/// S3-compatible endpoint. Returns the number of bytes uploaded.
243///
244/// The whole file is read into memory before the put — snapshot files are
245/// SQLite databases of modest size (mini-app tables are small by design), so
246/// multipart streaming is intentionally not implemented.
247///
248/// # Errors
249/// - [`MiniAppError::Upload`] on local read failure, client construction
250///   failure, or a rejected/failed put.
251#[cfg(feature = "s3-upload")]
252pub async fn upload_snapshot(
253    config: &S3UploadConfig,
254    local_path: &std::path::Path,
255    key: &str,
256) -> Result<u64, MiniAppError> {
257    use object_store::ObjectStore;
258    use object_store::aws::AmazonS3Builder;
259
260    let bytes = tokio::fs::read(local_path)
261        .await
262        .map_err(|e| MiniAppError::Upload(format!("cannot read snapshot file: {e}")))?;
263    let len = bytes.len() as u64;
264
265    let mut builder = AmazonS3Builder::new()
266        .with_endpoint(&config.endpoint)
267        .with_bucket_name(&config.bucket)
268        .with_access_key_id(&config.access_key_id)
269        .with_secret_access_key(&config.secret_access_key)
270        // SigV4 requires a region even for providers that ignore it; B2 users
271        // must set MINI_APP_S3_REGION to match their endpoint region.
272        .with_region(config.region.as_deref().unwrap_or(DEFAULT_REGION))
273        .with_virtual_hosted_style_request(config.virtual_hosted_style);
274    if config.checksum_sha256 {
275        builder = builder.with_checksum_algorithm(object_store::aws::Checksum::SHA256);
276    }
277    // Allow plain-http endpoints (local MinIO smoke); TLS endpoints unaffected.
278    if config.endpoint.starts_with("http://") {
279        builder = builder.with_allow_http(true);
280    }
281    let store = builder
282        .build()
283        .map_err(|e| MiniAppError::Upload(format!("cannot build s3 client: {e}")))?;
284
285    let object_path = object_store::path::Path::from(key);
286    store
287        .put(&object_path, bytes::Bytes::from(bytes).into())
288        .await
289        .map_err(|e| MiniAppError::Upload(format!("put '{key}' failed: {e}")))?;
290
291    Ok(len)
292}
293
294#[cfg(test)]
295mod tests {
296    use super::*;
297
298    fn full_vars() -> HashMap<String, String> {
299        HashMap::from([
300            (
301                ENV_ENDPOINT.to_string(),
302                "https://s3.example.com".to_string(),
303            ),
304            (ENV_BUCKET.to_string(), "my-bucket".to_string()),
305            (ENV_ACCESS_KEY_ID.to_string(), "AKID".to_string()),
306            (ENV_SECRET_ACCESS_KEY.to_string(), "SECRET".to_string()),
307        ])
308    }
309
310    /// T1: full required vars resolve, optionals fall back to defaults.
311    #[test]
312    fn from_vars_resolves_with_defaults() {
313        let config = S3UploadConfig::from_vars(&full_vars()).expect("must resolve");
314        assert_eq!(config.endpoint, "https://s3.example.com");
315        assert_eq!(config.bucket, "my-bucket");
316        assert_eq!(config.prefix, DEFAULT_PREFIX);
317        assert_eq!(config.region, None);
318        assert!(!config.virtual_hosted_style, "default must be path style");
319        assert!(!config.checksum_sha256, "default must send no checksum");
320    }
321
322    /// T2: checksum flag parses none/sha256 strictly.
323    #[test]
324    fn from_vars_checksum_parse() {
325        for (raw, expected) in [
326            ("none", false),
327            ("NONE", false),
328            ("sha256", true),
329            ("SHA256", true),
330        ] {
331            let mut vars = full_vars();
332            vars.insert(ENV_CHECKSUM.to_string(), raw.to_string());
333            let config = S3UploadConfig::from_vars(&vars).expect("must resolve");
334            assert_eq!(config.checksum_sha256, expected, "raw value '{raw}'");
335        }
336
337        let mut vars = full_vars();
338        vars.insert(ENV_CHECKSUM.to_string(), "crc32".to_string());
339        let err = S3UploadConfig::from_vars(&vars).expect_err("unsupported algo must fail");
340        let MiniAppError::UploadNotConfigured(msg) = &err else {
341            panic!("expected UploadNotConfigured, got {err:?}");
342        };
343        assert!(
344            msg.contains(ENV_CHECKSUM),
345            "message must name the offending var: {msg}"
346        );
347    }
348
349    /// T2: addressing-style flag parses true/false variants strictly.
350    #[test]
351    fn from_vars_virtual_hosted_style_parse() {
352        for (raw, expected) in [
353            ("true", true),
354            ("TRUE", true),
355            ("1", true),
356            ("false", false),
357            ("0", false),
358        ] {
359            let mut vars = full_vars();
360            vars.insert(ENV_VIRTUAL_HOSTED_STYLE.to_string(), raw.to_string());
361            let config = S3UploadConfig::from_vars(&vars).expect("must resolve");
362            assert_eq!(config.virtual_hosted_style, expected, "raw value '{raw}'");
363        }
364
365        let mut vars = full_vars();
366        vars.insert(ENV_VIRTUAL_HOSTED_STYLE.to_string(), "maybe".to_string());
367        let err = S3UploadConfig::from_vars(&vars).expect_err("junk value must fail");
368        let MiniAppError::UploadNotConfigured(msg) = &err else {
369            panic!("expected UploadNotConfigured, got {err:?}");
370        };
371        assert!(
372            msg.contains(ENV_VIRTUAL_HOSTED_STYLE),
373            "message must name the offending var: {msg}"
374        );
375    }
376
377    /// T1: optional vars are picked up when present.
378    #[test]
379    fn from_vars_resolves_optionals() {
380        let mut vars = full_vars();
381        vars.insert(ENV_PREFIX.to_string(), "backups/mini".to_string());
382        vars.insert(ENV_REGION.to_string(), "us-west-004".to_string());
383        let config = S3UploadConfig::from_vars(&vars).expect("must resolve");
384        assert_eq!(config.prefix, "backups/mini");
385        assert_eq!(config.region.as_deref(), Some("us-west-004"));
386    }
387
388    /// T3: empty map lists every missing required var by name.
389    #[test]
390    fn from_vars_empty_reports_all_missing() {
391        let err = S3UploadConfig::from_vars(&HashMap::new()).expect_err("must fail");
392        let MiniAppError::UploadNotConfigured(msg) = &err else {
393            panic!("expected UploadNotConfigured, got {err:?}");
394        };
395        for var in [
396            ENV_ENDPOINT,
397            ENV_BUCKET,
398            ENV_ACCESS_KEY_ID,
399            ENV_SECRET_ACCESS_KEY,
400        ] {
401            assert!(msg.contains(var), "message must name '{var}': {msg}");
402        }
403        assert_eq!(err.code(), crate::error::codes::UPLOAD_NOT_CONFIGURED);
404    }
405
406    /// T2: empty-string values count as missing (e.g. `MINI_APP_S3_BUCKET=`).
407    #[test]
408    fn from_vars_empty_string_counts_as_missing() {
409        let mut vars = full_vars();
410        vars.insert(ENV_BUCKET.to_string(), "  ".to_string());
411        let err = S3UploadConfig::from_vars(&vars).expect_err("must fail");
412        let MiniAppError::UploadNotConfigured(msg) = &err else {
413            panic!("expected UploadNotConfigured, got {err:?}");
414        };
415        assert!(msg.contains(ENV_BUCKET), "message must name bucket: {msg}");
416        assert!(
417            !msg.contains(ENV_ENDPOINT),
418            "endpoint was provided and must not be listed: {msg}"
419        );
420    }
421
422    /// T1: region is derived from `s3.<region>.<domain>` endpoints when the
423    /// env var is absent (B2 users must not need to repeat the region).
424    #[test]
425    fn from_vars_region_derived_from_endpoint() {
426        let mut vars = full_vars();
427        vars.insert(
428            ENV_ENDPOINT.to_string(),
429            "https://s3.us-east-005.backblazeb2.com".to_string(),
430        );
431        let config = S3UploadConfig::from_vars(&vars).expect("must resolve");
432        assert_eq!(config.region.as_deref(), Some("us-east-005"));
433
434        // Explicit env always wins over derivation.
435        vars.insert(ENV_REGION.to_string(), "eu-central-003".to_string());
436        let config = S3UploadConfig::from_vars(&vars).expect("must resolve");
437        assert_eq!(config.region.as_deref(), Some("eu-central-003"));
438    }
439
440    /// T2: derivation shapes — regional hosts yield a region, others None.
441    #[test]
442    fn derive_region_shapes() {
443        for (endpoint, expected) in [
444            (
445                "https://s3.us-east-005.backblazeb2.com",
446                Some("us-east-005"),
447            ),
448            ("https://s3.us-west-2.amazonaws.com", Some("us-west-2")),
449            (
450                "https://s3.us-west-2.amazonaws.com/extra/path",
451                Some("us-west-2"),
452            ),
453            ("https://s3.amazonaws.com", None),
454            ("http://localhost:9000", None),
455            ("https://account.r2.cloudflarestorage.com", None),
456        ] {
457            assert_eq!(
458                derive_region_from_endpoint(endpoint).as_deref(),
459                expected,
460                "endpoint '{endpoint}'"
461            );
462        }
463    }
464
465    /// T2: key_for joins with exactly one slash regardless of prefix shape.
466    #[test]
467    fn key_for_prefix_join() {
468        let mut config = S3UploadConfig::from_vars(&full_vars()).expect("must resolve");
469
470        config.prefix = "snaps/".to_string();
471        assert_eq!(config.key_for("issue.100.db"), "snaps/issue.100.db");
472
473        config.prefix = "snaps".to_string();
474        assert_eq!(config.key_for("issue.100.db"), "snaps/issue.100.db");
475
476        config.prefix = String::new();
477        assert_eq!(config.key_for("issue.100.db"), "issue.100.db");
478    }
479}
mini_app_core/snapshot_upload.rs

mini_app_core/
snapshot_upload.rs
