Skip to main content

rivet/config/
mod.rs

1pub mod cursor;
2mod destination;
3mod export;
4mod format;
5mod lints;
6mod notifications;
7pub mod resolve;
8pub mod schema;
9mod source;
10
11pub use cursor::IncrementalCursorMode;
12pub use destination::*;
13pub use export::*;
14pub use format::*;
15pub use notifications::*;
16#[allow(unused_imports)]
17pub(crate) use resolve::resolve_env_vars;
18pub use resolve::{parse_file_size, resolve_vars};
19pub use schema::generate_config_schema_pretty;
20pub use source::*;
21
22use schemars::JsonSchema;
23use serde::Deserialize;
24
25/// Top-level Rivet configuration root.
26///
27/// Operators write this struct as YAML (typically `rivet.yaml`).  The
28/// `JsonSchema` derive is the source of truth for the `schemas/rivet.schema.json`
29/// artifact and the `rivet schema config` command's output (v0.7.3 P0).
30#[derive(Debug, Deserialize, JsonSchema, Clone)]
31#[serde(deny_unknown_fields)]
32pub struct Config {
33    pub source: SourceConfig,
34    pub exports: Vec<ExportConfig>,
35    #[serde(default)]
36    pub notifications: Option<NotificationsConfig>,
37    #[serde(default)]
38    pub parallel_exports: bool,
39    #[serde(default)]
40    pub parallel_export_processes: bool,
41    /// The warehouse **load** target — consumed by `rivet load` (and `rivet load
42    /// --cdc`), so ONE config drives both the export and the downstream load. The
43    /// extraction commands (`rivet check` / `run` / `apply`) accept and ignore it:
44    /// it shapes the load, not the extract.
45    // Kept as a raw `serde_json::Value` so the extraction path neither depends on
46    // nor validates the load schema — the load module (`crate::load::plan`) parses
47    // it into a typed `LoadSection`.
48    #[serde(default)]
49    pub load: Option<serde_json::Value>,
50}
51
52impl Config {
53    pub fn load(path: &str) -> crate::error::Result<Self> {
54        Self::load_with_params(path, None)
55    }
56
57    pub fn load_with_params(
58        path: &str,
59        params: Option<&std::collections::HashMap<String, String>>,
60    ) -> crate::error::Result<Self> {
61        // F11 (0.7.5 audit): raw `std::io::Error` lost the path on
62        // not-found.  Wrap with the file path + a hint so the operator
63        // can see *which* config the tool could not open.
64        let contents = std::fs::read_to_string(path).map_err(|e| {
65            if e.kind() == std::io::ErrorKind::NotFound {
66                anyhow::anyhow!(
67                    "config file '{}' not found.\n  Hint: check the path, or run `rivet init` to generate one.",
68                    path
69                )
70            } else {
71                anyhow::anyhow!("cannot read config file '{}': {}", path, e)
72            }
73        })?;
74        // Warn about typo'd `--param` keys once per CLI invocation, using the
75        // un-resolved YAML as the haystack so the placeholders are still there.
76        // We pass the raw `contents` (not `resolved`) on purpose: after
77        // resolution the placeholders are gone, and every key would look unused.
78        resolve::warn_unused_params(&contents, params);
79        let resolved = resolve_vars(&contents, params)?;
80        // F12 (0.7.5 audit): YAML parse errors did not name the config
81        // file.  When loading from disk we know the path — thread it
82        // into the parse error.
83        // `.context` (not `anyhow!("…{:#}", e)`, which stringifies into a fresh
84        // error) so a typed `CodedError` raised by validation survives the chain —
85        // `error::error_code` downcasts it for the `[CODE]` prefix. `{e:#}` in
86        // `main` still renders the full "config file '…': <message>" chain.
87        Self::from_yaml(&resolved).map_err(|e| e.context(format!("config file '{}'", path)))
88    }
89
90    pub fn from_yaml(yaml: &str) -> crate::error::Result<Self> {
91        // Intercept the unquoted-`{partition}` footgun before the raw-YAML
92        // pre-scans below (they `from_str::<Value>` too and would re-emit the
93        // same cryptic libyaml message). A document that does not even parse as
94        // a `Value` is malformed; if the failure looks like a flow-mapping
95        // scanner error *and* the source carries an unquoted brace value, point
96        // straight at the quoting fix. On a valid config this `Value` parse
97        // succeeds and the block is skipped, so the success path is unchanged.
98        if let Err(e) = serde_yaml_ng::from_str::<serde_yaml_ng::Value>(yaml) {
99            let m = e.to_string();
100            if let Some(hint) = Self::unquoted_template_brace_hint(yaml, &m) {
101                return Err(anyhow::anyhow!("{e}\n  {hint}"));
102            }
103            // A TAB in indentation is the most common beginner YAML mistake; it
104            // trips libyaml before serde ever sees a field, so it must be caught
105            // here in the raw scan, not in `enhance_parse_error`.
106            if let Some(hint) = Self::tab_indent_hint(yaml, &m) {
107                return Err(anyhow::anyhow!("{e}\n  {hint}"));
108            }
109        }
110        Self::check_misplaced_tuning_fields(yaml)?;
111        Self::check_csv_compression(yaml)?;
112        Self::check_tls_mode_downgrade(yaml)?;
113        let config: Config = serde_yaml_ng::from_str(yaml).map_err(|e| {
114            // A well-formed flow map (`prefix: {partition}`) parses as a YAML
115            // value but serde then rejects it with `invalid type: map, expected
116            // a string`. That is the same unquoted-brace footgun, surfacing one
117            // layer later than the scanner errors caught above — so try the same
118            // hint here before falling back to the generic field-typo enhancer.
119            if let Some(hint) = Self::unquoted_template_brace_hint(yaml, &e.to_string()) {
120                anyhow::anyhow!("{e}\n  {hint}")
121            } else {
122                lints::enhance_parse_error(e)
123            }
124        })?;
125        config.validate()?;
126        Ok(config)
127    }
128
129    /// Detect the unquoted-`{partition}` (or `{date}`, …) template footgun and
130    /// return an actionable quoting hint, or `None` when the error is unrelated.
131    ///
132    /// A YAML value that *starts* a flow mapping — `prefix: {partition}` — is
133    /// the common copy-paste mistake: `{partition}` is the required token for
134    /// `partition_by`, but unquoted it parses as a YAML map (or, with trailing
135    /// text, trips the libyaml scanner). serde then emits a cryptic
136    /// `did not find expected ',' or '}'` / `while parsing a flow mapping` /
137    /// `invalid type: map, expected a string` with no hint that a pair of
138    /// quotes is the fix.
139    ///
140    /// Two guards keep this from firing on unrelated parse errors: the error
141    /// message must carry one of the flow-mapping symptoms, AND the raw source
142    /// must actually contain an unquoted `{…}` value. Both must hold, so a
143    /// valid config (every brace value quoted) never sees the hint, and a
144    /// genuine map-typed field error elsewhere is left alone.
145    /// A YAML document indented with a TAB trips libyaml with `found character
146    /// that cannot start any token`. Point straight at the fix (spaces, not
147    /// tabs) — but only when the cited line really begins with a tab, so an
148    /// unrelated scanner error is left with its original message.
149    fn tab_indent_hint(yaml: &str, err_msg: &str) -> Option<String> {
150        if !err_msg.contains("tab character") {
151            return None;
152        }
153        // serde_yaml_ng reports `found a tab character … at line N column C …`;
154        // the FIRST `line N` is where the offending tab is.
155        let line_no: usize = err_msg
156            .split_once("line ")
157            .and_then(|(_, rest)| rest.split([' ', ',']).next())
158            .and_then(|n| n.parse().ok())?;
159        let line = yaml.lines().nth(line_no.checked_sub(1)?)?;
160        let leading = &line[..line.len() - line.trim_start().len()];
161        leading.contains('\t').then(|| {
162            format!(
163                "line {line_no} is indented with a TAB — YAML requires spaces. Replace the tab(s) with spaces."
164            )
165        })
166    }
167
168    fn unquoted_template_brace_hint(yaml: &str, err_msg: &str) -> Option<String> {
169        const FLOW_SYMPTOMS: &[&str] = &[
170            "did not find expected ',' or '}'",
171            "while parsing a flow mapping",
172            // A bare `key: {token}` parses as a map, then serde rejects the
173            // map where it wanted a scalar — same root cause, later layer.
174            "invalid type: map, expected a string",
175            // `key: {token}/more` runs the flow map into block context.
176            "did not find expected key",
177        ];
178        if !FLOW_SYMPTOMS.iter().any(|s| err_msg.contains(s)) {
179            return None;
180        }
181        if !yaml.lines().any(line_has_unquoted_brace_value) {
182            return None;
183        }
184        Some(
185            "a YAML value containing { } (such as {partition} or {date}) must be quoted, \
186             e.g. prefix: \"exports/{partition}/\""
187                .to_string(),
188        )
189    }
190
191    /// Reject `format: csv` paired with an explicitly-requested compression
192    /// codec (Finding #10). The CSV writer has no compression encoder, so the
193    /// codec is silently dropped on write while the run manifest still records
194    /// it — a degraded, dishonest no-op. We reject loudly at config-validate
195    /// time so `rivet check` / `rivet doctor` catch it before any run.
196    ///
197    /// This is a raw-YAML scan (like [`Self::check_misplaced_tuning_fields`])
198    /// rather than a `validate_export` check on purpose: `ExportConfig.
199    /// compression` is `#[serde(default)]` and `CompressionType::default()` is
200    /// `Zstd`, so a parsed export cannot distinguish "user asked for zstd" from
201    /// "user omitted the field". Only a user who *wrote* `compression:`/
202    /// `compression_profile:` is asking for something the CSV writer cannot
203    /// honour; the bare-`format: csv` default writes uncompressed and is fine.
204    fn check_csv_compression(yaml: &str) -> crate::error::Result<()> {
205        let root: serde_yaml_ng::Value = serde_yaml_ng::from_str(yaml)?;
206        let Some(exports) = root.get("exports").and_then(|e| e.as_sequence()) else {
207            return Ok(());
208        };
209        for export in exports {
210            if export.get("format").and_then(|f| f.as_str()) != Some("csv") {
211                continue;
212            }
213            let name = export
214                .get("name")
215                .and_then(|n| n.as_str())
216                .unwrap_or("<unnamed>");
217
218            // Explicit `compression:` codec that the CSV writer cannot apply.
219            // An unrecognised label is left for serde to reject during the real
220            // parse; we only act on a codec we understand and that CSV cannot
221            // honour (everything but `none`).
222            if let Some(codec) = export.get("compression").and_then(|c| c.as_str())
223                && let Some(ct) = CompressionType::from_label(codec)
224                && !format::compression_supported(FormatType::Csv, ct)
225            {
226                anyhow::bail!(
227                    "export '{}': CSV output does not support compression: {}. \
228                     CSV has no compression encoder, so the codec would be silently dropped \
229                     while the manifest records it.\n  \
230                     Hint: use `format: parquet` for compression, or set `compression: none`.",
231                    name,
232                    codec,
233                );
234            }
235
236            // A `compression_profile:` other than `none` resolves to a real
237            // codec too (fast→snappy, balanced/compact→zstd) — same no-op.
238            if let Some(profile) = export.get("compression_profile").and_then(|c| c.as_str())
239                && profile != CompressionProfile::None.label()
240            {
241                anyhow::bail!(
242                    "export '{}': CSV output does not support compression_profile: {} \
243                     (it resolves to a compression codec the CSV writer cannot apply).\n  \
244                     Hint: use `format: parquet` for compression, or set `compression_profile: none`.",
245                    name,
246                    profile,
247                );
248            }
249        }
250        Ok(())
251    }
252
253    /// V13: reject a `source.tls` block that pairs an *explicitly chosen*
254    /// enforced `mode:` with a verification-disabling danger knob
255    /// (`accept_invalid_certs` / `accept_invalid_hostnames`). `mode: verify-full`
256    /// promises chain + hostname verification, but the knob silently downgrades
257    /// it to "trust anything" — a MITM exposure that contradicts the stated
258    /// intent (see `src/source/tls.rs::build_native_tls`, whose comment claims
259    /// this is warned about at config-time but is not).
260    ///
261    /// Like [`Self::check_csv_compression`], this is a raw-YAML scan rather than
262    /// a `validate` check on purpose: `TlsMode` is `#[serde(default)]` and the
263    /// default is `VerifyFull`, so a parsed config cannot distinguish "user
264    /// wrote `mode: verify-full`" (a contradiction to flag) from "user omitted
265    /// `mode:`" (the common dev-container case `tls: { accept_invalid_certs:
266    /// true }` against a loopback self-signed cert — which must keep working).
267    /// Only an *explicit* enforced `mode:` next to a danger knob is the footgun.
268    fn check_tls_mode_downgrade(yaml: &str) -> crate::error::Result<()> {
269        let root: serde_yaml_ng::Value = serde_yaml_ng::from_str(yaml)?;
270        let Some(tls) = root.get("source").and_then(|s| s.get("tls")) else {
271            return Ok(());
272        };
273
274        // Only an explicitly written `mode:` is a deliberate, contradicted
275        // choice; an omitted mode is the dev-container default path.
276        let Some(mode) = tls.get("mode").and_then(|m| m.as_str()) else {
277            return Ok(());
278        };
279        // `disable` carries no verification promise to contradict; the danger
280        // knobs are a no-op there. Flag only the enforced modes.
281        if mode == "disable" {
282            return Ok(());
283        }
284
285        let knob = if tls
286            .get("accept_invalid_certs")
287            .and_then(|v| v.as_bool())
288            .unwrap_or(false)
289        {
290            Some("accept_invalid_certs")
291        } else if tls
292            .get("accept_invalid_hostnames")
293            .and_then(|v| v.as_bool())
294            .unwrap_or(false)
295        {
296            Some("accept_invalid_hostnames")
297        } else {
298            None
299        };
300
301        if let Some(knob) = knob {
302            anyhow::bail!(
303                "source.tls: {} disables certificate verification, silently downgrading the \
304                 chosen `mode: {}` to trust-anything (MITM exposure — credentials and rows \
305                 readable/forgeable on the wire).\n  \
306                 Hint: drop the danger knob and trust a private CA with `tls.ca_file: <pem>`; \
307                 only use a danger knob for a loopback self-signed dev container, and then omit \
308                 the explicit `mode:` so the contradiction is gone.",
309                knob,
310                mode,
311            );
312        }
313        Ok(())
314    }
315
316    /// Detect tuning-related fields placed directly under `source:` or an
317    /// `exports[]` entry instead of inside the `tuning:` sub-key. Without this
318    /// check serde silently ignores unknown keys and the user gets unexpected
319    /// defaults (e.g. batch_size=10 000 instead of the intended 1 000).
320    fn check_misplaced_tuning_fields(yaml: &str) -> crate::error::Result<()> {
321        const TUNING_FIELDS: &[&str] = &[
322            "batch_size",
323            "batch_size_memory_mb",
324            "throttle_ms",
325            "statement_timeout_s",
326            "max_retries",
327            "retry_backoff_ms",
328            "lock_timeout_s",
329            "memory_threshold_mb",
330            "profile",
331        ];
332
333        let root: serde_yaml_ng::Value = serde_yaml_ng::from_str(yaml)?;
334
335        if let Some(source) = root.get("source") {
336            let misplaced: Vec<&str> = TUNING_FIELDS
337                .iter()
338                .copied()
339                .filter(|&f| source.get(f).is_some())
340                .collect();
341            if !misplaced.is_empty() {
342                anyhow::bail!(
343                    "source: field(s) [{}] belong under 'source.tuning:', not directly under 'source:'. \
344                     Example:\n  source:\n    tuning:\n      {}: <value>",
345                    misplaced.join(", "),
346                    misplaced[0],
347                );
348            }
349        }
350
351        if let Some(exports) = root.get("exports").and_then(|e| e.as_sequence()) {
352            for (i, export) in exports.iter().enumerate() {
353                let name = export
354                    .get("name")
355                    .and_then(|n| n.as_str())
356                    .unwrap_or("<unnamed>");
357                let misplaced: Vec<&str> = TUNING_FIELDS
358                    .iter()
359                    .copied()
360                    .filter(|&f| export.get(f).is_some())
361                    .collect();
362                if !misplaced.is_empty() {
363                    anyhow::bail!(
364                        "export '{}' (index {}): field(s) [{}] belong under 'exports[].tuning:', \
365                         not directly in the export. Example:\n  exports:\n    - name: {}\n      tuning:\n        {}: <value>",
366                        name,
367                        i,
368                        misplaced.join(", "),
369                        name,
370                        misplaced[0],
371                    );
372                }
373            }
374        }
375
376        Ok(())
377    }
378
379    /// Reject a config before any plan/connect step. The body is split into
380    /// three cohesive validators so each can be read — and unit-tested — on its
381    /// own: the export-list shape, the source connection block, and the
382    /// per-export rules. The end-to-end surface (`Config::from_yaml`) is
383    /// covered by `config/tests/{validation,secops}.rs`; the split additionally
384    /// lets a rule be exercised directly via `validate_export`.
385    fn validate(&self) -> crate::error::Result<()> {
386        self.validate_exports_list()?;
387        self.validate_source_connection()?;
388        for export in &self.exports {
389            self.validate_export(export)?;
390        }
391        self.validate_cdc_resource_conflicts()?;
392        self.validate_non_sql_source_modes()?;
393        Ok(())
394    }
395
396    /// Non-SQL sources (MongoDB) support only `mode: full` today. Chunked /
397    /// incremental / keyset / time-window all build SQL predicates over an
398    /// introspectable columnar schema, and CDC (change streams) is a separate,
399    /// not-yet-implemented seam. Rejecting the other modes here is precisely
400    /// what makes the SQL-only builders' `unreachable!` arms
401    /// (`sql::quote_ident`, `query::cursor_rhs`, …) provably unreachable for a
402    /// document-store source.
403    fn validate_non_sql_source_modes(&self) -> crate::error::Result<()> {
404        if self.source.source_type.is_sql() {
405            return Ok(());
406        }
407        for e in &self.exports {
408            if !matches!(e.mode, ExportMode::Full | ExportMode::Cdc) {
409                crate::config_bail!(
410                    crate::error::codes::CONFIG_SOURCE_MODE_UNSUPPORTED,
411                    "export '{}': source type '{:?}' supports `mode: full` (batch) and \
412                     `mode: cdc` (change streams) (got `mode: {:?}`). MongoDB has no SQL, so \
413                     chunked / incremental / keyset / time-window are not available; every \
414                     document exports as `_id` + a `document` JSON column.",
415                    e.name,
416                    self.source.source_type,
417                    e.mode
418                );
419            }
420            // An impossible combination must be a config error, not a silent
421            // behavior downgrade: the parallel `_id`-range path keeps NO keyset
422            // checkpoint, so `resume: true` was silently ignored — the whole
423            // collection re-read every run with no warning (bug-hunt find).
424            if e.parallel > 1 && self.source.mongo.as_ref().is_some_and(|m| m.resume) {
425                crate::config_bail!(
426                    crate::error::codes::CONFIG_SOURCE_MODE_UNSUPPORTED,
427                    "export '{}': `source.mongo.resume: true` cannot be combined with \
428                     `parallel: {}` — the parallel `_id`-range fan-out keeps no keyset \
429                     checkpoint, so resume would be silently ignored. Drop `parallel` \
430                     to keep cross-run resume, or drop `resume` to keep the fan-out.",
431                    e.name,
432                    e.parallel
433                );
434            }
435        }
436        Ok(())
437    }
438
439    /// CDC stream resources are per-export and their **defaults collide**: two
440    /// PostgreSQL cdc exports without an explicit `slot:` both resolve to
441    /// `rivet_slot` — each export's ack advances `confirmed_flush_lsn` past
442    /// changes the *other* never read (mutual, silent data loss). Two MySQL
443    /// exports both default to `server_id: 4271` — the server kills the older
444    /// replica connection. A shared `checkpoint:` path makes exports overwrite
445    /// each other's resume position on any engine. All three are config bugs a
446    /// naive multi-table CDC config hits by default, so reject them at load, on
447    /// the RESOLVED values (defaults included). SQL Server `capture_instance`
448    /// sharing is deliberately allowed: the change-table poll is read-only and
449    /// resume state lives in the per-export checkpoint.
450    fn validate_cdc_resource_conflicts(&self) -> crate::error::Result<()> {
451        use std::collections::HashMap;
452
453        let mut slots: HashMap<String, &str> = HashMap::new();
454        let mut server_ids: HashMap<u32, &str> = HashMap::new();
455        let mut checkpoints: HashMap<&str, &str> = HashMap::new();
456
457        for e in self.exports.iter().filter(|e| e.mode == ExportMode::Cdc) {
458            let cdc = e.cdc.as_ref();
459            match self.source.source_type {
460                SourceType::Postgres => {
461                    let slot = cdc
462                        .and_then(|c| c.slot.clone())
463                        .unwrap_or_else(|| DEFAULT_PG_SLOT.to_string());
464                    if let Some(prev) = slots.insert(slot.clone(), &e.name) {
465                        crate::config_bail!(
466                            crate::error::codes::CONFIG_CDC_RESOURCE_CONFLICT,
467                            "exports '{prev}' and '{}': same PostgreSQL slot '{slot}' — a slot \
468                             has ONE consumer; each export's ack would advance it past changes \
469                             the other never read (silent data loss). Set a distinct `cdc.slot:` \
470                             per export (the default is '{DEFAULT_PG_SLOT}').",
471                            e.name
472                        );
473                    }
474                }
475                SourceType::Mysql => {
476                    let sid = cdc
477                        .and_then(|c| c.server_id)
478                        .unwrap_or(DEFAULT_MYSQL_SERVER_ID);
479                    if let Some(prev) = server_ids.insert(sid, &e.name) {
480                        crate::config_bail!(
481                            crate::error::codes::CONFIG_CDC_RESOURCE_CONFLICT,
482                            "exports '{prev}' and '{}': same MySQL server_id {sid} — the server \
483                             kills the older replica connection when a new one registers with \
484                             the same id. Set a distinct `cdc.server_id:` per export (the \
485                             default is {DEFAULT_MYSQL_SERVER_ID}).",
486                            e.name
487                        );
488                    }
489                }
490                SourceType::Mssql => {}
491                // MongoDB change streams watch the whole database — no per-export
492                // slot or server_id to collide. Two Mongo CDC exports sharing a
493                // `checkpoint:` path IS still a conflict, caught by the shared
494                // checkpoint check below.
495                SourceType::Mongo => {}
496            }
497            if let Some(ckpt) = cdc.and_then(|c| c.checkpoint.as_deref())
498                && let Some(prev) = checkpoints.insert(ckpt, &e.name)
499            {
500                crate::config_bail!(
501                    crate::error::codes::CONFIG_CDC_RESOURCE_CONFLICT,
502                    "exports '{prev}' and '{}': same checkpoint path '{ckpt}' — each export \
503                     must own its resume position or they overwrite each other's. Set a \
504                     distinct `cdc.checkpoint:` per export.",
505                    e.name
506                );
507            }
508        }
509        Ok(())
510    }
511
512    /// Whole-config shape: at least one export, names unique.
513    fn validate_exports_list(&self) -> crate::error::Result<()> {
514        // An empty `exports:` list is almost always a typo (wrong config file,
515        // dropped anchor, merged doc with the anchor section missing). Running
516        // with zero exports is a silent no-op that looks like success in CI;
517        // reject fast instead. See QA backlog Task 5.1.
518        if self.exports.is_empty() {
519            crate::config_bail!(
520                crate::error::codes::CONFIG_NO_EXPORTS,
521                "exports: at least one export must be defined (got empty list)"
522            );
523        }
524
525        // Duplicate export names break state tracking: `export_state`,
526        // `file_log`, and `chunk_run` are all keyed by `export_name`, so
527        // two configs with the same name silently share cursor/file-log rows.
528        // QA backlog Task 5.1.
529        let mut seen: std::collections::HashSet<&str> =
530            std::collections::HashSet::with_capacity(self.exports.len());
531        for e in &self.exports {
532            if !seen.insert(e.name.as_str()) {
533                crate::config_bail!(
534                    crate::error::codes::CONFIG_DUPLICATE_EXPORT,
535                    "exports: duplicate export name '{}' (each export must have a unique name; state is keyed by name)",
536                    e.name
537                );
538            }
539        }
540        Ok(())
541    }
542
543    /// Source connection block: exactly one connection method, well-formed,
544    /// and the source-level tuning that is shared by every export.
545    fn validate_source_connection(&self) -> crate::error::Result<()> {
546        if let Some(t) = &self.source.tuning
547            && t.batch_size.is_some()
548            && t.batch_size_memory_mb.is_some()
549        {
550            anyhow::bail!(
551                "tuning: batch_size and batch_size_memory_mb are mutually exclusive. \
552                 Prefer batch_size_memory_mb (rivet sizes the batch to a memory budget, \
553                 adapting to row width); set batch_size only to pin an exact row count."
554            );
555        }
556
557        if !self.source.has_url_fields() && !self.source.has_structured_fields() {
558            // First-run footgun: a config that forgot the source block
559            // entirely.  Show the recommended path (`url_env`) up-front;
560            // operators who actually want structured fields know to look
561            // for them.
562            anyhow::bail!(
563                "source: no connection method configured. Add one of:\n  url_env: DATABASE_URL                          (URL from env var — recommended)\n  url: 'postgresql://user:pass@host:5432/db'      (inline — not recommended for committed configs)\n  url_file: /etc/rivet/source.url                 (URL from file — rotation-friendly)\n  host/user/database/...                          (structured fields under `source:`)"
564            );
565        }
566
567        if self.source.has_url_fields() {
568            let url_count = [
569                &self.source.url,
570                &self.source.url_env,
571                &self.source.url_file,
572            ]
573            .iter()
574            .filter(|u| u.is_some())
575            .count();
576            if url_count > 1 {
577                anyhow::bail!(
578                    "source: specify exactly one of 'url', 'url_env', or 'url_file' (got {} set).\n  Hint: pick one — `url_env` is recommended so credentials never enter the YAML.",
579                    url_count
580                );
581            }
582        }
583
584        if self.source.has_url_fields() && self.source.has_structured_fields() {
585            anyhow::bail!(
586                "source: pick either URL-based config (url/url_env/url_file) OR structured fields (host/user/database/port/password_env), not both.\n  Hint: remove whichever block you don't want; mixing the two is ambiguous."
587            );
588        }
589
590        if self.source.has_structured_fields() {
591            if self.source.host.is_none() {
592                anyhow::bail!(
593                    "source: structured config is missing 'host'.\n  Hint: add `host: localhost` (or your DB host) under `source:` in rivet.yaml.\n  Or switch to URL-based config: `url_env: DATABASE_URL`."
594                );
595            }
596            if self.source.user.is_none() {
597                anyhow::bail!(
598                    "source: structured config is missing 'user'.\n  Hint: add `user: <username>` under `source:` in rivet.yaml."
599                );
600            }
601            if self.source.database.is_none() {
602                anyhow::bail!(
603                    "source: structured config is missing 'database'.\n  Hint: add `database: <dbname>` under `source:` in rivet.yaml."
604                );
605            }
606            if self.source.password.is_some() && self.source.password_env.is_some() {
607                anyhow::bail!(
608                    "source: specify 'password' OR 'password_env', not both.\n  Hint: prefer `password_env: DB_PASSWORD` so credentials never enter the YAML."
609                );
610            }
611        }
612        Ok(())
613    }
614
615    /// Per-export rules: effective tuning, query source, `query_file` SecOps,
616    /// destination auth, compression, and the mode/chunk matrix. Takes `&self`
617    /// because effective tuning merges the source-level block.
618    fn validate_export(&self, export: &ExportConfig) -> crate::error::Result<()> {
619        // V5: `name` is keyed into output paths, file logs, and on-disk state,
620        // yet is otherwise free-form. A traversal (`../../etc/x`), absolute or
621        // slash-bearing (`/abs/x`, `sub/dir`), leading-dot, or NUL-bearing name
622        // escapes the intended output tree and corrupts name-keyed state.
623        // Mirror the `query_file` `..`/absolute guard: reject at config-load,
624        // accepting only a filename-safe charset.
625        if !is_filename_safe_name(&export.name) {
626            anyhow::bail!(
627                "export name '{}' is not filename-safe: it must not be absolute, contain \
628                 '/', '\\', '..', a NUL, or start with '.' (the name is used in output paths \
629                 and state keys). Use a plain identifier like `orders` or `daily_events`.",
630                export.name.escape_default(),
631            );
632        }
633
634        let merged =
635            crate::tuning::merge_tuning_config(self.source.tuning.as_ref(), export.tuning.as_ref());
636        if let Some(t) = merged
637            && t.batch_size.is_some()
638            && t.batch_size_memory_mb.is_some()
639        {
640            anyhow::bail!(
641                "export '{}': effective tuning has both batch_size and batch_size_memory_mb (mutually exclusive)",
642                export.name
643            );
644        }
645        if let Some(et) = &export.tuning
646            && et.batch_size.is_some()
647            && et.batch_size_memory_mb.is_some()
648        {
649            anyhow::bail!(
650                "export '{}': tuning.batch_size and tuning.batch_size_memory_mb are mutually exclusive",
651                export.name
652            );
653        }
654
655        // Before the generic exactly-one counting: the `table:`/`tables:` pair
656        // gets its specific message (the generic one doesn't mention `tables`).
657        if export.table.is_some() && export.tables.is_some() {
658            anyhow::bail!(
659                "export '{}': `table:` and `tables:` are mutually exclusive — use \
660                 `tables: [a, b]` for a multi-table stream",
661                export.name
662            );
663        }
664
665        let set_count = [
666            export.query.is_some(),
667            export.query_file.is_some(),
668            export.table.is_some(),
669            // A multi-table CDC stream (`tables:`) is that export's source
670            // specification — the CDC arm below owns its shape rules.
671            export.tables.is_some(),
672        ]
673        .iter()
674        .filter(|b| **b)
675        .count();
676        if set_count == 0 {
677            anyhow::bail!(
678                "export '{}': specify exactly one of 'query', 'query_file', 'table', or 'tables'. \
679                 Use table: <name> for a whole table (enables PK auto-chunking); \
680                 query: \"SELECT …\" for an inline one-liner; \
681                 query_file: <path> for SQL you keep in version control.",
682                export.name
683            );
684        }
685        if set_count > 1 {
686            anyhow::bail!(
687                "export '{}': specify exactly one of 'query', 'query_file', 'table', or 'tables' (got {} set)",
688                export.name,
689                set_count
690            );
691        }
692        // SecOps: syntactic `query_file` checks must run at config-validate
693        // time so `rivet check` / `rivet doctor` catch them before any
694        // plan step. The same checks repeat (with a canonicalize-based
695        // symlink probe) in `ExportConfig::resolve_query` because the
696        // file may have been swapped between validation and read.
697        if let Some(file) = &export.query_file {
698            let p = std::path::Path::new(file);
699            if p.is_absolute() {
700                anyhow::bail!(
701                    "export '{}': query_file must be a relative path: '{}'",
702                    export.name,
703                    file
704                );
705            }
706            if p.components().any(|c| c == std::path::Component::ParentDir) {
707                anyhow::bail!(
708                    "export '{}': query_file path must not contain '..': '{}'",
709                    export.name,
710                    file
711                );
712            }
713        }
714        // V2/V12: a custom cloud `endpoint` is handed straight to the opendal
715        // S3/GCS/Azure builder with no validation, so a committed config can
716        // silently redirect every upload to an attacker host (exfiltration) or
717        // send credentials + rows over cleartext `http://`. The legitimate use
718        // is a local emulator (Minio / Azurite / fake-gcs on `127.0.0.1`), so
719        // accept a loopback host (any scheme), and otherwise accept a remote
720        // endpoint only when the operator has explicitly opted into anonymous
721        // (emulator) mode. Reject every other custom endpoint at config-load.
722        if matches!(
723            export.destination.destination_type,
724            DestinationType::S3 | DestinationType::Gcs | DestinationType::Azure
725        ) && let Some(endpoint) = &export.destination.endpoint
726        {
727            // Loopback emulator (Minio/Azurite/fake-gcs) is the legitimate
728            // local-dev path — accept any scheme. A non-loopback (or
729            // unparseable) custom endpoint is only accepted when the operator
730            // has explicitly opted into anonymous (emulator) mode, where no
731            // credentials are sent. Everything else is rejected.
732            let loopback = endpoint_host(endpoint).is_some_and(|host| is_loopback_host(&host));
733            if !loopback && !export.destination.allow_anonymous {
734                anyhow::bail!(
735                    "export '{}': destination.endpoint '{}' points at a non-loopback host. \
736                     A custom endpoint redirects every upload there — committing one is a \
737                     data-exfiltration / cleartext-credential risk.\n  \
738                     Hint: drop `endpoint:` to use the provider default, point it at a \
739                     loopback emulator (e.g. http://127.0.0.1:9000 with allow_anonymous: true \
740                     for Minio/Azurite), or set `allow_anonymous: true` for an anonymous \
741                     emulator.",
742                    export.name,
743                    endpoint,
744                );
745            }
746        }
747
748        // V15: a `type: local` destination `path` (or `prefix`) is written
749        // verbatim to the filesystem. A `..` component lets a committed config
750        // climb out of the intended output tree (`../../../../tmp/x`) — mirror
751        // the `query_file` traversal guard and reject it at config-load.
752        //
753        // Absolute paths are deliberately *not* rejected: `path: /output` is a
754        // legitimate Docker volume-mount pattern (see `examples/rivet.yaml`) and
755        // an explicit operator choice, not a hidden escape. The `..` climb is
756        // the unambiguous traversal footgun.
757        if export.destination.destination_type == DestinationType::Local {
758            for (field, value) in [
759                ("path", export.destination.path.as_deref()),
760                ("prefix", export.destination.prefix.as_deref()),
761            ] {
762                let Some(value) = value else { continue };
763                if std::path::Path::new(value)
764                    .components()
765                    .any(|c| c == std::path::Component::ParentDir)
766                {
767                    anyhow::bail!(
768                        "export '{}': local destination {} must not contain a '..' component: \
769                         '{}' (a parent-dir climb writes outside the output tree).",
770                        export.name,
771                        field,
772                        value
773                    );
774                }
775            }
776        }
777
778        if export.destination.destination_type == DestinationType::S3 {
779            let ak = export.destination.access_key_env.is_some();
780            let sk = export.destination.secret_key_env.is_some();
781            if ak != sk {
782                anyhow::bail!(
783                    "export '{}': S3 requires both access_key_env and secret_key_env, or neither (use default AWS credential chain)",
784                    export.name
785                );
786            }
787        }
788
789        if export.destination.destination_type == DestinationType::Gcs
790            && export.destination.allow_anonymous
791            && export.destination.credentials_file.is_some()
792        {
793            anyhow::bail!(
794                "export '{}': GCS allow_anonymous cannot be used together with credentials_file",
795                export.name
796            );
797        }
798
799        if export.destination.destination_type == DestinationType::Azure {
800            let has_name = export.destination.account_name.is_some();
801            let has_key = export.destination.account_key_env.is_some();
802            let has_sas = export.destination.sas_token_env.is_some();
803            if export.destination.allow_anonymous {
804                if has_name || has_key || has_sas {
805                    anyhow::bail!(
806                        "export '{}': Azure allow_anonymous cannot be combined with account_name/account_key_env/sas_token_env",
807                        export.name
808                    );
809                }
810            } else if has_key && has_sas {
811                anyhow::bail!(
812                    "export '{}': Azure account_key_env and sas_token_env are mutually exclusive — pick one auth mode",
813                    export.name
814                );
815            } else if !has_name {
816                anyhow::bail!(
817                    "export '{}': Azure requires account_name (plus account_key_env or sas_token_env), or allow_anonymous: true for Azurite",
818                    export.name
819                );
820            } else if !has_key && !has_sas {
821                anyhow::bail!(
822                    "export '{}': Azure requires account_key_env or sas_token_env (or allow_anonymous: true for Azurite)",
823                    export.name
824                );
825            }
826        }
827
828        if let Some(cred_path) = &export.destination.credentials_file
829            && !std::path::Path::new(cred_path).exists()
830        {
831            anyhow::bail!(
832                "export '{}': credentials_file '{}' does not exist",
833                export.name,
834                cred_path
835            );
836        }
837
838        if let Some(ref size_str) = export.max_file_size {
839            parse_file_size(size_str).map_err(|_| {
840                anyhow::anyhow!(
841                    "export '{}': invalid max_file_size '{}'",
842                    export.name,
843                    size_str
844                )
845            })?;
846        }
847
848        if let Some(level) = export.compression_level {
849            match export.compression {
850                CompressionType::Zstd => {
851                    if !(1..=22).contains(&level) {
852                        anyhow::bail!(
853                            "export '{}': zstd compression_level must be 1..22, got {}",
854                            export.name,
855                            level
856                        );
857                    }
858                }
859                CompressionType::Gzip => {
860                    if level > 10 {
861                        anyhow::bail!(
862                            "export '{}': gzip compression_level must be 0..10, got {}",
863                            export.name,
864                            level
865                        );
866                    }
867                }
868                _ => {
869                    anyhow::bail!(
870                        "export '{}': compression_level is only supported for zstd and gzip",
871                        export.name
872                    );
873                }
874            }
875        }
876
877        match export.mode {
878            ExportMode::Incremental => {
879                if export.cursor_column.is_none() {
880                    anyhow::bail!(
881                        "export '{}': incremental mode requires cursor_column",
882                        export.name
883                    );
884                }
885                match export.incremental_cursor_mode {
886                    IncrementalCursorMode::Coalesce => {
887                        if export.cursor_fallback_column.is_none() {
888                            anyhow::bail!(
889                                "export '{}': incremental_cursor_mode: coalesce requires cursor_fallback_column",
890                                export.name
891                            );
892                        }
893                    }
894                    IncrementalCursorMode::SingleColumn => {
895                        if export.cursor_fallback_column.is_some() {
896                            anyhow::bail!(
897                                "export '{}': cursor_fallback_column is only valid with incremental_cursor_mode: coalesce",
898                                export.name
899                            );
900                        }
901                    }
902                }
903            }
904            ExportMode::Chunked => {
905                // `chunk_column` is mandatory unless the user used the `table:`
906                // shortcut on a Postgres source — in that case it is auto-resolved
907                // from the table's single-integer PK at plan-build time (see
908                // `crate::plan::build::resolve_chunk_column`).
909                if export.chunk_column.is_none() && export.table.is_none() {
910                    anyhow::bail!(
911                        "export '{}': chunked mode needs a chunking strategy. Pick one:\n  \
912                         chunk_column: <int col>    range chunks on an integer column (most common)\n  \
913                         chunk_by_key: <unique col>  keyset pagination when there's no integer PK\n  \
914                         chunk_count: <N>            split the range into N equal chunks\n  \
915                         chunk_by_days: <D>          time-bucketed chunks (needs a date/timestamp column)\n  \
916                         Or use the `table:` shortcut on a single table — rivet auto-resolves the column from the primary key.",
917                        export.name
918                    );
919                }
920                // chunk_size == 0 would divide the range into zero-width
921                // slices and (before the saturating fix in generate_chunks)
922                // either infinite-loop or produce no progress. QA backlog
923                // Task 5.1.
924                if export.chunk_size == 0 {
925                    anyhow::bail!(
926                        "export '{}': chunked mode requires chunk_size >= 1 (got 0)",
927                        export.name
928                    );
929                }
930                // parallel == 0 means "spawn zero workers". Claiming tasks
931                // with no workers stalls the pipeline. QA backlog Task 5.1.
932                if export.parallel == 0 {
933                    anyhow::bail!(
934                        "export '{}': chunked mode requires parallel >= 1 (got 0)",
935                        export.name
936                    );
937                }
938                if let Some(0) = export.chunk_count {
939                    crate::config_bail!(
940                        crate::error::codes::CONFIG_CHUNK_COUNT_INVALID,
941                        "export '{}': chunk_count must be >= 1",
942                        export.name
943                    );
944                }
945                if export.chunk_count.is_some() && export.chunk_dense {
946                    anyhow::bail!(
947                        "export '{}': chunk_count and chunk_dense are mutually exclusive. \
948                         Use chunk_count for equal-sized chunks over a sparse key; \
949                         use chunk_dense only when the key has no gaps.",
950                        export.name
951                    );
952                }
953                if export.chunk_count.is_some() && export.chunk_by_days.is_some() {
954                    anyhow::bail!(
955                        "export '{}': chunk_count and chunk_by_days are mutually exclusive. \
956                         Use chunk_count: N to split an integer range into N chunks; \
957                         use chunk_by_days: D to bucket a date/timestamp column by D-day windows.",
958                        export.name
959                    );
960                }
961            }
962            ExportMode::TimeWindow => {
963                if export.time_column.is_none() {
964                    anyhow::bail!(
965                        "export '{}': time_window mode requires time_column",
966                        export.name
967                    );
968                }
969                if export.days_window.is_none() {
970                    anyhow::bail!(
971                        "export '{}': time_window mode requires days_window",
972                        export.name
973                    );
974                }
975            }
976            ExportMode::Full => {}
977            ExportMode::Cdc => {
978                match (&export.table, &export.tables) {
979                    (None, None) => anyhow::bail!(
980                        "export '{}': cdc mode requires `table:` (or `tables:` for a \
981                         multi-table stream)",
982                        export.name
983                    ),
984                    (Some(_), Some(_)) => anyhow::bail!(
985                        "export '{}': `table:` and `tables:` are mutually exclusive — \
986                         use `tables: [a, b]` for a multi-table stream",
987                        export.name
988                    ),
989                    (None, Some(ts)) => {
990                        if ts.is_empty() {
991                            anyhow::bail!(
992                                "export '{}': `tables:` must list at least one table",
993                                export.name
994                            );
995                        }
996                        let mut seen = std::collections::HashSet::new();
997                        for t in ts {
998                            if !seen.insert(t.as_str()) {
999                                anyhow::bail!(
1000                                    "export '{}': duplicate table '{}' in `tables:`",
1001                                    export.name,
1002                                    t
1003                                );
1004                            }
1005                        }
1006                        if self.source.source_type == SourceType::Mssql {
1007                            anyhow::bail!(
1008                                "export '{}': `tables:` is not yet supported for SQL Server — \
1009                                 its capture instances are per-table; use one cdc export per \
1010                                 table (capture_instance each)",
1011                                export.name
1012                            );
1013                        }
1014                    }
1015                    (Some(_), None) => {}
1016                }
1017                if export.query.is_some() || export.query_file.is_some() {
1018                    anyhow::bail!(
1019                        "export '{}': cdc mode reads the transaction log, not a query — \
1020                         remove query/query_file and use `table:`",
1021                        export.name
1022                    );
1023                }
1024            }
1025        }
1026
1027        if export.tables.is_some() && export.mode != ExportMode::Cdc {
1028            anyhow::bail!(
1029                "export '{}': `tables:` is only valid with `mode: cdc` (batch exports \
1030                 are one query/table per export)",
1031                export.name
1032            );
1033        }
1034
1035        if export.chunk_dense && export.mode != ExportMode::Chunked {
1036            anyhow::bail!(
1037                "export '{}': chunk_dense is only valid with mode: chunked",
1038                export.name
1039            );
1040        }
1041
1042        if export.cdc.is_some() && export.mode != ExportMode::Cdc {
1043            anyhow::bail!(
1044                "export '{}': a `cdc:` block is only valid with `mode: cdc`",
1045                export.name
1046            );
1047        }
1048
1049        // Table-qualified `columns:` override keys ("table.column"): the named
1050        // table must be one this export captures — a typo must fail at load,
1051        // never silently miss its target. Bare keys stay export-wide.
1052        {
1053            for key in export.columns.keys() {
1054                let Some((tbl, _col)) = key.split_once('.') else {
1055                    continue;
1056                };
1057                if export.query.is_some() || export.query_file.is_some() {
1058                    anyhow::bail!(
1059                        "export '{}': qualified column override '{key}' needs a table-shaped \
1060                         export (`table:`/`tables:`), not a query",
1061                        export.name
1062                    );
1063                }
1064                let bare = |t: &str| t.rsplit('.').next().unwrap_or(t).to_string();
1065                let captures = export
1066                    .table
1067                    .as_deref()
1068                    .map(bare)
1069                    .into_iter()
1070                    .chain(export.tables.iter().flatten().map(|t| bare(t)))
1071                    .any(|t| t == tbl);
1072                if !captures {
1073                    anyhow::bail!(
1074                        "export '{}': column override '{key}' names table '{tbl}', which this \
1075                         export does not capture — fix the table name or use a bare column key \
1076                         to apply it to every captured table",
1077                        export.name
1078                    );
1079                }
1080            }
1081        }
1082
1083        // `initial: snapshot` writes each table's snapshot under the reserved
1084        // sub-prefix `snapshot/` — a table actually NAMED "snapshot" would share
1085        // a prefix with another table's marker. Refuse the collision at load.
1086        if let Some(cdc) = &export.cdc
1087            && cdc.initial == Some(CdcInitialMode::Snapshot)
1088        {
1089            let clashes = |t: &str| t.rsplit('.').next().unwrap_or(t) == "snapshot";
1090            if export.table.as_deref().is_some_and(clashes)
1091                || export.tables.iter().flatten().any(|t| clashes(t))
1092            {
1093                anyhow::bail!(
1094                    "export '{}': a table named 'snapshot' collides with the reserved \
1095                     `snapshot/` sub-prefix that `cdc.initial: snapshot` writes — rename \
1096                     the table or use a separate export without `initial:`",
1097                    export.name
1098                );
1099            }
1100        }
1101
1102        // `initial: snapshot` needs a durable anchor BEFORE the snapshot reads.
1103        // PostgreSQL pins server-side (the slot); MySQL / SQL Server have no
1104        // server-side anchor, so the checkpoint file is mandatory there.
1105        if let Some(cdc) = &export.cdc
1106            && cdc.initial == Some(CdcInitialMode::Snapshot)
1107            && self.source.source_type != SourceType::Postgres
1108            && cdc.checkpoint.is_none()
1109        {
1110            anyhow::bail!(
1111                "export '{}': `cdc.initial: snapshot` on {:?} requires `cdc.checkpoint:` — \
1112                 the checkpoint file is the anchor that makes snapshot-then-stream gap-free",
1113                export.name,
1114                self.source.source_type
1115            );
1116        }
1117
1118        // MongoDB change streams have NO server-side resume anchor (unlike a
1119        // PostgreSQL slot): the checkpoint file IS the anchor. Without it, every
1120        // run re-anchors at "now" and silently loses every change since the last
1121        // run — so `mode: cdc` on mongo requires `cdc.checkpoint:` ALWAYS, not
1122        // only under `initial: snapshot` (bug-hunt find).
1123        if export.mode == ExportMode::Cdc
1124            && self.source.source_type == SourceType::Mongo
1125            && export
1126                .cdc
1127                .as_ref()
1128                .and_then(|c| c.checkpoint.as_ref())
1129                .is_none()
1130        {
1131            anyhow::bail!(
1132                "export '{}': MongoDB `mode: cdc` requires `cdc.checkpoint:` — a change \
1133                 stream has no server-side resume anchor, so without the checkpoint file \
1134                 each run re-anchors at the current time and silently loses every change \
1135                 between runs.",
1136                export.name
1137            );
1138        }
1139
1140        // A collection whose name contains a dot does not route through the
1141        // change-stream capture — its events are silently dropped. Refuse loudly
1142        // rather than report 0-row success forever (bug-hunt find). Batch handles
1143        // dotted names fine, so this is CDC-only.
1144        if export.mode == ExportMode::Cdc && self.source.source_type == SourceType::Mongo {
1145            let dotted = |t: &str| t.contains('.');
1146            if export.table.as_deref().is_some_and(dotted)
1147                || export.tables.iter().flatten().any(|t| dotted(t))
1148            {
1149                anyhow::bail!(
1150                    "export '{}': MongoDB `mode: cdc` does not support a collection name \
1151                     containing a dot — the change-stream router cannot address it and its \
1152                     events would be silently dropped. Rename the collection, or capture it \
1153                     with `mode: full` (batch handles dotted names).",
1154                    export.name
1155                );
1156            }
1157        }
1158
1159        if let Some(days) = export.chunk_by_days {
1160            if export.mode != ExportMode::Chunked {
1161                anyhow::bail!(
1162                    "export '{}': chunk_by_days requires mode: chunked",
1163                    export.name
1164                );
1165            }
1166            if export.chunk_dense {
1167                anyhow::bail!(
1168                    "export '{}': chunk_by_days cannot be combined with chunk_dense",
1169                    export.name
1170                );
1171            }
1172            if days == 0 {
1173                crate::config_bail!(
1174                    crate::error::codes::CONFIG_CHUNK_BY_DAYS_INVALID,
1175                    "export '{}': chunk_by_days must be at least 1",
1176                    export.name
1177                );
1178            }
1179        }
1180        Ok(())
1181    }
1182}
1183
1184/// True when a single YAML line carries a mapping value (text after `key:`)
1185/// that contains a `{` outside of any quotes — the unquoted-template-brace
1186/// shape (`prefix: {partition}`, `path: {date}/out`).
1187///
1188/// Quote-aware so a properly quoted value (`prefix: "exports/{partition}/"`)
1189/// does *not* match, and `$`-prefixed braces (`${VAR}` env placeholders) are
1190/// ignored — they are resolved before the parse and are not the footgun.
1191fn line_has_unquoted_brace_value(line: &str) -> bool {
1192    // Whole-line comments never carry a value — skip before splitting.
1193    if line.trim_start().starts_with('#') {
1194        return false;
1195    }
1196    // Split key from value at the first `": "` / `":\t"` / trailing `:`.
1197    // A YAML plain-key separator is a colon followed by whitespace or EOL.
1198    let bytes = line.as_bytes();
1199    let mut sep = None;
1200    let mut i = 0;
1201    while i < bytes.len() {
1202        if bytes[i] == b':' && (i + 1 == bytes.len() || bytes[i + 1].is_ascii_whitespace()) {
1203            sep = Some(i + 1);
1204            break;
1205        }
1206        i += 1;
1207    }
1208    let Some(value_start) = sep else {
1209        return false;
1210    };
1211    let value = line[value_start..].trim_start();
1212    // A trailing `#` after the value starts an inline comment; an empty or
1213    // comment-only value carries no brace to flag.
1214    if value.is_empty() || value.starts_with('#') {
1215        return false;
1216    }
1217
1218    let mut in_single = false;
1219    let mut in_double = false;
1220    let vbytes = value.as_bytes();
1221    for (j, &c) in vbytes.iter().enumerate() {
1222        match c {
1223            b'\'' if !in_double => in_single = !in_single,
1224            b'"' if !in_single => in_double = !in_double,
1225            b'{' if !in_single && !in_double => {
1226                // Ignore `${...}` env placeholders (resolved pre-parse).
1227                if j > 0 && vbytes[j - 1] == b'$' {
1228                    continue;
1229                }
1230                return true;
1231            }
1232            _ => {}
1233        }
1234    }
1235    false
1236}
1237
1238/// Extract the lower-cased host from a `scheme://host[:port][/path]` endpoint,
1239/// or `None` when it does not look like a URL.
1240///
1241/// SecOps helper for the cloud-`endpoint` exfiltration guard (V2/V12): the host
1242/// decides whether a custom endpoint is a local emulator (loopback) or a remote
1243/// redirect target. We reject every non-loopback custom endpoint regardless of
1244/// scheme (covering both the exfil and the cleartext-`http` gaps), so only the
1245/// host is needed. We hand-parse rather than pull in a URL crate — the inputs
1246/// are operator-typed endpoints, not arbitrary URIs. A bracketed IPv6 literal
1247/// authority (`http://[::1]:9000`) keeps its address so it compares against the
1248/// loopback list.
1249fn endpoint_host(endpoint: &str) -> Option<String> {
1250    let (scheme, rest) = endpoint.split_once("://")?;
1251    if scheme.is_empty() {
1252        return None;
1253    }
1254    // Authority ends at the first `/` (path), `?` (query), or `#` (fragment);
1255    // any `user[:pass]@` userinfo head is dropped (host is after the last `@`).
1256    let authority = rest
1257        .split(['/', '?', '#'])
1258        .next()
1259        .unwrap_or("")
1260        .rsplit('@')
1261        .next()
1262        .unwrap_or("");
1263    let host = if let Some(stripped) = authority.strip_prefix('[') {
1264        // Bracketed IPv6 literal: take up to the closing `]`.
1265        stripped.split(']').next().unwrap_or("")
1266    } else {
1267        // host[:port] — strip the port suffix.
1268        authority.split(':').next().unwrap_or("")
1269    };
1270    if host.is_empty() {
1271        return None;
1272    }
1273    Some(host.to_ascii_lowercase())
1274}
1275
1276/// True when `host` names the local machine — the legitimate cloud-emulator
1277/// target (Minio / Azurite / fake-gcs on `127.0.0.1`). Anything else is a
1278/// remote host and a potential exfiltration redirect.
1279fn is_loopback_host(host: &str) -> bool {
1280    // `localhost` is the only non-IP host that counts as loopback. Everything
1281    // else must PARSE as an IP literal in the loopback range — a lexical
1282    // `starts_with("127.")` would accept attacker-controlled DNS like
1283    // `127.attacker.com` or `127.0.0.1.evil.com` (both resolve off-box), turning
1284    // the credential-exfil gate into a bypass (V2/V12). Parse strictly: only a
1285    // real `127.0.0.0/8` / `::1` address is loopback; a hostname is not.
1286    if host == "localhost" {
1287        return true;
1288    }
1289    // Tolerate a bracketed IPv6 literal (`[::1]`) in case a caller forwards one.
1290    let h = host
1291        .strip_prefix('[')
1292        .and_then(|s| s.strip_suffix(']'))
1293        .unwrap_or(host);
1294    h.parse::<std::net::IpAddr>()
1295        .is_ok_and(|ip| ip.is_loopback())
1296}
1297
1298/// True when `name` is filename-safe: rejects path-traversal (`..`), absolute
1299/// or slash-bearing names (`/`, `\`), a leading `.` (hidden / current-dir), and
1300/// embedded NULs. `ExportConfig.name` is keyed into output paths and on-disk
1301/// state, so a `../../etc/x` or absolute name escapes the output tree (V5).
1302fn is_filename_safe_name(name: &str) -> bool {
1303    !name.is_empty()
1304        && !name.starts_with('.')
1305        && !name.contains('/')
1306        && !name.contains('\\')
1307        && !name.contains("..")
1308        && !name.contains('\0')
1309}
1310
1311#[cfg(test)]
1312mod tests;
1313
1314#[cfg(test)]
1315mod audit_csv_compression {
1316    //! Finding #10: `format: csv` + a compression codec is a silent no-op
1317    //! (the file stays uncompressed but the manifest records the codec). The
1318    //! combo must be rejected at config-validate time. These tests encode the
1319    //! new rule, so reverting the fix turns them red.
1320    use super::*;
1321
1322    fn yaml(format: &str, compression_line: &str) -> String {
1323        format!(
1324            "source:\n  type: postgres\n  url: \"postgresql://localhost/test\"\n\
1325             exports:\n  - name: t\n    query: \"SELECT 1\"\n    format: {format}\n\
1326             {compression_line}    destination:\n      type: local\n      path: ./out\n"
1327        )
1328    }
1329
1330    #[test]
1331    fn audit_csv_compression_is_rejected() {
1332        // csv + gzip → rejected, with an actionable message.
1333        let err = Config::from_yaml(&yaml("csv", "    compression: gzip\n")).unwrap_err();
1334        let msg = format!("{err:#}");
1335        assert!(
1336            msg.contains("CSV output does not support compression") && msg.contains("gzip"),
1337            "csv+gzip must be rejected with an actionable message; got: {msg}"
1338        );
1339        assert!(
1340            msg.contains("parquet") && msg.contains("none"),
1341            "message must point to the real options (parquet / none); got: {msg}"
1342        );
1343
1344        // Guard the boundaries: parquet+gzip and csv+none still validate.
1345        Config::from_yaml(&yaml("parquet", "    compression: gzip\n"))
1346            .expect("parquet+gzip must validate");
1347        Config::from_yaml(&yaml("csv", "    compression: none\n")).expect("csv+none must validate");
1348    }
1349
1350    #[test]
1351    fn audit_csv_every_real_codec_is_rejected() {
1352        // Each non-None codec is a silent no-op for CSV — none may slip through.
1353        for codec in ["zstd", "snappy", "gzip", "lz4"] {
1354            let err = Config::from_yaml(&yaml("csv", &format!("    compression: {codec}\n")))
1355                .unwrap_err();
1356            let msg = format!("{err:#}");
1357            assert!(
1358                msg.contains("CSV output does not support compression") && msg.contains(codec),
1359                "csv+{codec} must be rejected; got: {msg}"
1360            );
1361        }
1362    }
1363
1364    #[test]
1365    fn audit_csv_compression_profile_is_rejected() {
1366        // A `compression_profile:` other than `none` resolves to a real codec,
1367        // so it is the same silent no-op for CSV.
1368        for profile in ["fast", "balanced", "compact"] {
1369            let err = Config::from_yaml(&yaml(
1370                "csv",
1371                &format!("    compression_profile: {profile}\n"),
1372            ))
1373            .unwrap_err();
1374            let msg = format!("{err:#}");
1375            assert!(
1376                msg.contains("CSV output does not support compression_profile")
1377                    && msg.contains(profile),
1378                "csv+profile {profile} must be rejected; got: {msg}"
1379            );
1380        }
1381        // profile: none is a no-op request and is fine.
1382        Config::from_yaml(&yaml("csv", "    compression_profile: none\n"))
1383            .expect("csv + compression_profile: none must validate");
1384    }
1385
1386    #[test]
1387    fn audit_csv_default_compression_still_validates() {
1388        // Regression guard: a bare `format: csv` (no explicit codec) must keep
1389        // validating. `CompressionType::default()` is `Zstd`, but the user did
1390        // not *ask* for it — only an explicit codec is a no-op request. This
1391        // pins that the fix scans for explicit intent, not the struct default
1392        // (which would break ~60 existing csv configs).
1393        Config::from_yaml(&yaml("csv", "")).expect("bare format: csv must validate");
1394    }
1395
1396    #[test]
1397    fn audit_compression_supported_predicate() {
1398        // `compression_supported` is re-exported via `pub use format::*`.
1399        // Parquet supports every codec; CSV supports only None.
1400        for ct in [
1401            CompressionType::Zstd,
1402            CompressionType::Snappy,
1403            CompressionType::Gzip,
1404            CompressionType::Lz4,
1405            CompressionType::None,
1406        ] {
1407            assert!(compression_supported(FormatType::Parquet, ct));
1408        }
1409        assert!(compression_supported(
1410            FormatType::Csv,
1411            CompressionType::None
1412        ));
1413        for ct in [
1414            CompressionType::Zstd,
1415            CompressionType::Snappy,
1416            CompressionType::Gzip,
1417            CompressionType::Lz4,
1418        ] {
1419            assert!(
1420                !compression_supported(FormatType::Csv, ct),
1421                "CSV must not claim to support {}",
1422                ct.label()
1423            );
1424        }
1425    }
1426}
1427
1428#[cfg(test)]
1429mod reserved_load_extension {
1430    //! A downstream first-party loader carries its warehouse target in a
1431    //! top-level `load:` block so ONE config drives export + load. OSS must
1432    //! accept and ignore it — otherwise `check`/`run`/`apply` would reject the
1433    //! single-file config. Reverting the reserved field turns this red.
1434    use super::*;
1435
1436    #[test]
1437    fn reserved_top_level_load_block_is_accepted_and_ignored() {
1438        let yaml = "source:\n  type: postgres\n  url: \"postgresql://localhost/test\"\n\
1439             exports:\n  - name: t\n    query: \"SELECT 1\"\n    format: parquet\n    destination:\n      type: gcs\n      bucket: b\n      prefix: \"t/\"\n\
1440             load:\n  project: p\n  dataset: d\n  cleanup_source: true\n";
1441        let cfg = Config::from_yaml(yaml).expect("config with a reserved `load:` block must parse");
1442        assert!(cfg.load.is_some(), "the load block is captured opaquely");
1443    }
1444}
1445
1446#[cfg(test)]
1447mod audit_unquoted_template_brace {
1448    //! yaml-hint: an unquoted `{partition}` (or `{date}`) in a path/prefix
1449    //! value trips serde_yaml_ng's flow-mapping parser with a cryptic message
1450    //! that gives no clue the brace needs quoting. Since `{partition}` is the
1451    //! required token for `partition_by`, this is a common copy-paste footgun.
1452    //! `Config::from_yaml` augments the parser error with a quoting hint; these
1453    //! tests pin that behavior (and guard that valid configs are untouched).
1454    use super::*;
1455
1456    /// A full, otherwise-valid config whose `prefix:` value is whatever the
1457    /// caller passes verbatim (quoted or not). Only the `prefix:` line varies,
1458    /// so any parse error is attributable to the brace under test.
1459    fn yaml_with_prefix(prefix_value: &str) -> String {
1460        format!(
1461            "source:\n\
1462             \x20 type: postgres\n\
1463             \x20 url: \"postgresql://localhost/test\"\n\
1464             exports:\n\
1465             \x20 - name: t\n\
1466             \x20   query: \"SELECT 1\"\n\
1467             \x20   format: parquet\n\
1468             \x20   partition_by: created_date\n\
1469             \x20   destination:\n\
1470             \x20     type: local\n\
1471             \x20     path: ./out\n\
1472             \x20     prefix: {prefix_value}\n"
1473        )
1474    }
1475
1476    const HINT_FRAGMENT: &str =
1477        "a YAML value containing { } (such as {partition} or {date}) must be quoted";
1478
1479    #[test]
1480    fn bare_partition_token_gets_quoting_hint() {
1481        // `prefix: {partition}` parses as a YAML map, so serde rejects it with
1482        // `invalid type: map, expected a string` — no clue it's a quoting bug.
1483        let err = Config::from_yaml(&yaml_with_prefix("{partition}")).unwrap_err();
1484        let msg = format!("{err:#}");
1485        assert!(
1486            msg.contains(HINT_FRAGMENT),
1487            "bare {{partition}} must carry the quoting hint; got: {msg}"
1488        );
1489        // The original parser detail (type + location) is preserved.
1490        assert!(
1491            msg.contains("invalid type: map") || msg.contains("line"),
1492            "the original parser error must be kept; got: {msg}"
1493        );
1494    }
1495
1496    #[test]
1497    fn trailing_text_after_brace_gets_quoting_hint() {
1498        // `prefix: {date}/{partition}/` runs the flow map into block context:
1499        // serde emits `did not find expected key ... while parsing a block
1500        // mapping`. Same footgun, different libyaml symptom.
1501        let err = Config::from_yaml(&yaml_with_prefix("{date}/{partition}/")).unwrap_err();
1502        let msg = format!("{err:#}");
1503        assert!(
1504            msg.contains(HINT_FRAGMENT),
1505            "{{date}}/{{partition}}/ must carry the quoting hint; got: {msg}"
1506        );
1507    }
1508
1509    #[test]
1510    fn unclosed_brace_gets_quoting_hint() {
1511        // `prefix: {partition` (unclosed) is the canonical flow-mapping scanner
1512        // error: `did not find expected ',' or '}' ... while parsing a flow
1513        // mapping`. The hint must still fire.
1514        let err = Config::from_yaml(&yaml_with_prefix("{partition")).unwrap_err();
1515        let msg = format!("{err:#}");
1516        assert!(
1517            msg.contains(HINT_FRAGMENT),
1518            "unclosed brace must carry the quoting hint; got: {msg}"
1519        );
1520    }
1521
1522    #[test]
1523    fn quoted_brace_value_loads_ok() {
1524        // The fix itself, applied: a properly quoted brace value parses and
1525        // validates. This is the guard that the hint never reaches a valid
1526        // config and the success path is unchanged.
1527        let cfg = Config::from_yaml(&yaml_with_prefix("\"exports/{partition}/\""))
1528            .expect("quoted {partition} prefix must load");
1529        assert_eq!(
1530            cfg.exports[0].destination.prefix.as_deref(),
1531            Some("exports/{partition}/")
1532        );
1533    }
1534
1535    #[test]
1536    fn config_without_braces_is_untouched() {
1537        // No brace anywhere: a plain valid config still loads, and an unrelated
1538        // YAML error elsewhere must not pick up a spurious quoting hint.
1539        Config::from_yaml(&yaml_with_prefix("exports/data/"))
1540            .expect("a brace-free prefix must load");
1541    }
1542
1543    // ── line_has_unquoted_brace_value() unit coverage ──────────────────────
1544
1545    #[test]
1546    fn unquoted_brace_value_is_detected() {
1547        assert!(line_has_unquoted_brace_value("    prefix: {partition}"));
1548        assert!(line_has_unquoted_brace_value("      path: {date}/out"));
1549        assert!(line_has_unquoted_brace_value("prefix: {partition")); // unclosed
1550    }
1551
1552    #[test]
1553    fn quoted_brace_value_is_not_flagged() {
1554        // Quotes around the value hide the brace from the scanner — not a bug.
1555        assert!(!line_has_unquoted_brace_value(
1556            "    prefix: \"exports/{partition}/\""
1557        ));
1558        assert!(!line_has_unquoted_brace_value("    prefix: 'data/{date}/'"));
1559    }
1560
1561    #[test]
1562    fn env_placeholder_and_plain_values_are_not_flagged() {
1563        // `${VAR}` placeholders are resolved before the parse and are not the
1564        // footgun; plain brace-free values are obviously fine.
1565        assert!(!line_has_unquoted_brace_value("    url: ${DATABASE_URL}"));
1566        assert!(!line_has_unquoted_brace_value("    path: ./out"));
1567        assert!(!line_has_unquoted_brace_value("  # prefix: {partition}")); // comment
1568        assert!(!line_has_unquoted_brace_value("    prefix:")); // no value
1569    }
1570}
1571
1572#[cfg(test)]
1573mod sec_config_validation_regression {
1574    //! Regression edge-cases that pin the *compat boundaries* of the
1575    //! config-validation security fixes — the cases that distinguish a real
1576    //! attack from a legitimate loopback / dev-container / Docker pattern.
1577    //! These complement the RED tests in `sec_config_validation`: the RED
1578    //! tests assert the attack is rejected; these assert the fix stays narrow
1579    //! enough not to break local-dev usage (see CRITICAL COMPAT).
1580    use super::*;
1581
1582    /// A full, otherwise-valid config whose single export's `destination:`
1583    /// block is whatever the caller passes verbatim.
1584    fn yaml_with_destination(dest_block: &str) -> String {
1585        format!(
1586            "source:\n  type: postgres\n  url: \"postgresql://localhost/test\"\n\
1587             exports:\n  - name: t\n    query: \"SELECT 1\"\n    format: parquet\n\
1588             {dest_block}"
1589        )
1590    }
1591
1592    // ── endpoint_host / is_loopback_host helpers ─────────────────────────────
1593
1594    #[test]
1595    fn endpoint_host_parses_forms() {
1596        assert_eq!(
1597            endpoint_host("https://attacker.example.com").as_deref(),
1598            Some("attacker.example.com")
1599        );
1600        // Port and path are stripped from the host.
1601        assert_eq!(
1602            endpoint_host("http://127.0.0.1:10000/devstoreaccount1").as_deref(),
1603            Some("127.0.0.1")
1604        );
1605        // userinfo head is dropped (host is after the last `@`).
1606        assert_eq!(
1607            endpoint_host("http://user:pass@127.0.0.1:9000").as_deref(),
1608            Some("127.0.0.1")
1609        );
1610        // Bracketed IPv6 literal keeps its address.
1611        assert_eq!(endpoint_host("http://[::1]:9000").as_deref(), Some("::1"));
1612        // Not a URL → None (treated as a non-loopback custom endpoint upstream).
1613        assert_eq!(endpoint_host("not-a-url"), None);
1614        assert_eq!(endpoint_host("://nohost"), None);
1615    }
1616
1617    #[test]
1618    fn loopback_host_classification() {
1619        for h in ["127.0.0.1", "127.0.0.53", "localhost", "::1"] {
1620            assert!(is_loopback_host(h), "{h} must be loopback");
1621        }
1622        for h in ["attacker.example.com", "evil.com", "10.0.0.1", "::2"] {
1623            assert!(!is_loopback_host(h), "{h} must be remote");
1624        }
1625    }
1626
1627    // ── V2/V12 endpoint: loopback accepted regardless of allow_anonymous ─────
1628
1629    #[test]
1630    fn loopback_endpoint_without_allow_anonymous_still_accepted() {
1631        // A loopback emulator endpoint with credentials (no allow_anonymous) is
1632        // the Minio-with-keys local-dev pattern and must stay accepted — the
1633        // exfil guard targets *remote* hosts, not localhost.
1634        let cfg = yaml_with_destination(
1635            "    destination:\n      type: s3\n      bucket: b\n      region: us-east-1\n\
1636             \x20     endpoint: http://127.0.0.1:9000\n      access_key_env: AK\n      secret_key_env: SK\n",
1637        );
1638        Config::from_yaml(&cfg).expect("loopback endpoint with creds must stay accepted");
1639    }
1640
1641    #[test]
1642    fn remote_https_endpoint_with_allow_anonymous_is_the_only_remote_escape() {
1643        // The documented escape hatch: an explicit anonymous (emulator) opt-in
1644        // permits a non-loopback endpoint (no credentials are sent). Without
1645        // allow_anonymous the same endpoint is rejected (covered by the RED
1646        // test); with it, accepted.
1647        let cfg = yaml_with_destination(
1648            "    destination:\n      type: gcs\n      bucket: b\n\
1649             \x20     endpoint: https://emulator.example.com\n      allow_anonymous: true\n",
1650        );
1651        Config::from_yaml(&cfg).expect("remote endpoint + allow_anonymous opt-in must be accepted");
1652    }
1653
1654    // ── V15 local path: absolute allowed (Docker mount), `..` rejected ───────
1655
1656    #[test]
1657    fn absolute_local_path_is_allowed() {
1658        // `path: /output` is a legitimate Docker volume-mount pattern
1659        // (examples/rivet.yaml) and must keep validating — only `..` climbs are
1660        // the traversal footgun.
1661        let cfg =
1662            yaml_with_destination("    destination:\n      type: local\n      path: /output\n");
1663        Config::from_yaml(&cfg).expect("absolute local path (Docker mount) must validate");
1664    }
1665
1666    #[test]
1667    fn dotdot_in_local_prefix_is_rejected() {
1668        // `prefix` is guarded the same as `path`.
1669        let cfg = yaml_with_destination(
1670            "    destination:\n      type: local\n      path: ./out\n      prefix: a/../b\n",
1671        );
1672        let err = Config::from_yaml(&cfg).unwrap_err();
1673        let msg = format!("{err:#}");
1674        assert!(
1675            msg.contains("prefix") && msg.contains(".."),
1676            "a '..' in the local prefix must be rejected naming prefix/..; got: {msg}"
1677        );
1678    }
1679
1680    // ── V13 TLS: explicit enforced mode + knob rejected; default-mode kept ───
1681
1682    #[test]
1683    fn tls_danger_knob_without_explicit_mode_still_accepted() {
1684        // The dev-container pattern `tls: { accept_invalid_certs: true }` against
1685        // a loopback self-signed cert (e.g. the MSSQL docker container) omits
1686        // `mode:` — there is no *explicit* mode to contradict, so it must keep
1687        // validating. The RED test rejects only the explicit-mode contradiction.
1688        let yaml = "source:\n  type: mssql\n  url: \"sqlserver://sa:pw@127.0.0.1:1433/db\"\n  \
1689                    tls:\n    accept_invalid_certs: true\n\
1690                    exports:\n  - name: t\n    query: \"SELECT 1\"\n    format: parquet\n    \
1691                    destination:\n      type: local\n      path: ./out\n";
1692        Config::from_yaml(yaml)
1693            .expect("dev-container default-mode + accept_invalid_certs must stay accepted");
1694    }
1695
1696    #[test]
1697    fn tls_explicit_verify_ca_plus_invalid_hostnames_rejected() {
1698        // The hostname knob is flagged too, against any explicit enforced mode.
1699        let yaml = "source:\n  type: postgres\n  url: \"postgresql://localhost/test\"\n  \
1700                    tls:\n    mode: verify-ca\n    accept_invalid_hostnames: true\n\
1701                    exports:\n  - name: t\n    query: \"SELECT 1\"\n    format: parquet\n    \
1702                    destination:\n      type: local\n      path: ./out\n";
1703        let err = Config::from_yaml(yaml).unwrap_err();
1704        let msg = format!("{err:#}");
1705        assert!(
1706            msg.contains("accept_invalid_hostnames") && msg.contains("verify-ca"),
1707            "explicit verify-ca + accept_invalid_hostnames must be rejected; got: {msg}"
1708        );
1709    }
1710
1711    #[test]
1712    fn tls_explicit_disable_with_knob_is_not_flagged() {
1713        // `mode: disable` carries no verification promise to contradict, so the
1714        // danger knob is a no-op there and must not be rejected.
1715        let yaml = "source:\n  type: postgres\n  url: \"postgresql://localhost/test\"\n  \
1716                    tls:\n    mode: disable\n    accept_invalid_certs: true\n\
1717                    exports:\n  - name: t\n    query: \"SELECT 1\"\n    format: parquet\n    \
1718                    destination:\n      type: local\n      path: ./out\n";
1719        Config::from_yaml(yaml).expect("mode: disable + knob is a no-op and must validate");
1720    }
1721
1722    // ── V5 name: filename-safe predicate boundaries ──────────────────────────
1723
1724    #[test]
1725    fn filename_safe_name_boundaries() {
1726        for ok in ["t", "orders", "daily_events", "v2-2024", "name.with.dots"] {
1727            assert!(is_filename_safe_name(ok), "{ok:?} must be accepted");
1728        }
1729        for bad in [
1730            "",
1731            "..",
1732            "../x",
1733            "/abs",
1734            "sub/dir",
1735            "back\\slash",
1736            ".hidden",
1737            "with\u{0000}nul",
1738        ] {
1739            assert!(!is_filename_safe_name(bad), "{bad:?} must be rejected");
1740        }
1741    }
1742}
1743
1744#[cfg(test)]
1745mod sec_config_validation {
1746    //! RED security tests for config-load validation gaps (cluster:
1747    //! config-validation). Each asserts the SECURE behavior through the
1748    //! stable `Config::from_yaml` seam: a malicious config that is accepted
1749    //! today must be REJECTED (or, for warn-only knobs, surfaced as an
1750    //! error/loud warning) at config-load. These are expected to FAIL until
1751    //! the corresponding production fix lands.
1752    //!
1753    //! The pattern mirrors the existing `query_file` `..`/absolute-path guard
1754    //! in `validate_export` (see `config/tests/validation.rs`): a syntactic
1755    //! check that runs at config-validate time so `rivet check` / `rivet
1756    //! doctor` catch the problem before any connect/plan/upload step.
1757    use super::*;
1758
1759    /// A full, otherwise-valid config whose single export's `destination:`
1760    /// block is whatever the caller passes verbatim. Only the destination
1761    /// varies, so any rejection is attributable to the destination under test.
1762    fn yaml_with_destination(dest_block: &str) -> String {
1763        format!(
1764            "source:\n  type: postgres\n  url: \"postgresql://localhost/test\"\n\
1765             exports:\n  - name: t\n    query: \"SELECT 1\"\n    format: parquet\n\
1766             {dest_block}"
1767        )
1768    }
1769
1770    // ── V2/V12: cloud-endpoint exfiltration + http cleartext ────────────────
1771    //
1772    // `destination.endpoint` is passed straight to the opendal S3/GCS/Azure
1773    // builder with no validation (see `src/destination/{s3,gcs,azure}.rs`),
1774    // so a committed config can silently redirect every export to an
1775    // attacker-controlled host. Two distinct gaps:
1776    //   V2  — a custom *non-loopback* endpoint (data exfiltration target).
1777    //   V12 — an `http://` (plaintext) endpoint (credentials + data on the
1778    //         wire in cleartext).
1779    // The secure behavior is to reject (or require explicit opt-in) at
1780    // config-load. Loopback/emulator endpoints (Minio/Azurite/fake-gcs on
1781    // 127.0.0.1) MUST stay accepted — that path is exercised by the existing
1782    // `gcs_allow_anonymous_parses` test and the guard test below.
1783
1784    #[test]
1785    fn sec_s3_custom_endpoint_rejected() {
1786        // SEC-RED V2: a non-loopback custom S3 endpoint is an exfiltration
1787        // target — every part upload goes to attacker.example.com. Must be
1788        // rejected (or require explicit opt-in) at config-load. Accepted today.
1789        let cfg = yaml_with_destination(
1790            "    destination:\n      type: s3\n      bucket: my-bucket\n      region: us-east-1\n\
1791             \x20     endpoint: https://attacker.example.com\n",
1792        );
1793        let res = Config::from_yaml(&cfg);
1794        assert!(
1795            res.is_err(),
1796            "a non-loopback custom S3 endpoint (https://attacker.example.com) must be \
1797             rejected at config-load (data-exfiltration target); got Ok"
1798        );
1799        let msg = format!("{:#}", res.unwrap_err());
1800        assert!(
1801            msg.contains("endpoint"),
1802            "rejection must name the offending 'endpoint' field; got: {msg}"
1803        );
1804    }
1805
1806    #[test]
1807    fn sec_http_endpoint_rejected() {
1808        // SEC-RED V12: a plaintext http:// endpoint to a *remote* host sends
1809        // credentials and exported rows over the wire in cleartext. Must be
1810        // rejected (or require explicit opt-in) at config-load. Accepted today.
1811        // Use a non-loopback host so this is distinct from the Minio/Azurite
1812        // loopback emulator case (guarded below).
1813        let cfg = yaml_with_destination(
1814            "    destination:\n      type: s3\n      bucket: my-bucket\n      region: us-east-1\n\
1815             \x20     endpoint: http://evil.com\n",
1816        );
1817        let res = Config::from_yaml(&cfg);
1818        assert!(
1819            res.is_err(),
1820            "a plaintext http:// endpoint to a remote host (http://evil.com) must be \
1821             rejected at config-load (cleartext credentials + data); got Ok"
1822        );
1823        let msg = format!("{:#}", res.unwrap_err());
1824        assert!(
1825            msg.contains("endpoint") || msg.to_lowercase().contains("http"),
1826            "rejection must name the endpoint / cleartext problem; got: {msg}"
1827        );
1828    }
1829
1830    #[test]
1831    fn sec_loopback_endpoint_still_accepted_guard() {
1832        // SEC-RED V2/V12 (guard): a loopback emulator endpoint
1833        // (`http://127.0.0.1:9000` Minio, with allow_anonymous) is the
1834        // legitimate local-dev path and MUST stay accepted after the fix.
1835        // This pins that the endpoint rejection targets *remote* hosts, not
1836        // localhost — otherwise the fix breaks every Minio/Azurite/fake-gcs
1837        // integration test (see `gcs_allow_anonymous_parses`).
1838        let cfg = yaml_with_destination(
1839            "    destination:\n      type: s3\n      bucket: my-bucket\n      region: us-east-1\n\
1840             \x20     endpoint: http://127.0.0.1:9000\n      allow_anonymous: true\n",
1841        );
1842        Config::from_yaml(&cfg)
1843            .expect("a loopback emulator endpoint with allow_anonymous must stay accepted");
1844    }
1845
1846    // ── V5: export `name` path traversal ────────────────────────────────────
1847    //
1848    // `ExportConfig.name` is a free-form `String` keyed into state tracking,
1849    // file logs, and (via the destination layout) output paths — yet it is
1850    // never validated. A name like `../../../etc/x`, an absolute `/abs/x`, a
1851    // bare slash, or an embedded NUL can escape the intended output tree.
1852    // Mirror the `query_file` `..`/absolute guard: reject at config-load.
1853
1854    #[test]
1855    fn sec_export_name_traversal_rejected() {
1856        // SEC-RED V5: a traversal / absolute / slash / NUL export name escapes
1857        // the output tree (and corrupts name-keyed state). Must be rejected at
1858        // config-load. Accepted today.
1859        for bad in ["../../../etc/x", "/abs/x", "sub/dir", "with\u{0000}nul"] {
1860            // `name:` is JSON-encoded so embedded slashes / NULs survive the
1861            // YAML parse verbatim and reach validation.
1862            let name_yaml = serde_json::to_string(bad).expect("encode name");
1863            let cfg = format!(
1864                "source:\n  type: postgres\n  url: \"postgresql://localhost/test\"\n\
1865                 exports:\n  - name: {name_yaml}\n    query: \"SELECT 1\"\n    format: parquet\n\
1866                 \x20   destination:\n      type: local\n      path: ./out\n"
1867            );
1868            let res = Config::from_yaml(&cfg);
1869            assert!(
1870                res.is_err(),
1871                "export name {bad:?} (traversal/absolute/slash/NUL) must be rejected at \
1872                 config-load; got Ok"
1873            );
1874            let msg = format!("{:#}", res.unwrap_err());
1875            assert!(
1876                msg.contains("name"),
1877                "rejection of name {bad:?} must name the offending 'name' field; got: {msg}"
1878            );
1879        }
1880    }
1881
1882    #[test]
1883    fn sec_export_name_normal_still_accepted_guard() {
1884        // SEC-RED V5 (guard): a plain, well-formed export name must keep
1885        // loading after the fix. Pins that the traversal check is narrow.
1886        let cfg = yaml_with_destination("    destination:\n      type: local\n      path: ./out\n");
1887        Config::from_yaml(&cfg).expect("a normal export name ('t') must stay accepted");
1888    }
1889
1890    // ── V15: local destination `path` traversal ─────────────────────────────
1891    //
1892    // `destination.path` for a `type: local` export is written verbatim to the
1893    // filesystem. A relative `../../../../tmp/x` or absolute path lets a
1894    // committed config write outside the intended output directory. Must be
1895    // rejected (or at minimum loudly surfaced) at config-load. Accepted today.
1896
1897    #[test]
1898    fn sec_local_dest_path_traversal_rejected() {
1899        // SEC-RED V15: a traversal local-destination path writes outside the
1900        // intended output tree. Must be rejected at config-load. Accepted today.
1901        let cfg = yaml_with_destination(
1902            "    destination:\n      type: local\n      path: ../../../../tmp/x\n",
1903        );
1904        let res = Config::from_yaml(&cfg);
1905        assert!(
1906            res.is_err(),
1907            "a local destination path containing '..' (../../../../tmp/x) must be rejected \
1908             at config-load (writes outside the output tree); got Ok"
1909        );
1910        let msg = format!("{:#}", res.unwrap_err());
1911        assert!(
1912            msg.contains("path") || msg.contains(".."),
1913            "rejection must name the offending 'path' / traversal; got: {msg}"
1914        );
1915    }
1916
1917    // ── V13: dangerous TLS cert-knob combination ─────────────────────────────
1918    //
1919    // `tls: { mode: verify-full, accept_invalid_certs: true }` silently
1920    // *downgrades* the strongest mode to "accept any cert" — `verify-full`
1921    // promises chain + hostname verification, but the danger knob disables
1922    // chain verification (see `src/source/tls.rs::build_native_tls`). The
1923    // comment at `src/source/tls.rs:55-56` claims "Each one emits a warning at
1924    // config-time (see `Config::validate`)" — but `Config::validate` emits no
1925    // such warning today. The secure behavior is a LOUD error (or surfaced
1926    // warning) at config-load. No `Err`/warning is produced today, so this is
1927    // RED.
1928
1929    #[test]
1930    fn sec_accept_invalid_certs_warns() {
1931        // SEC-RED V13: verify-full + accept_invalid_certs: true is a silent
1932        // security downgrade that contradicts the chosen mode. It must be
1933        // loudly surfaced at config-load. The only stable secure seam is an
1934        // `Err` from `Config::from_yaml` (validate returns Ok today, and there
1935        // is no captured-warning seam exposed from here — see notes). Asserting
1936        // `Err` is the strongest secure assertion and is RED against current
1937        // code.
1938        let cfg = yaml_with_destination("    destination:\n      type: local\n      path: ./out\n");
1939        // Splice the TLS block into the source rather than the destination so
1940        // the rest of the config stays valid.
1941        let cfg = cfg.replace(
1942            "  url: \"postgresql://localhost/test\"\n",
1943            "  url: \"postgresql://localhost/test\"\n  tls:\n    mode: verify-full\n    accept_invalid_certs: true\n",
1944        );
1945        let res = Config::from_yaml(&cfg);
1946        assert!(
1947            res.is_err(),
1948            "tls mode: verify-full with accept_invalid_certs: true is a silent security \
1949             downgrade and must be loudly surfaced (error) at config-load; got Ok"
1950        );
1951        let msg = format!("{:#}", res.unwrap_err());
1952        assert!(
1953            msg.contains("accept_invalid_certs") || msg.to_lowercase().contains("verify"),
1954            "the surfaced error must name the dangerous knob / mode contradiction; got: {msg}"
1955        );
1956    }
1957}