rivet/config/mod.rs
1pub mod cursor;
2mod destination;
3mod export;
4mod format;
5mod lints;
6mod notifications;
7pub mod resolve;
8pub mod schema;
9mod source;
10
11pub use cursor::IncrementalCursorMode;
12pub use destination::*;
13pub use export::*;
14pub use format::*;
15pub use notifications::*;
16#[allow(unused_imports)]
17pub(crate) use resolve::resolve_env_vars;
18pub use resolve::{parse_file_size, resolve_vars};
19pub use schema::generate_config_schema_pretty;
20pub use source::*;
21
22use schemars::JsonSchema;
23use serde::Deserialize;
24
25/// Top-level Rivet configuration root.
26///
27/// Operators write this struct as YAML (typically `rivet.yaml`). The
28/// `JsonSchema` derive is the source of truth for the `schemas/rivet.schema.json`
29/// artifact and the `rivet schema config` command's output (v0.7.3 P0).
30#[derive(Debug, Deserialize, JsonSchema, Clone)]
31#[serde(deny_unknown_fields)]
32pub struct Config {
33 pub source: SourceConfig,
34 pub exports: Vec<ExportConfig>,
35 #[serde(default)]
36 pub notifications: Option<NotificationsConfig>,
37 #[serde(default)]
38 pub parallel_exports: bool,
39 #[serde(default)]
40 pub parallel_export_processes: bool,
41 /// The warehouse **load** target — consumed by `rivet load` (and `rivet load
42 /// --cdc`), so ONE config drives both the export and the downstream load. The
43 /// extraction commands (`rivet check` / `run` / `apply`) accept and ignore it:
44 /// it shapes the load, not the extract.
45 // Kept as a raw `serde_json::Value` so the extraction path neither depends on
46 // nor validates the load schema — the load module (`crate::load::plan`) parses
47 // it into a typed `LoadSection`.
48 #[serde(default)]
49 pub load: Option<serde_json::Value>,
50}
51
52impl Config {
53 pub fn load(path: &str) -> crate::error::Result<Self> {
54 Self::load_with_params(path, None)
55 }
56
57 pub fn load_with_params(
58 path: &str,
59 params: Option<&std::collections::HashMap<String, String>>,
60 ) -> crate::error::Result<Self> {
61 // F11 (0.7.5 audit): raw `std::io::Error` lost the path on
62 // not-found. Wrap with the file path + a hint so the operator
63 // can see *which* config the tool could not open.
64 let contents = std::fs::read_to_string(path).map_err(|e| {
65 if e.kind() == std::io::ErrorKind::NotFound {
66 anyhow::anyhow!(
67 "config file '{}' not found.\n Hint: check the path, or run `rivet init` to generate one.",
68 path
69 )
70 } else {
71 anyhow::anyhow!("cannot read config file '{}': {}", path, e)
72 }
73 })?;
74 // Warn about typo'd `--param` keys once per CLI invocation, using the
75 // un-resolved YAML as the haystack so the placeholders are still there.
76 // We pass the raw `contents` (not `resolved`) on purpose: after
77 // resolution the placeholders are gone, and every key would look unused.
78 resolve::warn_unused_params(&contents, params);
79 let resolved = resolve_vars(&contents, params)?;
80 // F12 (0.7.5 audit): YAML parse errors did not name the config
81 // file. When loading from disk we know the path — thread it
82 // into the parse error.
83 // `.context` (not `anyhow!("…{:#}", e)`, which stringifies into a fresh
84 // error) so a typed `CodedError` raised by validation survives the chain —
85 // `error::error_code` downcasts it for the `[CODE]` prefix. `{e:#}` in
86 // `main` still renders the full "config file '…': <message>" chain.
87 Self::from_yaml(&resolved).map_err(|e| e.context(format!("config file '{}'", path)))
88 }
89
90 pub fn from_yaml(yaml: &str) -> crate::error::Result<Self> {
91 // Intercept the unquoted-`{partition}` footgun before the raw-YAML
92 // pre-scans below (they `from_str::<Value>` too and would re-emit the
93 // same cryptic libyaml message). A document that does not even parse as
94 // a `Value` is malformed; if the failure looks like a flow-mapping
95 // scanner error *and* the source carries an unquoted brace value, point
96 // straight at the quoting fix. On a valid config this `Value` parse
97 // succeeds and the block is skipped, so the success path is unchanged.
98 if let Err(e) = serde_yaml_ng::from_str::<serde_yaml_ng::Value>(yaml) {
99 let m = e.to_string();
100 if let Some(hint) = Self::unquoted_template_brace_hint(yaml, &m) {
101 return Err(anyhow::anyhow!("{e}\n {hint}"));
102 }
103 // A TAB in indentation is the most common beginner YAML mistake; it
104 // trips libyaml before serde ever sees a field, so it must be caught
105 // here in the raw scan, not in `enhance_parse_error`.
106 if let Some(hint) = Self::tab_indent_hint(yaml, &m) {
107 return Err(anyhow::anyhow!("{e}\n {hint}"));
108 }
109 }
110 Self::check_misplaced_tuning_fields(yaml)?;
111 Self::check_csv_compression(yaml)?;
112 Self::check_tls_mode_downgrade(yaml)?;
113 let config: Config = serde_yaml_ng::from_str(yaml).map_err(|e| {
114 // A well-formed flow map (`prefix: {partition}`) parses as a YAML
115 // value but serde then rejects it with `invalid type: map, expected
116 // a string`. That is the same unquoted-brace footgun, surfacing one
117 // layer later than the scanner errors caught above — so try the same
118 // hint here before falling back to the generic field-typo enhancer.
119 if let Some(hint) = Self::unquoted_template_brace_hint(yaml, &e.to_string()) {
120 anyhow::anyhow!("{e}\n {hint}")
121 } else {
122 lints::enhance_parse_error(e)
123 }
124 })?;
125 config.validate()?;
126 Ok(config)
127 }
128
129 /// Detect the unquoted-`{partition}` (or `{date}`, …) template footgun and
130 /// return an actionable quoting hint, or `None` when the error is unrelated.
131 ///
132 /// A YAML value that *starts* a flow mapping — `prefix: {partition}` — is
133 /// the common copy-paste mistake: `{partition}` is the required token for
134 /// `partition_by`, but unquoted it parses as a YAML map (or, with trailing
135 /// text, trips the libyaml scanner). serde then emits a cryptic
136 /// `did not find expected ',' or '}'` / `while parsing a flow mapping` /
137 /// `invalid type: map, expected a string` with no hint that a pair of
138 /// quotes is the fix.
139 ///
140 /// Two guards keep this from firing on unrelated parse errors: the error
141 /// message must carry one of the flow-mapping symptoms, AND the raw source
142 /// must actually contain an unquoted `{…}` value. Both must hold, so a
143 /// valid config (every brace value quoted) never sees the hint, and a
144 /// genuine map-typed field error elsewhere is left alone.
145 /// A YAML document indented with a TAB trips libyaml with `found character
146 /// that cannot start any token`. Point straight at the fix (spaces, not
147 /// tabs) — but only when the cited line really begins with a tab, so an
148 /// unrelated scanner error is left with its original message.
149 fn tab_indent_hint(yaml: &str, err_msg: &str) -> Option<String> {
150 if !err_msg.contains("tab character") {
151 return None;
152 }
153 // serde_yaml_ng reports `found a tab character … at line N column C …`;
154 // the FIRST `line N` is where the offending tab is.
155 let line_no: usize = err_msg
156 .split_once("line ")
157 .and_then(|(_, rest)| rest.split([' ', ',']).next())
158 .and_then(|n| n.parse().ok())?;
159 let line = yaml.lines().nth(line_no.checked_sub(1)?)?;
160 let leading = &line[..line.len() - line.trim_start().len()];
161 leading.contains('\t').then(|| {
162 format!(
163 "line {line_no} is indented with a TAB — YAML requires spaces. Replace the tab(s) with spaces."
164 )
165 })
166 }
167
168 fn unquoted_template_brace_hint(yaml: &str, err_msg: &str) -> Option<String> {
169 const FLOW_SYMPTOMS: &[&str] = &[
170 "did not find expected ',' or '}'",
171 "while parsing a flow mapping",
172 // A bare `key: {token}` parses as a map, then serde rejects the
173 // map where it wanted a scalar — same root cause, later layer.
174 "invalid type: map, expected a string",
175 // `key: {token}/more` runs the flow map into block context.
176 "did not find expected key",
177 ];
178 if !FLOW_SYMPTOMS.iter().any(|s| err_msg.contains(s)) {
179 return None;
180 }
181 if !yaml.lines().any(line_has_unquoted_brace_value) {
182 return None;
183 }
184 Some(
185 "a YAML value containing { } (such as {partition} or {date}) must be quoted, \
186 e.g. prefix: \"exports/{partition}/\""
187 .to_string(),
188 )
189 }
190
191 /// Reject `format: csv` paired with an explicitly-requested compression
192 /// codec (Finding #10). The CSV writer has no compression encoder, so the
193 /// codec is silently dropped on write while the run manifest still records
194 /// it — a degraded, dishonest no-op. We reject loudly at config-validate
195 /// time so `rivet check` / `rivet doctor` catch it before any run.
196 ///
197 /// This is a raw-YAML scan (like [`Self::check_misplaced_tuning_fields`])
198 /// rather than a `validate_export` check on purpose: `ExportConfig.
199 /// compression` is `#[serde(default)]` and `CompressionType::default()` is
200 /// `Zstd`, so a parsed export cannot distinguish "user asked for zstd" from
201 /// "user omitted the field". Only a user who *wrote* `compression:`/
202 /// `compression_profile:` is asking for something the CSV writer cannot
203 /// honour; the bare-`format: csv` default writes uncompressed and is fine.
204 fn check_csv_compression(yaml: &str) -> crate::error::Result<()> {
205 let root: serde_yaml_ng::Value = serde_yaml_ng::from_str(yaml)?;
206 let Some(exports) = root.get("exports").and_then(|e| e.as_sequence()) else {
207 return Ok(());
208 };
209 for export in exports {
210 if export.get("format").and_then(|f| f.as_str()) != Some("csv") {
211 continue;
212 }
213 let name = export
214 .get("name")
215 .and_then(|n| n.as_str())
216 .unwrap_or("<unnamed>");
217
218 // Explicit `compression:` codec that the CSV writer cannot apply.
219 // An unrecognised label is left for serde to reject during the real
220 // parse; we only act on a codec we understand and that CSV cannot
221 // honour (everything but `none`).
222 if let Some(codec) = export.get("compression").and_then(|c| c.as_str())
223 && let Some(ct) = CompressionType::from_label(codec)
224 && !format::compression_supported(FormatType::Csv, ct)
225 {
226 anyhow::bail!(
227 "export '{}': CSV output does not support compression: {}. \
228 CSV has no compression encoder, so the codec would be silently dropped \
229 while the manifest records it.\n \
230 Hint: use `format: parquet` for compression, or set `compression: none`.",
231 name,
232 codec,
233 );
234 }
235
236 // A `compression_profile:` other than `none` resolves to a real
237 // codec too (fast→snappy, balanced/compact→zstd) — same no-op.
238 if let Some(profile) = export.get("compression_profile").and_then(|c| c.as_str())
239 && profile != CompressionProfile::None.label()
240 {
241 anyhow::bail!(
242 "export '{}': CSV output does not support compression_profile: {} \
243 (it resolves to a compression codec the CSV writer cannot apply).\n \
244 Hint: use `format: parquet` for compression, or set `compression_profile: none`.",
245 name,
246 profile,
247 );
248 }
249 }
250 Ok(())
251 }
252
253 /// V13: reject a `source.tls` block that pairs an *explicitly chosen*
254 /// enforced `mode:` with a verification-disabling danger knob
255 /// (`accept_invalid_certs` / `accept_invalid_hostnames`). `mode: verify-full`
256 /// promises chain + hostname verification, but the knob silently downgrades
257 /// it to "trust anything" — a MITM exposure that contradicts the stated
258 /// intent (see `src/source/tls.rs::build_native_tls`, whose comment claims
259 /// this is warned about at config-time but is not).
260 ///
261 /// Like [`Self::check_csv_compression`], this is a raw-YAML scan rather than
262 /// a `validate` check on purpose: `TlsMode` is `#[serde(default)]` and the
263 /// default is `VerifyFull`, so a parsed config cannot distinguish "user
264 /// wrote `mode: verify-full`" (a contradiction to flag) from "user omitted
265 /// `mode:`" (the common dev-container case `tls: { accept_invalid_certs:
266 /// true }` against a loopback self-signed cert — which must keep working).
267 /// Only an *explicit* enforced `mode:` next to a danger knob is the footgun.
268 fn check_tls_mode_downgrade(yaml: &str) -> crate::error::Result<()> {
269 let root: serde_yaml_ng::Value = serde_yaml_ng::from_str(yaml)?;
270 let Some(tls) = root.get("source").and_then(|s| s.get("tls")) else {
271 return Ok(());
272 };
273
274 // Only an explicitly written `mode:` is a deliberate, contradicted
275 // choice; an omitted mode is the dev-container default path.
276 let Some(mode) = tls.get("mode").and_then(|m| m.as_str()) else {
277 return Ok(());
278 };
279 // `disable` carries no verification promise to contradict; the danger
280 // knobs are a no-op there. Flag only the enforced modes.
281 if mode == "disable" {
282 return Ok(());
283 }
284
285 let knob = if tls
286 .get("accept_invalid_certs")
287 .and_then(|v| v.as_bool())
288 .unwrap_or(false)
289 {
290 Some("accept_invalid_certs")
291 } else if tls
292 .get("accept_invalid_hostnames")
293 .and_then(|v| v.as_bool())
294 .unwrap_or(false)
295 {
296 Some("accept_invalid_hostnames")
297 } else {
298 None
299 };
300
301 if let Some(knob) = knob {
302 anyhow::bail!(
303 "source.tls: {} disables certificate verification, silently downgrading the \
304 chosen `mode: {}` to trust-anything (MITM exposure — credentials and rows \
305 readable/forgeable on the wire).\n \
306 Hint: drop the danger knob and trust a private CA with `tls.ca_file: <pem>`; \
307 only use a danger knob for a loopback self-signed dev container, and then omit \
308 the explicit `mode:` so the contradiction is gone.",
309 knob,
310 mode,
311 );
312 }
313 Ok(())
314 }
315
316 /// Detect tuning-related fields placed directly under `source:` or an
317 /// `exports[]` entry instead of inside the `tuning:` sub-key. Without this
318 /// check serde silently ignores unknown keys and the user gets unexpected
319 /// defaults (e.g. batch_size=10 000 instead of the intended 1 000).
320 fn check_misplaced_tuning_fields(yaml: &str) -> crate::error::Result<()> {
321 const TUNING_FIELDS: &[&str] = &[
322 "batch_size",
323 "batch_size_memory_mb",
324 "throttle_ms",
325 "statement_timeout_s",
326 "max_retries",
327 "retry_backoff_ms",
328 "lock_timeout_s",
329 "memory_threshold_mb",
330 "profile",
331 ];
332
333 let root: serde_yaml_ng::Value = serde_yaml_ng::from_str(yaml)?;
334
335 if let Some(source) = root.get("source") {
336 let misplaced: Vec<&str> = TUNING_FIELDS
337 .iter()
338 .copied()
339 .filter(|&f| source.get(f).is_some())
340 .collect();
341 if !misplaced.is_empty() {
342 anyhow::bail!(
343 "source: field(s) [{}] belong under 'source.tuning:', not directly under 'source:'. \
344 Example:\n source:\n tuning:\n {}: <value>",
345 misplaced.join(", "),
346 misplaced[0],
347 );
348 }
349 }
350
351 if let Some(exports) = root.get("exports").and_then(|e| e.as_sequence()) {
352 for (i, export) in exports.iter().enumerate() {
353 let name = export
354 .get("name")
355 .and_then(|n| n.as_str())
356 .unwrap_or("<unnamed>");
357 let misplaced: Vec<&str> = TUNING_FIELDS
358 .iter()
359 .copied()
360 .filter(|&f| export.get(f).is_some())
361 .collect();
362 if !misplaced.is_empty() {
363 anyhow::bail!(
364 "export '{}' (index {}): field(s) [{}] belong under 'exports[].tuning:', \
365 not directly in the export. Example:\n exports:\n - name: {}\n tuning:\n {}: <value>",
366 name,
367 i,
368 misplaced.join(", "),
369 name,
370 misplaced[0],
371 );
372 }
373 }
374 }
375
376 Ok(())
377 }
378
379 /// Reject a config before any plan/connect step. The body is split into
380 /// three cohesive validators so each can be read — and unit-tested — on its
381 /// own: the export-list shape, the source connection block, and the
382 /// per-export rules. The end-to-end surface (`Config::from_yaml`) is
383 /// covered by `config/tests/{validation,secops}.rs`; the split additionally
384 /// lets a rule be exercised directly via `validate_export`.
385 fn validate(&self) -> crate::error::Result<()> {
386 self.validate_exports_list()?;
387 self.validate_source_connection()?;
388 for export in &self.exports {
389 self.validate_export(export)?;
390 }
391 self.validate_cdc_resource_conflicts()?;
392 self.validate_non_sql_source_modes()?;
393 Ok(())
394 }
395
396 /// Non-SQL sources (MongoDB) support only `mode: full` today. Chunked /
397 /// incremental / keyset / time-window all build SQL predicates over an
398 /// introspectable columnar schema, and CDC (change streams) is a separate,
399 /// not-yet-implemented seam. Rejecting the other modes here is precisely
400 /// what makes the SQL-only builders' `unreachable!` arms
401 /// (`sql::quote_ident`, `query::cursor_rhs`, …) provably unreachable for a
402 /// document-store source.
403 fn validate_non_sql_source_modes(&self) -> crate::error::Result<()> {
404 if self.source.source_type.is_sql() {
405 return Ok(());
406 }
407 for e in &self.exports {
408 if !matches!(e.mode, ExportMode::Full | ExportMode::Cdc) {
409 crate::config_bail!(
410 crate::error::codes::CONFIG_SOURCE_MODE_UNSUPPORTED,
411 "export '{}': source type '{:?}' supports `mode: full` (batch) and \
412 `mode: cdc` (change streams) (got `mode: {:?}`). MongoDB has no SQL, so \
413 chunked / incremental / keyset / time-window are not available; every \
414 document exports as `_id` + a `document` JSON column.",
415 e.name,
416 self.source.source_type,
417 e.mode
418 );
419 }
420 // An impossible combination must be a config error, not a silent
421 // behavior downgrade: the parallel `_id`-range path keeps NO keyset
422 // checkpoint, so `resume: true` was silently ignored — the whole
423 // collection re-read every run with no warning (bug-hunt find).
424 if e.parallel > 1 && self.source.mongo.as_ref().is_some_and(|m| m.resume) {
425 crate::config_bail!(
426 crate::error::codes::CONFIG_SOURCE_MODE_UNSUPPORTED,
427 "export '{}': `source.mongo.resume: true` cannot be combined with \
428 `parallel: {}` — the parallel `_id`-range fan-out keeps no keyset \
429 checkpoint, so resume would be silently ignored. Drop `parallel` \
430 to keep cross-run resume, or drop `resume` to keep the fan-out.",
431 e.name,
432 e.parallel
433 );
434 }
435 }
436 Ok(())
437 }
438
439 /// CDC stream resources are per-export and their **defaults collide**: two
440 /// PostgreSQL cdc exports without an explicit `slot:` both resolve to
441 /// `rivet_slot` — each export's ack advances `confirmed_flush_lsn` past
442 /// changes the *other* never read (mutual, silent data loss). Two MySQL
443 /// exports both default to `server_id: 4271` — the server kills the older
444 /// replica connection. A shared `checkpoint:` path makes exports overwrite
445 /// each other's resume position on any engine. All three are config bugs a
446 /// naive multi-table CDC config hits by default, so reject them at load, on
447 /// the RESOLVED values (defaults included). SQL Server `capture_instance`
448 /// sharing is deliberately allowed: the change-table poll is read-only and
449 /// resume state lives in the per-export checkpoint.
450 fn validate_cdc_resource_conflicts(&self) -> crate::error::Result<()> {
451 use std::collections::HashMap;
452
453 let mut slots: HashMap<String, &str> = HashMap::new();
454 let mut server_ids: HashMap<u32, &str> = HashMap::new();
455 let mut checkpoints: HashMap<&str, &str> = HashMap::new();
456
457 for e in self.exports.iter().filter(|e| e.mode == ExportMode::Cdc) {
458 let cdc = e.cdc.as_ref();
459 match self.source.source_type {
460 SourceType::Postgres => {
461 let slot = cdc
462 .and_then(|c| c.slot.clone())
463 .unwrap_or_else(|| DEFAULT_PG_SLOT.to_string());
464 if let Some(prev) = slots.insert(slot.clone(), &e.name) {
465 crate::config_bail!(
466 crate::error::codes::CONFIG_CDC_RESOURCE_CONFLICT,
467 "exports '{prev}' and '{}': same PostgreSQL slot '{slot}' — a slot \
468 has ONE consumer; each export's ack would advance it past changes \
469 the other never read (silent data loss). Set a distinct `cdc.slot:` \
470 per export (the default is '{DEFAULT_PG_SLOT}').",
471 e.name
472 );
473 }
474 }
475 SourceType::Mysql => {
476 let sid = cdc
477 .and_then(|c| c.server_id)
478 .unwrap_or(DEFAULT_MYSQL_SERVER_ID);
479 if let Some(prev) = server_ids.insert(sid, &e.name) {
480 crate::config_bail!(
481 crate::error::codes::CONFIG_CDC_RESOURCE_CONFLICT,
482 "exports '{prev}' and '{}': same MySQL server_id {sid} — the server \
483 kills the older replica connection when a new one registers with \
484 the same id. Set a distinct `cdc.server_id:` per export (the \
485 default is {DEFAULT_MYSQL_SERVER_ID}).",
486 e.name
487 );
488 }
489 }
490 SourceType::Mssql => {}
491 // MongoDB change streams watch the whole database — no per-export
492 // slot or server_id to collide. Two Mongo CDC exports sharing a
493 // `checkpoint:` path IS still a conflict, caught by the shared
494 // checkpoint check below.
495 SourceType::Mongo => {}
496 }
497 if let Some(ckpt) = cdc.and_then(|c| c.checkpoint.as_deref())
498 && let Some(prev) = checkpoints.insert(ckpt, &e.name)
499 {
500 crate::config_bail!(
501 crate::error::codes::CONFIG_CDC_RESOURCE_CONFLICT,
502 "exports '{prev}' and '{}': same checkpoint path '{ckpt}' — each export \
503 must own its resume position or they overwrite each other's. Set a \
504 distinct `cdc.checkpoint:` per export.",
505 e.name
506 );
507 }
508 }
509 Ok(())
510 }
511
512 /// Whole-config shape: at least one export, names unique.
513 fn validate_exports_list(&self) -> crate::error::Result<()> {
514 // An empty `exports:` list is almost always a typo (wrong config file,
515 // dropped anchor, merged doc with the anchor section missing). Running
516 // with zero exports is a silent no-op that looks like success in CI;
517 // reject fast instead. See QA backlog Task 5.1.
518 if self.exports.is_empty() {
519 crate::config_bail!(
520 crate::error::codes::CONFIG_NO_EXPORTS,
521 "exports: at least one export must be defined (got empty list)"
522 );
523 }
524
525 // Duplicate export names break state tracking: `export_state`,
526 // `file_log`, and `chunk_run` are all keyed by `export_name`, so
527 // two configs with the same name silently share cursor/file-log rows.
528 // QA backlog Task 5.1.
529 let mut seen: std::collections::HashSet<&str> =
530 std::collections::HashSet::with_capacity(self.exports.len());
531 for e in &self.exports {
532 if !seen.insert(e.name.as_str()) {
533 crate::config_bail!(
534 crate::error::codes::CONFIG_DUPLICATE_EXPORT,
535 "exports: duplicate export name '{}' (each export must have a unique name; state is keyed by name)",
536 e.name
537 );
538 }
539 }
540 Ok(())
541 }
542
543 /// Source connection block: exactly one connection method, well-formed,
544 /// and the source-level tuning that is shared by every export.
545 fn validate_source_connection(&self) -> crate::error::Result<()> {
546 if let Some(t) = &self.source.tuning
547 && t.batch_size.is_some()
548 && t.batch_size_memory_mb.is_some()
549 {
550 anyhow::bail!(
551 "tuning: batch_size and batch_size_memory_mb are mutually exclusive. \
552 Prefer batch_size_memory_mb (rivet sizes the batch to a memory budget, \
553 adapting to row width); set batch_size only to pin an exact row count."
554 );
555 }
556
557 if !self.source.has_url_fields() && !self.source.has_structured_fields() {
558 // First-run footgun: a config that forgot the source block
559 // entirely. Show the recommended path (`url_env`) up-front;
560 // operators who actually want structured fields know to look
561 // for them.
562 anyhow::bail!(
563 "source: no connection method configured. Add one of:\n url_env: DATABASE_URL (URL from env var — recommended)\n url: 'postgresql://user:pass@host:5432/db' (inline — not recommended for committed configs)\n url_file: /etc/rivet/source.url (URL from file — rotation-friendly)\n host/user/database/... (structured fields under `source:`)"
564 );
565 }
566
567 if self.source.has_url_fields() {
568 let url_count = [
569 &self.source.url,
570 &self.source.url_env,
571 &self.source.url_file,
572 ]
573 .iter()
574 .filter(|u| u.is_some())
575 .count();
576 if url_count > 1 {
577 anyhow::bail!(
578 "source: specify exactly one of 'url', 'url_env', or 'url_file' (got {} set).\n Hint: pick one — `url_env` is recommended so credentials never enter the YAML.",
579 url_count
580 );
581 }
582 }
583
584 if self.source.has_url_fields() && self.source.has_structured_fields() {
585 anyhow::bail!(
586 "source: pick either URL-based config (url/url_env/url_file) OR structured fields (host/user/database/port/password_env), not both.\n Hint: remove whichever block you don't want; mixing the two is ambiguous."
587 );
588 }
589
590 if self.source.has_structured_fields() {
591 if self.source.host.is_none() {
592 anyhow::bail!(
593 "source: structured config is missing 'host'.\n Hint: add `host: localhost` (or your DB host) under `source:` in rivet.yaml.\n Or switch to URL-based config: `url_env: DATABASE_URL`."
594 );
595 }
596 if self.source.user.is_none() {
597 anyhow::bail!(
598 "source: structured config is missing 'user'.\n Hint: add `user: <username>` under `source:` in rivet.yaml."
599 );
600 }
601 if self.source.database.is_none() {
602 anyhow::bail!(
603 "source: structured config is missing 'database'.\n Hint: add `database: <dbname>` under `source:` in rivet.yaml."
604 );
605 }
606 if self.source.password.is_some() && self.source.password_env.is_some() {
607 anyhow::bail!(
608 "source: specify 'password' OR 'password_env', not both.\n Hint: prefer `password_env: DB_PASSWORD` so credentials never enter the YAML."
609 );
610 }
611 }
612 Ok(())
613 }
614
615 /// Per-export rules: effective tuning, query source, `query_file` SecOps,
616 /// destination auth, compression, and the mode/chunk matrix. Takes `&self`
617 /// because effective tuning merges the source-level block.
618 fn validate_export(&self, export: &ExportConfig) -> crate::error::Result<()> {
619 // V5: `name` is keyed into output paths, file logs, and on-disk state,
620 // yet is otherwise free-form. A traversal (`../../etc/x`), absolute or
621 // slash-bearing (`/abs/x`, `sub/dir`), leading-dot, or NUL-bearing name
622 // escapes the intended output tree and corrupts name-keyed state.
623 // Mirror the `query_file` `..`/absolute guard: reject at config-load,
624 // accepting only a filename-safe charset.
625 if !is_filename_safe_name(&export.name) {
626 anyhow::bail!(
627 "export name '{}' is not filename-safe: it must not be absolute, contain \
628 '/', '\\', '..', a NUL, or start with '.' (the name is used in output paths \
629 and state keys). Use a plain identifier like `orders` or `daily_events`.",
630 export.name.escape_default(),
631 );
632 }
633
634 let merged =
635 crate::tuning::merge_tuning_config(self.source.tuning.as_ref(), export.tuning.as_ref());
636 if let Some(t) = merged
637 && t.batch_size.is_some()
638 && t.batch_size_memory_mb.is_some()
639 {
640 anyhow::bail!(
641 "export '{}': effective tuning has both batch_size and batch_size_memory_mb (mutually exclusive)",
642 export.name
643 );
644 }
645 if let Some(et) = &export.tuning
646 && et.batch_size.is_some()
647 && et.batch_size_memory_mb.is_some()
648 {
649 anyhow::bail!(
650 "export '{}': tuning.batch_size and tuning.batch_size_memory_mb are mutually exclusive",
651 export.name
652 );
653 }
654
655 // Before the generic exactly-one counting: the `table:`/`tables:` pair
656 // gets its specific message (the generic one doesn't mention `tables`).
657 if export.table.is_some() && export.tables.is_some() {
658 anyhow::bail!(
659 "export '{}': `table:` and `tables:` are mutually exclusive — use \
660 `tables: [a, b]` for a multi-table stream",
661 export.name
662 );
663 }
664
665 let set_count = [
666 export.query.is_some(),
667 export.query_file.is_some(),
668 export.table.is_some(),
669 // A multi-table CDC stream (`tables:`) is that export's source
670 // specification — the CDC arm below owns its shape rules.
671 export.tables.is_some(),
672 ]
673 .iter()
674 .filter(|b| **b)
675 .count();
676 if set_count == 0 {
677 anyhow::bail!(
678 "export '{}': specify exactly one of 'query', 'query_file', 'table', or 'tables'. \
679 Use table: <name> for a whole table (enables PK auto-chunking); \
680 query: \"SELECT …\" for an inline one-liner; \
681 query_file: <path> for SQL you keep in version control.",
682 export.name
683 );
684 }
685 if set_count > 1 {
686 anyhow::bail!(
687 "export '{}': specify exactly one of 'query', 'query_file', 'table', or 'tables' (got {} set)",
688 export.name,
689 set_count
690 );
691 }
692 // SecOps: syntactic `query_file` checks must run at config-validate
693 // time so `rivet check` / `rivet doctor` catch them before any
694 // plan step. The same checks repeat (with a canonicalize-based
695 // symlink probe) in `ExportConfig::resolve_query` because the
696 // file may have been swapped between validation and read.
697 if let Some(file) = &export.query_file {
698 let p = std::path::Path::new(file);
699 if p.is_absolute() {
700 anyhow::bail!(
701 "export '{}': query_file must be a relative path: '{}'",
702 export.name,
703 file
704 );
705 }
706 if p.components().any(|c| c == std::path::Component::ParentDir) {
707 anyhow::bail!(
708 "export '{}': query_file path must not contain '..': '{}'",
709 export.name,
710 file
711 );
712 }
713 }
714 // V2/V12: a custom cloud `endpoint` is handed straight to the opendal
715 // S3/GCS/Azure builder with no validation, so a committed config can
716 // silently redirect every upload to an attacker host (exfiltration) or
717 // send credentials + rows over cleartext `http://`. The legitimate use
718 // is a local emulator (Minio / Azurite / fake-gcs on `127.0.0.1`), so
719 // accept a loopback host (any scheme), and otherwise accept a remote
720 // endpoint only when the operator has explicitly opted into anonymous
721 // (emulator) mode. Reject every other custom endpoint at config-load.
722 if matches!(
723 export.destination.destination_type,
724 DestinationType::S3 | DestinationType::Gcs | DestinationType::Azure
725 ) && let Some(endpoint) = &export.destination.endpoint
726 {
727 // Loopback emulator (Minio/Azurite/fake-gcs) is the legitimate
728 // local-dev path — accept any scheme. A non-loopback (or
729 // unparseable) custom endpoint is only accepted when the operator
730 // has explicitly opted into anonymous (emulator) mode, where no
731 // credentials are sent. Everything else is rejected.
732 let loopback = endpoint_host(endpoint).is_some_and(|host| is_loopback_host(&host));
733 if !loopback && !export.destination.allow_anonymous {
734 anyhow::bail!(
735 "export '{}': destination.endpoint '{}' points at a non-loopback host. \
736 A custom endpoint redirects every upload there — committing one is a \
737 data-exfiltration / cleartext-credential risk.\n \
738 Hint: drop `endpoint:` to use the provider default, point it at a \
739 loopback emulator (e.g. http://127.0.0.1:9000 with allow_anonymous: true \
740 for Minio/Azurite), or set `allow_anonymous: true` for an anonymous \
741 emulator.",
742 export.name,
743 endpoint,
744 );
745 }
746 }
747
748 // V15: a `type: local` destination `path` (or `prefix`) is written
749 // verbatim to the filesystem. A `..` component lets a committed config
750 // climb out of the intended output tree (`../../../../tmp/x`) — mirror
751 // the `query_file` traversal guard and reject it at config-load.
752 //
753 // Absolute paths are deliberately *not* rejected: `path: /output` is a
754 // legitimate Docker volume-mount pattern (see `examples/rivet.yaml`) and
755 // an explicit operator choice, not a hidden escape. The `..` climb is
756 // the unambiguous traversal footgun.
757 if export.destination.destination_type == DestinationType::Local {
758 for (field, value) in [
759 ("path", export.destination.path.as_deref()),
760 ("prefix", export.destination.prefix.as_deref()),
761 ] {
762 let Some(value) = value else { continue };
763 if std::path::Path::new(value)
764 .components()
765 .any(|c| c == std::path::Component::ParentDir)
766 {
767 anyhow::bail!(
768 "export '{}': local destination {} must not contain a '..' component: \
769 '{}' (a parent-dir climb writes outside the output tree).",
770 export.name,
771 field,
772 value
773 );
774 }
775 }
776 }
777
778 if export.destination.destination_type == DestinationType::S3 {
779 let ak = export.destination.access_key_env.is_some();
780 let sk = export.destination.secret_key_env.is_some();
781 if ak != sk {
782 anyhow::bail!(
783 "export '{}': S3 requires both access_key_env and secret_key_env, or neither (use default AWS credential chain)",
784 export.name
785 );
786 }
787 }
788
789 if export.destination.destination_type == DestinationType::Gcs
790 && export.destination.allow_anonymous
791 && export.destination.credentials_file.is_some()
792 {
793 anyhow::bail!(
794 "export '{}': GCS allow_anonymous cannot be used together with credentials_file",
795 export.name
796 );
797 }
798
799 if export.destination.destination_type == DestinationType::Azure {
800 let has_name = export.destination.account_name.is_some();
801 let has_key = export.destination.account_key_env.is_some();
802 let has_sas = export.destination.sas_token_env.is_some();
803 if export.destination.allow_anonymous {
804 if has_name || has_key || has_sas {
805 anyhow::bail!(
806 "export '{}': Azure allow_anonymous cannot be combined with account_name/account_key_env/sas_token_env",
807 export.name
808 );
809 }
810 } else if has_key && has_sas {
811 anyhow::bail!(
812 "export '{}': Azure account_key_env and sas_token_env are mutually exclusive — pick one auth mode",
813 export.name
814 );
815 } else if !has_name {
816 anyhow::bail!(
817 "export '{}': Azure requires account_name (plus account_key_env or sas_token_env), or allow_anonymous: true for Azurite",
818 export.name
819 );
820 } else if !has_key && !has_sas {
821 anyhow::bail!(
822 "export '{}': Azure requires account_key_env or sas_token_env (or allow_anonymous: true for Azurite)",
823 export.name
824 );
825 }
826 }
827
828 if let Some(cred_path) = &export.destination.credentials_file
829 && !std::path::Path::new(cred_path).exists()
830 {
831 anyhow::bail!(
832 "export '{}': credentials_file '{}' does not exist",
833 export.name,
834 cred_path
835 );
836 }
837
838 if let Some(ref size_str) = export.max_file_size {
839 parse_file_size(size_str).map_err(|_| {
840 anyhow::anyhow!(
841 "export '{}': invalid max_file_size '{}'",
842 export.name,
843 size_str
844 )
845 })?;
846 }
847
848 if let Some(level) = export.compression_level {
849 match export.compression {
850 CompressionType::Zstd => {
851 if !(1..=22).contains(&level) {
852 anyhow::bail!(
853 "export '{}': zstd compression_level must be 1..22, got {}",
854 export.name,
855 level
856 );
857 }
858 }
859 CompressionType::Gzip => {
860 if level > 10 {
861 anyhow::bail!(
862 "export '{}': gzip compression_level must be 0..10, got {}",
863 export.name,
864 level
865 );
866 }
867 }
868 _ => {
869 anyhow::bail!(
870 "export '{}': compression_level is only supported for zstd and gzip",
871 export.name
872 );
873 }
874 }
875 }
876
877 match export.mode {
878 ExportMode::Incremental => {
879 if export.cursor_column.is_none() {
880 anyhow::bail!(
881 "export '{}': incremental mode requires cursor_column",
882 export.name
883 );
884 }
885 match export.incremental_cursor_mode {
886 IncrementalCursorMode::Coalesce => {
887 if export.cursor_fallback_column.is_none() {
888 anyhow::bail!(
889 "export '{}': incremental_cursor_mode: coalesce requires cursor_fallback_column",
890 export.name
891 );
892 }
893 }
894 IncrementalCursorMode::SingleColumn => {
895 if export.cursor_fallback_column.is_some() {
896 anyhow::bail!(
897 "export '{}': cursor_fallback_column is only valid with incremental_cursor_mode: coalesce",
898 export.name
899 );
900 }
901 }
902 }
903 }
904 ExportMode::Chunked => {
905 // `chunk_column` is mandatory unless the user used the `table:`
906 // shortcut on a Postgres source — in that case it is auto-resolved
907 // from the table's single-integer PK at plan-build time (see
908 // `crate::plan::build::resolve_chunk_column`).
909 if export.chunk_column.is_none() && export.table.is_none() {
910 anyhow::bail!(
911 "export '{}': chunked mode needs a chunking strategy. Pick one:\n \
912 chunk_column: <int col> range chunks on an integer column (most common)\n \
913 chunk_by_key: <unique col> keyset pagination when there's no integer PK\n \
914 chunk_count: <N> split the range into N equal chunks\n \
915 chunk_by_days: <D> time-bucketed chunks (needs a date/timestamp column)\n \
916 Or use the `table:` shortcut on a single table — rivet auto-resolves the column from the primary key.",
917 export.name
918 );
919 }
920 // chunk_size == 0 would divide the range into zero-width
921 // slices and (before the saturating fix in generate_chunks)
922 // either infinite-loop or produce no progress. QA backlog
923 // Task 5.1.
924 if export.chunk_size == 0 {
925 anyhow::bail!(
926 "export '{}': chunked mode requires chunk_size >= 1 (got 0)",
927 export.name
928 );
929 }
930 // parallel == 0 means "spawn zero workers". Claiming tasks
931 // with no workers stalls the pipeline. QA backlog Task 5.1.
932 if export.parallel == 0 {
933 anyhow::bail!(
934 "export '{}': chunked mode requires parallel >= 1 (got 0)",
935 export.name
936 );
937 }
938 if let Some(0) = export.chunk_count {
939 crate::config_bail!(
940 crate::error::codes::CONFIG_CHUNK_COUNT_INVALID,
941 "export '{}': chunk_count must be >= 1",
942 export.name
943 );
944 }
945 if export.chunk_count.is_some() && export.chunk_dense {
946 anyhow::bail!(
947 "export '{}': chunk_count and chunk_dense are mutually exclusive. \
948 Use chunk_count for equal-sized chunks over a sparse key; \
949 use chunk_dense only when the key has no gaps.",
950 export.name
951 );
952 }
953 if export.chunk_count.is_some() && export.chunk_by_days.is_some() {
954 anyhow::bail!(
955 "export '{}': chunk_count and chunk_by_days are mutually exclusive. \
956 Use chunk_count: N to split an integer range into N chunks; \
957 use chunk_by_days: D to bucket a date/timestamp column by D-day windows.",
958 export.name
959 );
960 }
961 }
962 ExportMode::TimeWindow => {
963 if export.time_column.is_none() {
964 anyhow::bail!(
965 "export '{}': time_window mode requires time_column",
966 export.name
967 );
968 }
969 if export.days_window.is_none() {
970 anyhow::bail!(
971 "export '{}': time_window mode requires days_window",
972 export.name
973 );
974 }
975 }
976 ExportMode::Full => {}
977 ExportMode::Cdc => {
978 match (&export.table, &export.tables) {
979 (None, None) => anyhow::bail!(
980 "export '{}': cdc mode requires `table:` (or `tables:` for a \
981 multi-table stream)",
982 export.name
983 ),
984 (Some(_), Some(_)) => anyhow::bail!(
985 "export '{}': `table:` and `tables:` are mutually exclusive — \
986 use `tables: [a, b]` for a multi-table stream",
987 export.name
988 ),
989 (None, Some(ts)) => {
990 if ts.is_empty() {
991 anyhow::bail!(
992 "export '{}': `tables:` must list at least one table",
993 export.name
994 );
995 }
996 let mut seen = std::collections::HashSet::new();
997 for t in ts {
998 if !seen.insert(t.as_str()) {
999 anyhow::bail!(
1000 "export '{}': duplicate table '{}' in `tables:`",
1001 export.name,
1002 t
1003 );
1004 }
1005 }
1006 if self.source.source_type == SourceType::Mssql {
1007 anyhow::bail!(
1008 "export '{}': `tables:` is not yet supported for SQL Server — \
1009 its capture instances are per-table; use one cdc export per \
1010 table (capture_instance each)",
1011 export.name
1012 );
1013 }
1014 }
1015 (Some(_), None) => {}
1016 }
1017 if export.query.is_some() || export.query_file.is_some() {
1018 anyhow::bail!(
1019 "export '{}': cdc mode reads the transaction log, not a query — \
1020 remove query/query_file and use `table:`",
1021 export.name
1022 );
1023 }
1024 }
1025 }
1026
1027 if export.tables.is_some() && export.mode != ExportMode::Cdc {
1028 anyhow::bail!(
1029 "export '{}': `tables:` is only valid with `mode: cdc` (batch exports \
1030 are one query/table per export)",
1031 export.name
1032 );
1033 }
1034
1035 if export.chunk_dense && export.mode != ExportMode::Chunked {
1036 anyhow::bail!(
1037 "export '{}': chunk_dense is only valid with mode: chunked",
1038 export.name
1039 );
1040 }
1041
1042 if export.cdc.is_some() && export.mode != ExportMode::Cdc {
1043 anyhow::bail!(
1044 "export '{}': a `cdc:` block is only valid with `mode: cdc`",
1045 export.name
1046 );
1047 }
1048
1049 // Table-qualified `columns:` override keys ("table.column"): the named
1050 // table must be one this export captures — a typo must fail at load,
1051 // never silently miss its target. Bare keys stay export-wide.
1052 {
1053 for key in export.columns.keys() {
1054 let Some((tbl, _col)) = key.split_once('.') else {
1055 continue;
1056 };
1057 if export.query.is_some() || export.query_file.is_some() {
1058 anyhow::bail!(
1059 "export '{}': qualified column override '{key}' needs a table-shaped \
1060 export (`table:`/`tables:`), not a query",
1061 export.name
1062 );
1063 }
1064 let bare = |t: &str| t.rsplit('.').next().unwrap_or(t).to_string();
1065 let captures = export
1066 .table
1067 .as_deref()
1068 .map(bare)
1069 .into_iter()
1070 .chain(export.tables.iter().flatten().map(|t| bare(t)))
1071 .any(|t| t == tbl);
1072 if !captures {
1073 anyhow::bail!(
1074 "export '{}': column override '{key}' names table '{tbl}', which this \
1075 export does not capture — fix the table name or use a bare column key \
1076 to apply it to every captured table",
1077 export.name
1078 );
1079 }
1080 }
1081 }
1082
1083 // `initial: snapshot` writes each table's snapshot under the reserved
1084 // sub-prefix `snapshot/` — a table actually NAMED "snapshot" would share
1085 // a prefix with another table's marker. Refuse the collision at load.
1086 if let Some(cdc) = &export.cdc
1087 && cdc.initial == Some(CdcInitialMode::Snapshot)
1088 {
1089 let clashes = |t: &str| t.rsplit('.').next().unwrap_or(t) == "snapshot";
1090 if export.table.as_deref().is_some_and(clashes)
1091 || export.tables.iter().flatten().any(|t| clashes(t))
1092 {
1093 anyhow::bail!(
1094 "export '{}': a table named 'snapshot' collides with the reserved \
1095 `snapshot/` sub-prefix that `cdc.initial: snapshot` writes — rename \
1096 the table or use a separate export without `initial:`",
1097 export.name
1098 );
1099 }
1100 }
1101
1102 // `initial: snapshot` needs a durable anchor BEFORE the snapshot reads.
1103 // PostgreSQL pins server-side (the slot); MySQL / SQL Server have no
1104 // server-side anchor, so the checkpoint file is mandatory there.
1105 if let Some(cdc) = &export.cdc
1106 && cdc.initial == Some(CdcInitialMode::Snapshot)
1107 && self.source.source_type != SourceType::Postgres
1108 && cdc.checkpoint.is_none()
1109 {
1110 anyhow::bail!(
1111 "export '{}': `cdc.initial: snapshot` on {:?} requires `cdc.checkpoint:` — \
1112 the checkpoint file is the anchor that makes snapshot-then-stream gap-free",
1113 export.name,
1114 self.source.source_type
1115 );
1116 }
1117
1118 // MongoDB change streams have NO server-side resume anchor (unlike a
1119 // PostgreSQL slot): the checkpoint file IS the anchor. Without it, every
1120 // run re-anchors at "now" and silently loses every change since the last
1121 // run — so `mode: cdc` on mongo requires `cdc.checkpoint:` ALWAYS, not
1122 // only under `initial: snapshot` (bug-hunt find).
1123 if export.mode == ExportMode::Cdc
1124 && self.source.source_type == SourceType::Mongo
1125 && export
1126 .cdc
1127 .as_ref()
1128 .and_then(|c| c.checkpoint.as_ref())
1129 .is_none()
1130 {
1131 anyhow::bail!(
1132 "export '{}': MongoDB `mode: cdc` requires `cdc.checkpoint:` — a change \
1133 stream has no server-side resume anchor, so without the checkpoint file \
1134 each run re-anchors at the current time and silently loses every change \
1135 between runs.",
1136 export.name
1137 );
1138 }
1139
1140 // A collection whose name contains a dot does not route through the
1141 // change-stream capture — its events are silently dropped. Refuse loudly
1142 // rather than report 0-row success forever (bug-hunt find). Batch handles
1143 // dotted names fine, so this is CDC-only.
1144 if export.mode == ExportMode::Cdc && self.source.source_type == SourceType::Mongo {
1145 let dotted = |t: &str| t.contains('.');
1146 if export.table.as_deref().is_some_and(dotted)
1147 || export.tables.iter().flatten().any(|t| dotted(t))
1148 {
1149 anyhow::bail!(
1150 "export '{}': MongoDB `mode: cdc` does not support a collection name \
1151 containing a dot — the change-stream router cannot address it and its \
1152 events would be silently dropped. Rename the collection, or capture it \
1153 with `mode: full` (batch handles dotted names).",
1154 export.name
1155 );
1156 }
1157 }
1158
1159 if let Some(days) = export.chunk_by_days {
1160 if export.mode != ExportMode::Chunked {
1161 anyhow::bail!(
1162 "export '{}': chunk_by_days requires mode: chunked",
1163 export.name
1164 );
1165 }
1166 if export.chunk_dense {
1167 anyhow::bail!(
1168 "export '{}': chunk_by_days cannot be combined with chunk_dense",
1169 export.name
1170 );
1171 }
1172 if days == 0 {
1173 crate::config_bail!(
1174 crate::error::codes::CONFIG_CHUNK_BY_DAYS_INVALID,
1175 "export '{}': chunk_by_days must be at least 1",
1176 export.name
1177 );
1178 }
1179 }
1180 Ok(())
1181 }
1182}
1183
1184/// True when a single YAML line carries a mapping value (text after `key:`)
1185/// that contains a `{` outside of any quotes — the unquoted-template-brace
1186/// shape (`prefix: {partition}`, `path: {date}/out`).
1187///
1188/// Quote-aware so a properly quoted value (`prefix: "exports/{partition}/"`)
1189/// does *not* match, and `$`-prefixed braces (`${VAR}` env placeholders) are
1190/// ignored — they are resolved before the parse and are not the footgun.
1191fn line_has_unquoted_brace_value(line: &str) -> bool {
1192 // Whole-line comments never carry a value — skip before splitting.
1193 if line.trim_start().starts_with('#') {
1194 return false;
1195 }
1196 // Split key from value at the first `": "` / `":\t"` / trailing `:`.
1197 // A YAML plain-key separator is a colon followed by whitespace or EOL.
1198 let bytes = line.as_bytes();
1199 let mut sep = None;
1200 let mut i = 0;
1201 while i < bytes.len() {
1202 if bytes[i] == b':' && (i + 1 == bytes.len() || bytes[i + 1].is_ascii_whitespace()) {
1203 sep = Some(i + 1);
1204 break;
1205 }
1206 i += 1;
1207 }
1208 let Some(value_start) = sep else {
1209 return false;
1210 };
1211 let value = line[value_start..].trim_start();
1212 // A trailing `#` after the value starts an inline comment; an empty or
1213 // comment-only value carries no brace to flag.
1214 if value.is_empty() || value.starts_with('#') {
1215 return false;
1216 }
1217
1218 let mut in_single = false;
1219 let mut in_double = false;
1220 let vbytes = value.as_bytes();
1221 for (j, &c) in vbytes.iter().enumerate() {
1222 match c {
1223 b'\'' if !in_double => in_single = !in_single,
1224 b'"' if !in_single => in_double = !in_double,
1225 b'{' if !in_single && !in_double => {
1226 // Ignore `${...}` env placeholders (resolved pre-parse).
1227 if j > 0 && vbytes[j - 1] == b'$' {
1228 continue;
1229 }
1230 return true;
1231 }
1232 _ => {}
1233 }
1234 }
1235 false
1236}
1237
1238/// Extract the lower-cased host from a `scheme://host[:port][/path]` endpoint,
1239/// or `None` when it does not look like a URL.
1240///
1241/// SecOps helper for the cloud-`endpoint` exfiltration guard (V2/V12): the host
1242/// decides whether a custom endpoint is a local emulator (loopback) or a remote
1243/// redirect target. We reject every non-loopback custom endpoint regardless of
1244/// scheme (covering both the exfil and the cleartext-`http` gaps), so only the
1245/// host is needed. We hand-parse rather than pull in a URL crate — the inputs
1246/// are operator-typed endpoints, not arbitrary URIs. A bracketed IPv6 literal
1247/// authority (`http://[::1]:9000`) keeps its address so it compares against the
1248/// loopback list.
1249fn endpoint_host(endpoint: &str) -> Option<String> {
1250 let (scheme, rest) = endpoint.split_once("://")?;
1251 if scheme.is_empty() {
1252 return None;
1253 }
1254 // Authority ends at the first `/` (path), `?` (query), or `#` (fragment);
1255 // any `user[:pass]@` userinfo head is dropped (host is after the last `@`).
1256 let authority = rest
1257 .split(['/', '?', '#'])
1258 .next()
1259 .unwrap_or("")
1260 .rsplit('@')
1261 .next()
1262 .unwrap_or("");
1263 let host = if let Some(stripped) = authority.strip_prefix('[') {
1264 // Bracketed IPv6 literal: take up to the closing `]`.
1265 stripped.split(']').next().unwrap_or("")
1266 } else {
1267 // host[:port] — strip the port suffix.
1268 authority.split(':').next().unwrap_or("")
1269 };
1270 if host.is_empty() {
1271 return None;
1272 }
1273 Some(host.to_ascii_lowercase())
1274}
1275
1276/// True when `host` names the local machine — the legitimate cloud-emulator
1277/// target (Minio / Azurite / fake-gcs on `127.0.0.1`). Anything else is a
1278/// remote host and a potential exfiltration redirect.
1279fn is_loopback_host(host: &str) -> bool {
1280 // `localhost` is the only non-IP host that counts as loopback. Everything
1281 // else must PARSE as an IP literal in the loopback range — a lexical
1282 // `starts_with("127.")` would accept attacker-controlled DNS like
1283 // `127.attacker.com` or `127.0.0.1.evil.com` (both resolve off-box), turning
1284 // the credential-exfil gate into a bypass (V2/V12). Parse strictly: only a
1285 // real `127.0.0.0/8` / `::1` address is loopback; a hostname is not.
1286 if host == "localhost" {
1287 return true;
1288 }
1289 // Tolerate a bracketed IPv6 literal (`[::1]`) in case a caller forwards one.
1290 let h = host
1291 .strip_prefix('[')
1292 .and_then(|s| s.strip_suffix(']'))
1293 .unwrap_or(host);
1294 h.parse::<std::net::IpAddr>()
1295 .is_ok_and(|ip| ip.is_loopback())
1296}
1297
1298/// True when `name` is filename-safe: rejects path-traversal (`..`), absolute
1299/// or slash-bearing names (`/`, `\`), a leading `.` (hidden / current-dir), and
1300/// embedded NULs. `ExportConfig.name` is keyed into output paths and on-disk
1301/// state, so a `../../etc/x` or absolute name escapes the output tree (V5).
1302fn is_filename_safe_name(name: &str) -> bool {
1303 !name.is_empty()
1304 && !name.starts_with('.')
1305 && !name.contains('/')
1306 && !name.contains('\\')
1307 && !name.contains("..")
1308 && !name.contains('\0')
1309}
1310
1311#[cfg(test)]
1312mod tests;
1313
1314#[cfg(test)]
1315mod audit_csv_compression {
1316 //! Finding #10: `format: csv` + a compression codec is a silent no-op
1317 //! (the file stays uncompressed but the manifest records the codec). The
1318 //! combo must be rejected at config-validate time. These tests encode the
1319 //! new rule, so reverting the fix turns them red.
1320 use super::*;
1321
1322 fn yaml(format: &str, compression_line: &str) -> String {
1323 format!(
1324 "source:\n type: postgres\n url: \"postgresql://localhost/test\"\n\
1325 exports:\n - name: t\n query: \"SELECT 1\"\n format: {format}\n\
1326 {compression_line} destination:\n type: local\n path: ./out\n"
1327 )
1328 }
1329
1330 #[test]
1331 fn audit_csv_compression_is_rejected() {
1332 // csv + gzip → rejected, with an actionable message.
1333 let err = Config::from_yaml(&yaml("csv", " compression: gzip\n")).unwrap_err();
1334 let msg = format!("{err:#}");
1335 assert!(
1336 msg.contains("CSV output does not support compression") && msg.contains("gzip"),
1337 "csv+gzip must be rejected with an actionable message; got: {msg}"
1338 );
1339 assert!(
1340 msg.contains("parquet") && msg.contains("none"),
1341 "message must point to the real options (parquet / none); got: {msg}"
1342 );
1343
1344 // Guard the boundaries: parquet+gzip and csv+none still validate.
1345 Config::from_yaml(&yaml("parquet", " compression: gzip\n"))
1346 .expect("parquet+gzip must validate");
1347 Config::from_yaml(&yaml("csv", " compression: none\n")).expect("csv+none must validate");
1348 }
1349
1350 #[test]
1351 fn audit_csv_every_real_codec_is_rejected() {
1352 // Each non-None codec is a silent no-op for CSV — none may slip through.
1353 for codec in ["zstd", "snappy", "gzip", "lz4"] {
1354 let err = Config::from_yaml(&yaml("csv", &format!(" compression: {codec}\n")))
1355 .unwrap_err();
1356 let msg = format!("{err:#}");
1357 assert!(
1358 msg.contains("CSV output does not support compression") && msg.contains(codec),
1359 "csv+{codec} must be rejected; got: {msg}"
1360 );
1361 }
1362 }
1363
1364 #[test]
1365 fn audit_csv_compression_profile_is_rejected() {
1366 // A `compression_profile:` other than `none` resolves to a real codec,
1367 // so it is the same silent no-op for CSV.
1368 for profile in ["fast", "balanced", "compact"] {
1369 let err = Config::from_yaml(&yaml(
1370 "csv",
1371 &format!(" compression_profile: {profile}\n"),
1372 ))
1373 .unwrap_err();
1374 let msg = format!("{err:#}");
1375 assert!(
1376 msg.contains("CSV output does not support compression_profile")
1377 && msg.contains(profile),
1378 "csv+profile {profile} must be rejected; got: {msg}"
1379 );
1380 }
1381 // profile: none is a no-op request and is fine.
1382 Config::from_yaml(&yaml("csv", " compression_profile: none\n"))
1383 .expect("csv + compression_profile: none must validate");
1384 }
1385
1386 #[test]
1387 fn audit_csv_default_compression_still_validates() {
1388 // Regression guard: a bare `format: csv` (no explicit codec) must keep
1389 // validating. `CompressionType::default()` is `Zstd`, but the user did
1390 // not *ask* for it — only an explicit codec is a no-op request. This
1391 // pins that the fix scans for explicit intent, not the struct default
1392 // (which would break ~60 existing csv configs).
1393 Config::from_yaml(&yaml("csv", "")).expect("bare format: csv must validate");
1394 }
1395
1396 #[test]
1397 fn audit_compression_supported_predicate() {
1398 // `compression_supported` is re-exported via `pub use format::*`.
1399 // Parquet supports every codec; CSV supports only None.
1400 for ct in [
1401 CompressionType::Zstd,
1402 CompressionType::Snappy,
1403 CompressionType::Gzip,
1404 CompressionType::Lz4,
1405 CompressionType::None,
1406 ] {
1407 assert!(compression_supported(FormatType::Parquet, ct));
1408 }
1409 assert!(compression_supported(
1410 FormatType::Csv,
1411 CompressionType::None
1412 ));
1413 for ct in [
1414 CompressionType::Zstd,
1415 CompressionType::Snappy,
1416 CompressionType::Gzip,
1417 CompressionType::Lz4,
1418 ] {
1419 assert!(
1420 !compression_supported(FormatType::Csv, ct),
1421 "CSV must not claim to support {}",
1422 ct.label()
1423 );
1424 }
1425 }
1426}
1427
1428#[cfg(test)]
1429mod reserved_load_extension {
1430 //! A downstream first-party loader carries its warehouse target in a
1431 //! top-level `load:` block so ONE config drives export + load. OSS must
1432 //! accept and ignore it — otherwise `check`/`run`/`apply` would reject the
1433 //! single-file config. Reverting the reserved field turns this red.
1434 use super::*;
1435
1436 #[test]
1437 fn reserved_top_level_load_block_is_accepted_and_ignored() {
1438 let yaml = "source:\n type: postgres\n url: \"postgresql://localhost/test\"\n\
1439 exports:\n - name: t\n query: \"SELECT 1\"\n format: parquet\n destination:\n type: gcs\n bucket: b\n prefix: \"t/\"\n\
1440 load:\n project: p\n dataset: d\n cleanup_source: true\n";
1441 let cfg = Config::from_yaml(yaml).expect("config with a reserved `load:` block must parse");
1442 assert!(cfg.load.is_some(), "the load block is captured opaquely");
1443 }
1444}
1445
1446#[cfg(test)]
1447mod audit_unquoted_template_brace {
1448 //! yaml-hint: an unquoted `{partition}` (or `{date}`) in a path/prefix
1449 //! value trips serde_yaml_ng's flow-mapping parser with a cryptic message
1450 //! that gives no clue the brace needs quoting. Since `{partition}` is the
1451 //! required token for `partition_by`, this is a common copy-paste footgun.
1452 //! `Config::from_yaml` augments the parser error with a quoting hint; these
1453 //! tests pin that behavior (and guard that valid configs are untouched).
1454 use super::*;
1455
1456 /// A full, otherwise-valid config whose `prefix:` value is whatever the
1457 /// caller passes verbatim (quoted or not). Only the `prefix:` line varies,
1458 /// so any parse error is attributable to the brace under test.
1459 fn yaml_with_prefix(prefix_value: &str) -> String {
1460 format!(
1461 "source:\n\
1462 \x20 type: postgres\n\
1463 \x20 url: \"postgresql://localhost/test\"\n\
1464 exports:\n\
1465 \x20 - name: t\n\
1466 \x20 query: \"SELECT 1\"\n\
1467 \x20 format: parquet\n\
1468 \x20 partition_by: created_date\n\
1469 \x20 destination:\n\
1470 \x20 type: local\n\
1471 \x20 path: ./out\n\
1472 \x20 prefix: {prefix_value}\n"
1473 )
1474 }
1475
1476 const HINT_FRAGMENT: &str =
1477 "a YAML value containing { } (such as {partition} or {date}) must be quoted";
1478
1479 #[test]
1480 fn bare_partition_token_gets_quoting_hint() {
1481 // `prefix: {partition}` parses as a YAML map, so serde rejects it with
1482 // `invalid type: map, expected a string` — no clue it's a quoting bug.
1483 let err = Config::from_yaml(&yaml_with_prefix("{partition}")).unwrap_err();
1484 let msg = format!("{err:#}");
1485 assert!(
1486 msg.contains(HINT_FRAGMENT),
1487 "bare {{partition}} must carry the quoting hint; got: {msg}"
1488 );
1489 // The original parser detail (type + location) is preserved.
1490 assert!(
1491 msg.contains("invalid type: map") || msg.contains("line"),
1492 "the original parser error must be kept; got: {msg}"
1493 );
1494 }
1495
1496 #[test]
1497 fn trailing_text_after_brace_gets_quoting_hint() {
1498 // `prefix: {date}/{partition}/` runs the flow map into block context:
1499 // serde emits `did not find expected key ... while parsing a block
1500 // mapping`. Same footgun, different libyaml symptom.
1501 let err = Config::from_yaml(&yaml_with_prefix("{date}/{partition}/")).unwrap_err();
1502 let msg = format!("{err:#}");
1503 assert!(
1504 msg.contains(HINT_FRAGMENT),
1505 "{{date}}/{{partition}}/ must carry the quoting hint; got: {msg}"
1506 );
1507 }
1508
1509 #[test]
1510 fn unclosed_brace_gets_quoting_hint() {
1511 // `prefix: {partition` (unclosed) is the canonical flow-mapping scanner
1512 // error: `did not find expected ',' or '}' ... while parsing a flow
1513 // mapping`. The hint must still fire.
1514 let err = Config::from_yaml(&yaml_with_prefix("{partition")).unwrap_err();
1515 let msg = format!("{err:#}");
1516 assert!(
1517 msg.contains(HINT_FRAGMENT),
1518 "unclosed brace must carry the quoting hint; got: {msg}"
1519 );
1520 }
1521
1522 #[test]
1523 fn quoted_brace_value_loads_ok() {
1524 // The fix itself, applied: a properly quoted brace value parses and
1525 // validates. This is the guard that the hint never reaches a valid
1526 // config and the success path is unchanged.
1527 let cfg = Config::from_yaml(&yaml_with_prefix("\"exports/{partition}/\""))
1528 .expect("quoted {partition} prefix must load");
1529 assert_eq!(
1530 cfg.exports[0].destination.prefix.as_deref(),
1531 Some("exports/{partition}/")
1532 );
1533 }
1534
1535 #[test]
1536 fn config_without_braces_is_untouched() {
1537 // No brace anywhere: a plain valid config still loads, and an unrelated
1538 // YAML error elsewhere must not pick up a spurious quoting hint.
1539 Config::from_yaml(&yaml_with_prefix("exports/data/"))
1540 .expect("a brace-free prefix must load");
1541 }
1542
1543 // ── line_has_unquoted_brace_value() unit coverage ──────────────────────
1544
1545 #[test]
1546 fn unquoted_brace_value_is_detected() {
1547 assert!(line_has_unquoted_brace_value(" prefix: {partition}"));
1548 assert!(line_has_unquoted_brace_value(" path: {date}/out"));
1549 assert!(line_has_unquoted_brace_value("prefix: {partition")); // unclosed
1550 }
1551
1552 #[test]
1553 fn quoted_brace_value_is_not_flagged() {
1554 // Quotes around the value hide the brace from the scanner — not a bug.
1555 assert!(!line_has_unquoted_brace_value(
1556 " prefix: \"exports/{partition}/\""
1557 ));
1558 assert!(!line_has_unquoted_brace_value(" prefix: 'data/{date}/'"));
1559 }
1560
1561 #[test]
1562 fn env_placeholder_and_plain_values_are_not_flagged() {
1563 // `${VAR}` placeholders are resolved before the parse and are not the
1564 // footgun; plain brace-free values are obviously fine.
1565 assert!(!line_has_unquoted_brace_value(" url: ${DATABASE_URL}"));
1566 assert!(!line_has_unquoted_brace_value(" path: ./out"));
1567 assert!(!line_has_unquoted_brace_value(" # prefix: {partition}")); // comment
1568 assert!(!line_has_unquoted_brace_value(" prefix:")); // no value
1569 }
1570}
1571
1572#[cfg(test)]
1573mod sec_config_validation_regression {
1574 //! Regression edge-cases that pin the *compat boundaries* of the
1575 //! config-validation security fixes — the cases that distinguish a real
1576 //! attack from a legitimate loopback / dev-container / Docker pattern.
1577 //! These complement the RED tests in `sec_config_validation`: the RED
1578 //! tests assert the attack is rejected; these assert the fix stays narrow
1579 //! enough not to break local-dev usage (see CRITICAL COMPAT).
1580 use super::*;
1581
1582 /// A full, otherwise-valid config whose single export's `destination:`
1583 /// block is whatever the caller passes verbatim.
1584 fn yaml_with_destination(dest_block: &str) -> String {
1585 format!(
1586 "source:\n type: postgres\n url: \"postgresql://localhost/test\"\n\
1587 exports:\n - name: t\n query: \"SELECT 1\"\n format: parquet\n\
1588 {dest_block}"
1589 )
1590 }
1591
1592 // ── endpoint_host / is_loopback_host helpers ─────────────────────────────
1593
1594 #[test]
1595 fn endpoint_host_parses_forms() {
1596 assert_eq!(
1597 endpoint_host("https://attacker.example.com").as_deref(),
1598 Some("attacker.example.com")
1599 );
1600 // Port and path are stripped from the host.
1601 assert_eq!(
1602 endpoint_host("http://127.0.0.1:10000/devstoreaccount1").as_deref(),
1603 Some("127.0.0.1")
1604 );
1605 // userinfo head is dropped (host is after the last `@`).
1606 assert_eq!(
1607 endpoint_host("http://user:pass@127.0.0.1:9000").as_deref(),
1608 Some("127.0.0.1")
1609 );
1610 // Bracketed IPv6 literal keeps its address.
1611 assert_eq!(endpoint_host("http://[::1]:9000").as_deref(), Some("::1"));
1612 // Not a URL → None (treated as a non-loopback custom endpoint upstream).
1613 assert_eq!(endpoint_host("not-a-url"), None);
1614 assert_eq!(endpoint_host("://nohost"), None);
1615 }
1616
1617 #[test]
1618 fn loopback_host_classification() {
1619 for h in ["127.0.0.1", "127.0.0.53", "localhost", "::1"] {
1620 assert!(is_loopback_host(h), "{h} must be loopback");
1621 }
1622 for h in ["attacker.example.com", "evil.com", "10.0.0.1", "::2"] {
1623 assert!(!is_loopback_host(h), "{h} must be remote");
1624 }
1625 }
1626
1627 // ── V2/V12 endpoint: loopback accepted regardless of allow_anonymous ─────
1628
1629 #[test]
1630 fn loopback_endpoint_without_allow_anonymous_still_accepted() {
1631 // A loopback emulator endpoint with credentials (no allow_anonymous) is
1632 // the Minio-with-keys local-dev pattern and must stay accepted — the
1633 // exfil guard targets *remote* hosts, not localhost.
1634 let cfg = yaml_with_destination(
1635 " destination:\n type: s3\n bucket: b\n region: us-east-1\n\
1636 \x20 endpoint: http://127.0.0.1:9000\n access_key_env: AK\n secret_key_env: SK\n",
1637 );
1638 Config::from_yaml(&cfg).expect("loopback endpoint with creds must stay accepted");
1639 }
1640
1641 #[test]
1642 fn remote_https_endpoint_with_allow_anonymous_is_the_only_remote_escape() {
1643 // The documented escape hatch: an explicit anonymous (emulator) opt-in
1644 // permits a non-loopback endpoint (no credentials are sent). Without
1645 // allow_anonymous the same endpoint is rejected (covered by the RED
1646 // test); with it, accepted.
1647 let cfg = yaml_with_destination(
1648 " destination:\n type: gcs\n bucket: b\n\
1649 \x20 endpoint: https://emulator.example.com\n allow_anonymous: true\n",
1650 );
1651 Config::from_yaml(&cfg).expect("remote endpoint + allow_anonymous opt-in must be accepted");
1652 }
1653
1654 // ── V15 local path: absolute allowed (Docker mount), `..` rejected ───────
1655
1656 #[test]
1657 fn absolute_local_path_is_allowed() {
1658 // `path: /output` is a legitimate Docker volume-mount pattern
1659 // (examples/rivet.yaml) and must keep validating — only `..` climbs are
1660 // the traversal footgun.
1661 let cfg =
1662 yaml_with_destination(" destination:\n type: local\n path: /output\n");
1663 Config::from_yaml(&cfg).expect("absolute local path (Docker mount) must validate");
1664 }
1665
1666 #[test]
1667 fn dotdot_in_local_prefix_is_rejected() {
1668 // `prefix` is guarded the same as `path`.
1669 let cfg = yaml_with_destination(
1670 " destination:\n type: local\n path: ./out\n prefix: a/../b\n",
1671 );
1672 let err = Config::from_yaml(&cfg).unwrap_err();
1673 let msg = format!("{err:#}");
1674 assert!(
1675 msg.contains("prefix") && msg.contains(".."),
1676 "a '..' in the local prefix must be rejected naming prefix/..; got: {msg}"
1677 );
1678 }
1679
1680 // ── V13 TLS: explicit enforced mode + knob rejected; default-mode kept ───
1681
1682 #[test]
1683 fn tls_danger_knob_without_explicit_mode_still_accepted() {
1684 // The dev-container pattern `tls: { accept_invalid_certs: true }` against
1685 // a loopback self-signed cert (e.g. the MSSQL docker container) omits
1686 // `mode:` — there is no *explicit* mode to contradict, so it must keep
1687 // validating. The RED test rejects only the explicit-mode contradiction.
1688 let yaml = "source:\n type: mssql\n url: \"sqlserver://sa:pw@127.0.0.1:1433/db\"\n \
1689 tls:\n accept_invalid_certs: true\n\
1690 exports:\n - name: t\n query: \"SELECT 1\"\n format: parquet\n \
1691 destination:\n type: local\n path: ./out\n";
1692 Config::from_yaml(yaml)
1693 .expect("dev-container default-mode + accept_invalid_certs must stay accepted");
1694 }
1695
1696 #[test]
1697 fn tls_explicit_verify_ca_plus_invalid_hostnames_rejected() {
1698 // The hostname knob is flagged too, against any explicit enforced mode.
1699 let yaml = "source:\n type: postgres\n url: \"postgresql://localhost/test\"\n \
1700 tls:\n mode: verify-ca\n accept_invalid_hostnames: true\n\
1701 exports:\n - name: t\n query: \"SELECT 1\"\n format: parquet\n \
1702 destination:\n type: local\n path: ./out\n";
1703 let err = Config::from_yaml(yaml).unwrap_err();
1704 let msg = format!("{err:#}");
1705 assert!(
1706 msg.contains("accept_invalid_hostnames") && msg.contains("verify-ca"),
1707 "explicit verify-ca + accept_invalid_hostnames must be rejected; got: {msg}"
1708 );
1709 }
1710
1711 #[test]
1712 fn tls_explicit_disable_with_knob_is_not_flagged() {
1713 // `mode: disable` carries no verification promise to contradict, so the
1714 // danger knob is a no-op there and must not be rejected.
1715 let yaml = "source:\n type: postgres\n url: \"postgresql://localhost/test\"\n \
1716 tls:\n mode: disable\n accept_invalid_certs: true\n\
1717 exports:\n - name: t\n query: \"SELECT 1\"\n format: parquet\n \
1718 destination:\n type: local\n path: ./out\n";
1719 Config::from_yaml(yaml).expect("mode: disable + knob is a no-op and must validate");
1720 }
1721
1722 // ── V5 name: filename-safe predicate boundaries ──────────────────────────
1723
1724 #[test]
1725 fn filename_safe_name_boundaries() {
1726 for ok in ["t", "orders", "daily_events", "v2-2024", "name.with.dots"] {
1727 assert!(is_filename_safe_name(ok), "{ok:?} must be accepted");
1728 }
1729 for bad in [
1730 "",
1731 "..",
1732 "../x",
1733 "/abs",
1734 "sub/dir",
1735 "back\\slash",
1736 ".hidden",
1737 "with\u{0000}nul",
1738 ] {
1739 assert!(!is_filename_safe_name(bad), "{bad:?} must be rejected");
1740 }
1741 }
1742}
1743
1744#[cfg(test)]
1745mod sec_config_validation {
1746 //! RED security tests for config-load validation gaps (cluster:
1747 //! config-validation). Each asserts the SECURE behavior through the
1748 //! stable `Config::from_yaml` seam: a malicious config that is accepted
1749 //! today must be REJECTED (or, for warn-only knobs, surfaced as an
1750 //! error/loud warning) at config-load. These are expected to FAIL until
1751 //! the corresponding production fix lands.
1752 //!
1753 //! The pattern mirrors the existing `query_file` `..`/absolute-path guard
1754 //! in `validate_export` (see `config/tests/validation.rs`): a syntactic
1755 //! check that runs at config-validate time so `rivet check` / `rivet
1756 //! doctor` catch the problem before any connect/plan/upload step.
1757 use super::*;
1758
1759 /// A full, otherwise-valid config whose single export's `destination:`
1760 /// block is whatever the caller passes verbatim. Only the destination
1761 /// varies, so any rejection is attributable to the destination under test.
1762 fn yaml_with_destination(dest_block: &str) -> String {
1763 format!(
1764 "source:\n type: postgres\n url: \"postgresql://localhost/test\"\n\
1765 exports:\n - name: t\n query: \"SELECT 1\"\n format: parquet\n\
1766 {dest_block}"
1767 )
1768 }
1769
1770 // ── V2/V12: cloud-endpoint exfiltration + http cleartext ────────────────
1771 //
1772 // `destination.endpoint` is passed straight to the opendal S3/GCS/Azure
1773 // builder with no validation (see `src/destination/{s3,gcs,azure}.rs`),
1774 // so a committed config can silently redirect every export to an
1775 // attacker-controlled host. Two distinct gaps:
1776 // V2 — a custom *non-loopback* endpoint (data exfiltration target).
1777 // V12 — an `http://` (plaintext) endpoint (credentials + data on the
1778 // wire in cleartext).
1779 // The secure behavior is to reject (or require explicit opt-in) at
1780 // config-load. Loopback/emulator endpoints (Minio/Azurite/fake-gcs on
1781 // 127.0.0.1) MUST stay accepted — that path is exercised by the existing
1782 // `gcs_allow_anonymous_parses` test and the guard test below.
1783
1784 #[test]
1785 fn sec_s3_custom_endpoint_rejected() {
1786 // SEC-RED V2: a non-loopback custom S3 endpoint is an exfiltration
1787 // target — every part upload goes to attacker.example.com. Must be
1788 // rejected (or require explicit opt-in) at config-load. Accepted today.
1789 let cfg = yaml_with_destination(
1790 " destination:\n type: s3\n bucket: my-bucket\n region: us-east-1\n\
1791 \x20 endpoint: https://attacker.example.com\n",
1792 );
1793 let res = Config::from_yaml(&cfg);
1794 assert!(
1795 res.is_err(),
1796 "a non-loopback custom S3 endpoint (https://attacker.example.com) must be \
1797 rejected at config-load (data-exfiltration target); got Ok"
1798 );
1799 let msg = format!("{:#}", res.unwrap_err());
1800 assert!(
1801 msg.contains("endpoint"),
1802 "rejection must name the offending 'endpoint' field; got: {msg}"
1803 );
1804 }
1805
1806 #[test]
1807 fn sec_http_endpoint_rejected() {
1808 // SEC-RED V12: a plaintext http:// endpoint to a *remote* host sends
1809 // credentials and exported rows over the wire in cleartext. Must be
1810 // rejected (or require explicit opt-in) at config-load. Accepted today.
1811 // Use a non-loopback host so this is distinct from the Minio/Azurite
1812 // loopback emulator case (guarded below).
1813 let cfg = yaml_with_destination(
1814 " destination:\n type: s3\n bucket: my-bucket\n region: us-east-1\n\
1815 \x20 endpoint: http://evil.com\n",
1816 );
1817 let res = Config::from_yaml(&cfg);
1818 assert!(
1819 res.is_err(),
1820 "a plaintext http:// endpoint to a remote host (http://evil.com) must be \
1821 rejected at config-load (cleartext credentials + data); got Ok"
1822 );
1823 let msg = format!("{:#}", res.unwrap_err());
1824 assert!(
1825 msg.contains("endpoint") || msg.to_lowercase().contains("http"),
1826 "rejection must name the endpoint / cleartext problem; got: {msg}"
1827 );
1828 }
1829
1830 #[test]
1831 fn sec_loopback_endpoint_still_accepted_guard() {
1832 // SEC-RED V2/V12 (guard): a loopback emulator endpoint
1833 // (`http://127.0.0.1:9000` Minio, with allow_anonymous) is the
1834 // legitimate local-dev path and MUST stay accepted after the fix.
1835 // This pins that the endpoint rejection targets *remote* hosts, not
1836 // localhost — otherwise the fix breaks every Minio/Azurite/fake-gcs
1837 // integration test (see `gcs_allow_anonymous_parses`).
1838 let cfg = yaml_with_destination(
1839 " destination:\n type: s3\n bucket: my-bucket\n region: us-east-1\n\
1840 \x20 endpoint: http://127.0.0.1:9000\n allow_anonymous: true\n",
1841 );
1842 Config::from_yaml(&cfg)
1843 .expect("a loopback emulator endpoint with allow_anonymous must stay accepted");
1844 }
1845
1846 // ── V5: export `name` path traversal ────────────────────────────────────
1847 //
1848 // `ExportConfig.name` is a free-form `String` keyed into state tracking,
1849 // file logs, and (via the destination layout) output paths — yet it is
1850 // never validated. A name like `../../../etc/x`, an absolute `/abs/x`, a
1851 // bare slash, or an embedded NUL can escape the intended output tree.
1852 // Mirror the `query_file` `..`/absolute guard: reject at config-load.
1853
1854 #[test]
1855 fn sec_export_name_traversal_rejected() {
1856 // SEC-RED V5: a traversal / absolute / slash / NUL export name escapes
1857 // the output tree (and corrupts name-keyed state). Must be rejected at
1858 // config-load. Accepted today.
1859 for bad in ["../../../etc/x", "/abs/x", "sub/dir", "with\u{0000}nul"] {
1860 // `name:` is JSON-encoded so embedded slashes / NULs survive the
1861 // YAML parse verbatim and reach validation.
1862 let name_yaml = serde_json::to_string(bad).expect("encode name");
1863 let cfg = format!(
1864 "source:\n type: postgres\n url: \"postgresql://localhost/test\"\n\
1865 exports:\n - name: {name_yaml}\n query: \"SELECT 1\"\n format: parquet\n\
1866 \x20 destination:\n type: local\n path: ./out\n"
1867 );
1868 let res = Config::from_yaml(&cfg);
1869 assert!(
1870 res.is_err(),
1871 "export name {bad:?} (traversal/absolute/slash/NUL) must be rejected at \
1872 config-load; got Ok"
1873 );
1874 let msg = format!("{:#}", res.unwrap_err());
1875 assert!(
1876 msg.contains("name"),
1877 "rejection of name {bad:?} must name the offending 'name' field; got: {msg}"
1878 );
1879 }
1880 }
1881
1882 #[test]
1883 fn sec_export_name_normal_still_accepted_guard() {
1884 // SEC-RED V5 (guard): a plain, well-formed export name must keep
1885 // loading after the fix. Pins that the traversal check is narrow.
1886 let cfg = yaml_with_destination(" destination:\n type: local\n path: ./out\n");
1887 Config::from_yaml(&cfg).expect("a normal export name ('t') must stay accepted");
1888 }
1889
1890 // ── V15: local destination `path` traversal ─────────────────────────────
1891 //
1892 // `destination.path` for a `type: local` export is written verbatim to the
1893 // filesystem. A relative `../../../../tmp/x` or absolute path lets a
1894 // committed config write outside the intended output directory. Must be
1895 // rejected (or at minimum loudly surfaced) at config-load. Accepted today.
1896
1897 #[test]
1898 fn sec_local_dest_path_traversal_rejected() {
1899 // SEC-RED V15: a traversal local-destination path writes outside the
1900 // intended output tree. Must be rejected at config-load. Accepted today.
1901 let cfg = yaml_with_destination(
1902 " destination:\n type: local\n path: ../../../../tmp/x\n",
1903 );
1904 let res = Config::from_yaml(&cfg);
1905 assert!(
1906 res.is_err(),
1907 "a local destination path containing '..' (../../../../tmp/x) must be rejected \
1908 at config-load (writes outside the output tree); got Ok"
1909 );
1910 let msg = format!("{:#}", res.unwrap_err());
1911 assert!(
1912 msg.contains("path") || msg.contains(".."),
1913 "rejection must name the offending 'path' / traversal; got: {msg}"
1914 );
1915 }
1916
1917 // ── V13: dangerous TLS cert-knob combination ─────────────────────────────
1918 //
1919 // `tls: { mode: verify-full, accept_invalid_certs: true }` silently
1920 // *downgrades* the strongest mode to "accept any cert" — `verify-full`
1921 // promises chain + hostname verification, but the danger knob disables
1922 // chain verification (see `src/source/tls.rs::build_native_tls`). The
1923 // comment at `src/source/tls.rs:55-56` claims "Each one emits a warning at
1924 // config-time (see `Config::validate`)" — but `Config::validate` emits no
1925 // such warning today. The secure behavior is a LOUD error (or surfaced
1926 // warning) at config-load. No `Err`/warning is produced today, so this is
1927 // RED.
1928
1929 #[test]
1930 fn sec_accept_invalid_certs_warns() {
1931 // SEC-RED V13: verify-full + accept_invalid_certs: true is a silent
1932 // security downgrade that contradicts the chosen mode. It must be
1933 // loudly surfaced at config-load. The only stable secure seam is an
1934 // `Err` from `Config::from_yaml` (validate returns Ok today, and there
1935 // is no captured-warning seam exposed from here — see notes). Asserting
1936 // `Err` is the strongest secure assertion and is RED against current
1937 // code.
1938 let cfg = yaml_with_destination(" destination:\n type: local\n path: ./out\n");
1939 // Splice the TLS block into the source rather than the destination so
1940 // the rest of the config stays valid.
1941 let cfg = cfg.replace(
1942 " url: \"postgresql://localhost/test\"\n",
1943 " url: \"postgresql://localhost/test\"\n tls:\n mode: verify-full\n accept_invalid_certs: true\n",
1944 );
1945 let res = Config::from_yaml(&cfg);
1946 assert!(
1947 res.is_err(),
1948 "tls mode: verify-full with accept_invalid_certs: true is a silent security \
1949 downgrade and must be loudly surfaced (error) at config-load; got Ok"
1950 );
1951 let msg = format!("{:#}", res.unwrap_err());
1952 assert!(
1953 msg.contains("accept_invalid_certs") || msg.to_lowercase().contains("verify"),
1954 "the surfaced error must name the dangerous knob / mode contradiction; got: {msg}"
1955 );
1956 }
1957}