Skip to main content

faucet_core/
write_mode.rs

1//! Unified write-mode types + planner shared by every upsert-capable sink.
2
3use crate::error::FaucetError;
4use serde::{Deserialize, Serialize};
5use serde_json::{Map, Value};
6use std::collections::HashMap;
7
8/// Write semantics for a sink. Serialized snake_case. Default `Append`.
9// `#[non_exhaustive]`: this is a deliberate extension point — adding a write
10// mode (as `Overwrite` was, #492) is an additive change that ships as a minor
11// release. Downstream connectors that `match` on it must carry a wildcard arm;
12// the built-in sinks already gate on the specific modes they implement. Kept a
13// plain comment (not rustdoc) so the schema/rustdoc description is unchanged.
14#[derive(
15    Debug, Clone, Copy, Default, Serialize, Deserialize, schemars::JsonSchema, PartialEq, Eq,
16)]
17#[serde(rename_all = "snake_case")]
18#[non_exhaustive]
19pub enum WriteMode {
20    /// Insert every record (today's behaviour).
21    #[default]
22    Append,
23    /// Insert-or-update by `key`; optionally route delete-marked rows to deletes.
24    Upsert,
25    /// Delete by `key` for every record.
26    Delete,
27    /// Replace the entire destination with this run's records (truncate-load /
28    /// full refresh). The old contents are swapped out atomically only after the
29    /// run completes successfully, so a mid-run failure leaves them intact. No
30    /// `key` is required — it is a whole-dataset operation, not a keyed one.
31    Overwrite,
32}
33
34impl WriteMode {
35    /// Lowercase wire name, for error messages.
36    pub fn as_str(&self) -> &'static str {
37        match self {
38            WriteMode::Append => "append",
39            WriteMode::Upsert => "upsert",
40            WriteMode::Delete => "delete",
41            WriteMode::Overwrite => "overwrite",
42        }
43    }
44}
45
46/// Identifies a record as a delete (vs. an upsert) by a marker field's value.
47/// e.g. `{ field: "__op", values: ["d", "delete"] }`.
48#[derive(Debug, Clone, Default, Serialize, Deserialize, schemars::JsonSchema, PartialEq, Eq)]
49pub struct DeleteMarker {
50    /// Field name whose value flags a delete.
51    pub field: String,
52    /// Values of `field` that mean "this row is a delete".
53    pub values: Vec<String>,
54}
55
56/// Scope for a **scoped/windowed overwrite** (#518): with `write_mode:
57/// overwrite`, replace only the destination rows matching this scope instead of
58/// truncating the whole table. The sink-side sibling of scoped cleanup (#478).
59///
60/// v1 supports a half-open date/number **window** on a single column.
61#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
62#[serde(rename_all = "snake_case")]
63#[non_exhaustive]
64pub enum OverwriteScope {
65    /// Replace rows where `column` is in the half-open range `[from, to)`.
66    Window {
67        /// The destination column to window on.
68        column: String,
69        /// Inclusive lower bound.
70        from: Value,
71        /// Exclusive upper bound.
72        to: Value,
73    },
74}
75
76impl OverwriteScope {
77    /// The scoped column name.
78    pub fn column(&self) -> &str {
79        match self {
80            OverwriteScope::Window { column, .. } => column,
81        }
82    }
83
84    /// Validate the scope at config-load time.
85    pub fn validate(&self) -> Result<(), FaucetError> {
86        match self {
87            OverwriteScope::Window { column, from, to } => {
88                if column.trim().is_empty() {
89                    return Err(FaucetError::Config(
90                        "overwrite scope: window `column` must not be empty".into(),
91                    ));
92                }
93                if from.is_null() || to.is_null() {
94                    return Err(FaucetError::Config(
95                        "overwrite scope: window `from`/`to` must not be null".into(),
96                    ));
97                }
98                Ok(())
99            }
100        }
101    }
102
103    /// Render a SQL `WHERE` predicate over the (already-quoted) column using
104    /// escaped SQL **literals** for the bounds. Literals (not bind params) are
105    /// used so the engine coerces a string bound to the column's real type
106    /// (`date >= '2024-06-01'` works; a text-typed bind param would not). String
107    /// literals have their single quotes doubled, so a crafted bound cannot
108    /// break out of the quotes.
109    pub fn render_where_literal(&self, quoted_col: &str) -> String {
110        self.render_where_with(quoted_col, sql_literal)
111    }
112
113    /// [`render_where_literal`](Self::render_where_literal) with a
114    /// dialect-specific literal renderer, for dialects whose string literals
115    /// give some other character (BigQuery's backslash) escape meaning.
116    pub fn render_where_with(
117        &self,
118        quoted_col: &str,
119        literal: impl Fn(&Value) -> String,
120    ) -> String {
121        match self {
122            OverwriteScope::Window { from, to, .. } => format!(
123                "{quoted_col} >= {} AND {quoted_col} < {}",
124                literal(from),
125                literal(to)
126            ),
127        }
128    }
129}
130
131/// Render a JSON scalar as an **ANSI-standard** SQL literal: strings are
132/// single-quoted with embedded quotes doubled (`''`), numbers and booleans pass
133/// through in their canonical form, and anything else becomes `NULL`.
134///
135/// This is the house escaper for every dialect that follows the ANSI rule —
136/// Postgres (with `standard_conforming_strings`, the default since 9.1), MySQL
137/// under `NO_BACKSLASH_ESCAPES`, SQL Server, and SQLite. **Not BigQuery**:
138/// GoogleSQL treats `\'` as an escaped quote, so a trailing backslash would
139/// close the literal early. **Public so connectors escape
140/// through one audited implementation instead of hand-rolling a third
141/// convention** (#654 M14): the copies that existed before it was exported each
142/// invented their own rules, so a fix to one reached none of the others.
143///
144/// A dialect whose string literals treat some *other* character as a
145/// meta-character cannot use this function — doubling quotes alone does not
146/// neutralise a trailing escape character. Such a dialect must keep its own
147/// escaper, and that escaper's doc comment must state which extra characters it
148/// escapes and why (see `faucet_common_clickhouse::sql_literal`, where a
149/// backslash is a literal meta-character).
150pub fn sql_literal(v: &Value) -> String {
151    match v {
152        Value::String(s) => format!("'{}'", s.replace('\'', "''")),
153        Value::Number(n) => n.to_string(),
154        Value::Bool(b) => b.to_string(),
155        // Validated non-null upstream; any residual maps to NULL (never matches
156        // a range comparison, so the delete is a safe no-op rather than wrong).
157        _ => "NULL".to_owned(),
158    }
159}
160
161/// Shared write-mode config, embedded in each upsert-capable sink config via
162/// `#[serde(flatten)]` so `write_mode` / `key` / `delete_marker` appear at the
163/// sink-config top level.
164#[derive(Debug, Clone, Default, Serialize, Deserialize, schemars::JsonSchema)]
165pub struct WriteSpec {
166    /// Append (default), upsert, or delete.
167    #[serde(default)]
168    pub write_mode: WriteMode,
169    /// Key columns. Required and non-empty for upsert/delete; ignored for append.
170    #[serde(default)]
171    pub key: Vec<String>,
172    /// Optional. Upsert only: rows whose `field` matches one of `values` are
173    /// deletes; all others are upserts. The marker field is stripped from
174    /// upsert rows before writing.
175    #[serde(default, skip_serializing_if = "Option::is_none")]
176    pub delete_marker: Option<DeleteMarker>,
177    /// Runtime rollback settings (#706): the run id to journal under, the
178    /// run-id column, and whether to journal before-images / keep a replaced
179    /// table. Injected per invocation by the CLI when a config has a
180    /// top-level `rollback:` block — not meant to be written by hand.
181    #[serde(default, skip_serializing_if = "Option::is_none")]
182    pub rollback: Option<crate::rollback::RollbackWriteSpec>,
183}
184
185impl WriteSpec {
186    /// The run id this invocation writes under, when rollback is on.
187    pub fn rollback_run_id(&self) -> Option<&str> {
188        self.rollback.as_ref().map(|r| r.run_id.as_str())
189    }
190
191    /// Whether upserts/deletes must journal before-images (#706).
192    pub fn journals(&self) -> bool {
193        self.rollback.as_ref().is_some_and(|r| r.journal)
194    }
195
196    /// Whether an overwrite keeps the replaced table (#706).
197    pub fn keeps_previous(&self) -> bool {
198        self.rollback.as_ref().is_some_and(|r| r.keep_previous)
199    }
200
201    /// Validate internal consistency at config-load time.
202    pub fn validate(&self) -> Result<(), FaucetError> {
203        if matches!(self.write_mode, WriteMode::Upsert | WriteMode::Delete) && self.key.is_empty() {
204            return Err(FaucetError::Config(format!(
205                "write_mode: {} requires a non-empty `key`",
206                self.write_mode.as_str()
207            )));
208        }
209        Ok(())
210    }
211
212    /// Whether this spec makes writes converge by key — `write_mode: upsert`
213    /// or `delete` with a non-empty `key`. The canonical implementation of
214    /// [`Sink::dedups_by_key`](crate::Sink::dedups_by_key) for sinks that
215    /// flatten a `WriteSpec` into their config.
216    pub fn dedups_by_key(&self) -> bool {
217        matches!(self.write_mode, WriteMode::Upsert | WriteMode::Delete) && !self.key.is_empty()
218    }
219
220    /// Whether this spec requests full-destination replacement
221    /// ([`WriteMode::Overwrite`]). The canonical implementation of
222    /// [`Sink::is_overwrite`](crate::Sink::is_overwrite) for sinks that flatten
223    /// a `WriteSpec` into their config.
224    pub fn is_overwrite(&self) -> bool {
225        matches!(self.write_mode, WriteMode::Overwrite)
226    }
227}
228
229/// Ordered key column → value pairs, in `key` declaration order.
230#[derive(Debug, Clone, PartialEq)]
231pub struct KeyTuple(pub Vec<(String, Value)>);
232
233/// The partition of a page by write mode. Infallible to build — per-row
234/// failures (missing/null key) land in `failed` with their original page index
235/// so the caller can route them to a DLQ or abort.
236#[derive(Debug, Default)]
237pub struct WritePlan {
238    /// Rows to insert-or-update, deduped (last-write-wins), marker stripped.
239    pub upserts: Vec<Value>,
240    /// Key tuples to delete, deduped.
241    pub deletes: Vec<KeyTuple>,
242    /// `(page_index, message)` for rows whose key could not be extracted.
243    pub failed: Vec<(usize, String)>,
244}
245
246#[derive(Clone)]
247enum Action {
248    Upsert(Value),
249    Delete(KeyTuple),
250}
251
252/// Partition `page` into upserts + deletes per `spec`. The single place all six
253/// sinks share. `WriteMode::Append` should never reach here (callers route
254/// append separately); if it does, every row is treated as an upsert.
255///
256/// Keys deduplicate the way a typed SQL key column compares them: `7`, `7.0`
257/// and `"7"` are one key. A sink that keeps those apart (a document store
258/// matching on the JSON value) uses [`plan_writes_exact`].
259pub fn plan_writes(page: &[Value], spec: &WriteSpec) -> WritePlan {
260    plan_writes_by(page, spec, canonical)
261}
262
263/// [`plan_writes`] deduplicating only keys a document store treats as equal:
264/// `7` and `"7"` stay two rows, while numbers compare by value, so `7` and
265/// `7.0` are one — as a document store's `_id` match does (#862 M-018).
266pub fn plan_writes_exact(page: &[Value], spec: &WriteSpec) -> WritePlan {
267    plan_writes_by(page, spec, |k| {
268        let values: Vec<Value> = k.0.iter().map(|(_, v)| numeric_value(v)).collect();
269        crate::util::canonical_json(&Value::Array(values))
270    })
271}
272
273/// `v` with every whole float written as the integer it equals, so numbers
274/// compare by value; other values are unchanged.
275fn numeric_value(v: &Value) -> Value {
276    match v {
277        Value::Number(n) if n.is_f64() => {
278            let f = n.as_f64().unwrap_or(f64::NAN);
279            let whole = f.is_finite() && f.fract() == 0.0;
280            if whole && f >= i64::MIN as f64 && f < i64::MAX as f64 {
281                Value::from(f as i64)
282            } else if whole && f >= 0.0 && f < u64::MAX as f64 {
283                Value::from(f as u64)
284            } else {
285                v.clone()
286            }
287        }
288        Value::Array(items) => Value::Array(items.iter().map(numeric_value).collect()),
289        Value::Object(map) => Value::Object(
290            map.iter()
291                .map(|(k, v)| (k.clone(), numeric_value(v)))
292                .collect(),
293        ),
294        _ => v.clone(),
295    }
296}
297
298fn plan_writes_by(
299    page: &[Value],
300    spec: &WriteSpec,
301    canonical: impl Fn(&KeyTuple) -> String,
302) -> WritePlan {
303    debug_assert!(
304        matches!(spec.write_mode, WriteMode::Upsert | WriteMode::Delete),
305        "plan_writes is only for Upsert/Delete — Append and Overwrite are routed separately"
306    );
307    let mut plan = WritePlan::default();
308    let mut index: HashMap<String, usize> = HashMap::new();
309    let mut order: Vec<Action> = Vec::new();
310
311    for (i, rec) in page.iter().enumerate() {
312        let key_tuple = match extract_key(rec, &spec.key) {
313            Ok(k) => k,
314            Err(msg) => {
315                plan.failed.push((i, msg));
316                continue;
317            }
318        };
319        let canon = canonical(&key_tuple);
320
321        let is_delete = match spec.write_mode {
322            WriteMode::Delete => true,
323            WriteMode::Upsert => is_delete_marked(rec, spec.delete_marker.as_ref()),
324            WriteMode::Append | WriteMode::Overwrite => false,
325        };
326
327        let action = if is_delete {
328            Action::Delete(key_tuple)
329        } else {
330            Action::Upsert(strip_marker(rec.clone(), spec.delete_marker.as_ref()))
331        };
332
333        match index.get(&canon) {
334            Some(&slot) => order[slot] = action,
335            None => {
336                index.insert(canon, order.len());
337                order.push(action);
338            }
339        }
340    }
341
342    for action in order {
343        match action {
344            Action::Upsert(v) => plan.upserts.push(v),
345            Action::Delete(k) => plan.deletes.push(k),
346        }
347    }
348    plan
349}
350
351/// Split a keyed page into runs of consecutive upserts or deletes and plan
352/// each run, so the runs apply in page order. One [`plan_writes`] over the
353/// whole page applies every upsert before every delete; when a delete and a
354/// later upsert address keys that differ in JSON but are equal under the
355/// destination's collation (`"Bob"` / `"bob"`), that order deletes the row the
356/// page re-inserted. Rows with no usable key are left out — the caller reports
357/// them from its own [`plan_writes`] over the page.
358pub fn plan_writes_ordered(page: &[Value], spec: &WriteSpec) -> Vec<WritePlan> {
359    let mut plans = Vec::new();
360    let mut run_start: Option<usize> = None;
361    let mut run_is_delete = false;
362    let flush = |start: usize, end: usize, plans: &mut Vec<WritePlan>| {
363        let mut plan = plan_writes(&page[start..end], spec);
364        plan.failed.clear();
365        plans.push(plan);
366    };
367    for (i, rec) in page.iter().enumerate() {
368        if extract_key(rec, &spec.key).is_err() {
369            continue;
370        }
371        let is_delete = match spec.write_mode {
372            WriteMode::Delete => true,
373            WriteMode::Upsert => is_delete_marked(rec, spec.delete_marker.as_ref()),
374            WriteMode::Append | WriteMode::Overwrite => false,
375        };
376        match run_start {
377            Some(start) if is_delete != run_is_delete => {
378                flush(start, i, &mut plans);
379                run_start = Some(i);
380            }
381            Some(_) => {}
382            None => run_start = Some(i),
383        }
384        run_is_delete = is_delete;
385    }
386    if let Some(start) = run_start {
387        flush(start, page.len(), &mut plans);
388    }
389    plans
390}
391
392/// The rows of a keyed page whose key cannot be read, with the reason —
393/// what [`plan_writes`] reports as `failed`, without planning the page.
394pub fn key_failures(page: &[Value], spec: &WriteSpec) -> Vec<(usize, String)> {
395    page.iter()
396        .enumerate()
397        .filter_map(|(i, rec)| extract_key(rec, &spec.key).err().map(|m| (i, m)))
398        .collect()
399}
400
401/// The key tuple of a record in `key` order, or `None` when a key column is
402/// missing or null — the same rule [`plan_writes`] applies.
403pub fn record_key(rec: &Value, key: &[String]) -> Option<KeyTuple> {
404    extract_key(rec, key).ok()
405}
406
407/// Pull the key columns out of a record in `key` order. Missing key or null
408/// key value is an error.
409fn extract_key(rec: &Value, key: &[String]) -> Result<KeyTuple, String> {
410    let obj = rec
411        .as_object()
412        .ok_or_else(|| "record is not a JSON object".to_string())?;
413    let mut out = Vec::with_capacity(key.len());
414    for col in key {
415        match obj.get(col) {
416            None => return Err(format!("missing key column '{col}'")),
417            Some(Value::Null) => return Err(format!("null value for key column '{col}'")),
418            Some(v) => out.push((col.clone(), v.clone())),
419        }
420    }
421    Ok(KeyTuple(out))
422}
423
424fn is_delete_marked(rec: &Value, marker: Option<&DeleteMarker>) -> bool {
425    let Some(dm) = marker else { return false };
426    let Some(v) = rec.get(&dm.field) else {
427        return false;
428    };
429    let Some(s) = v.as_str() else { return false };
430    dm.values.iter().any(|m| m == s)
431}
432
433fn strip_marker(mut rec: Value, marker: Option<&DeleteMarker>) -> Value {
434    if let (Some(dm), Value::Object(map)) = (marker, &mut rec) {
435        map.remove(&dm.field);
436    }
437    rec
438}
439
440/// Stable canonical string for a key tuple, for dedup. Scalars compare by
441/// their text, as a database compares a key column, so `7`, `7.0` and `"7"`
442/// are one key — the rule `rollback::canonical_key` applies too.
443fn canonical(k: &KeyTuple) -> String {
444    let arr: Vec<String> = k.0.iter().map(|(_, v)| key_scalar_text(v)).collect();
445    serde_json::to_string(&arr).expect("a Vec<String> always serializes")
446}
447
448pub(crate) fn key_scalar_text(v: &Value) -> String {
449    match v {
450        Value::String(s) => s.clone(),
451        Value::Number(n) => match n.as_f64() {
452            Some(f)
453                if n.as_i64().is_none()
454                    && n.as_u64().is_none()
455                    && f.fract() == 0.0
456                    && f.abs() < 9_007_199_254_740_992.0 =>
457            {
458                (f as i64).to_string()
459            }
460            _ => n.to_string(),
461        },
462        Value::Bool(b) => b.to_string(),
463        other => crate::util::canonical_json(other),
464    }
465}
466
467/// Render a key tuple into a single document id (Elasticsearch `_id`).
468///
469/// A single-column key is rendered as its plain string / JSON form (no
470/// separator can collide). A **composite** key is rendered as a canonical JSON
471/// array of its values rather than a separator-join: a plain join is not
472/// injective — e.g. `["a_", "b"]` and `["a", "_b"]` both collapse to `"a__b"`
473/// under separator `"_"`, silently overwriting two distinct rows with one. JSON
474/// encoding escapes any separator-like characters in the values, so distinct key
475/// tuples always map to distinct ids.
476///
477/// Assumes each key column has a consistent JSON type across records (the
478/// normal case for SQL and CDC sources); it does not disambiguate, e.g., the
479/// integer `7` from the string `"7"` in the same column.
480pub fn key_to_doc_id(k: &KeyTuple, separator: &str) -> String {
481    let _ = separator; // retained for API stability; no separator can collide now
482    if k.0.len() == 1 {
483        return match &k.0[0].1 {
484            Value::String(s) => s.clone(),
485            other => crate::util::canonical_json(other),
486        };
487    }
488    let values: Vec<Value> = k.0.iter().map(|(_, v)| v.clone()).collect();
489    crate::util::canonical_json(&Value::Array(values))
490}
491
492/// Build a Mongo/ES filter document `{ col: value, … }` from a key tuple.
493pub fn key_to_filter(k: &KeyTuple) -> Map<String, Value> {
494    k.0.iter().map(|(c, v)| (c.clone(), v.clone())).collect()
495}
496
497#[cfg(test)]
498mod tests {
499    use super::*;
500    use serde_json::json;
501
502    #[test]
503    fn exact_planning_keeps_keys_of_different_json_types_apart() {
504        let spec = WriteSpec {
505            write_mode: WriteMode::Upsert,
506            key: vec!["id".into()],
507            ..Default::default()
508        };
509        let page = vec![
510            json!({"id": 7, "v": "a"}),
511            json!({"id": "7", "v": "b"}),
512            json!({"id": 7, "v": "c"}),
513        ];
514        let exact = plan_writes_exact(&page, &spec);
515        assert_eq!(
516            exact.upserts,
517            vec![json!({"id": 7, "v": "c"}), json!({"id": "7", "v": "b"})]
518        );
519        assert_eq!(
520            plan_writes(&page, &spec).upserts,
521            vec![json!({"id": 7, "v": "c"})]
522        );
523    }
524
525    #[test]
526    fn exact_planning_compares_numbers_by_value() {
527        let spec = WriteSpec {
528            write_mode: WriteMode::Upsert,
529            key: vec!["id".into()],
530            ..Default::default()
531        };
532        let page = vec![
533            json!({"id": 7, "v": 1}),
534            json!({"id": 7.0, "v": 2}),
535            json!({"id": 7.5, "v": 3}),
536            json!({"id": {"a": [1.0]}, "v": 4}),
537            json!({"id": {"a": [1]}, "v": 5}),
538            json!({"id": 9_007_199_254_740_993_u64, "v": 6}),
539            json!({"id": 9_007_199_254_740_992.0, "v": 7}),
540            json!({"id": 1.8e19, "v": 8}),
541            json!({"id": 1e300, "v": 9}),
542        ];
543        let v: Vec<Value> = plan_writes_exact(&page, &spec)
544            .upserts
545            .iter()
546            .map(|r| r["v"].clone())
547            .collect();
548        assert_eq!(
549            v,
550            vec![
551                json!(2),
552                json!(3),
553                json!(5),
554                json!(6),
555                json!(7),
556                json!(8),
557                json!(9)
558            ]
559        );
560    }
561
562    /// An object inserted in reverse-sorted key order (#817).
563    fn login_then_id() -> Value {
564        let mut m = serde_json::Map::new();
565        m.insert("login".into(), json!("o"));
566        m.insert("id".into(), json!(2));
567        Value::Object(m)
568    }
569
570    #[test]
571    fn object_key_values_render_with_sorted_keys() {
572        assert_eq!(key_scalar_text(&login_then_id()), r#"{"id":2,"login":"o"}"#);
573        let k = KeyTuple(vec![("k".into(), login_then_id())]);
574        assert_eq!(key_to_doc_id(&k, "_"), r#"{"id":2,"login":"o"}"#);
575    }
576
577    fn upsert_spec(keys: &[&str]) -> WriteSpec {
578        WriteSpec {
579            write_mode: WriteMode::Upsert,
580            key: keys.iter().map(|s| s.to_string()).collect(),
581            delete_marker: None,
582            rollback: None,
583        }
584    }
585
586    #[test]
587    fn ordered_plans_keep_page_order_across_upsert_delete_runs() {
588        let spec = WriteSpec {
589            write_mode: WriteMode::Upsert,
590            key: vec!["name".into()],
591            delete_marker: Some(DeleteMarker {
592                field: "op".into(),
593                values: vec!["d".into()],
594            }),
595            rollback: None,
596        };
597        let page = vec![
598            json!({"name": "Bob", "op": "d"}),
599            json!({"name": "bob", "v": 1}),
600            json!({"name": "carl", "v": 2}),
601            json!({"v": 3}),
602            json!({"name": "dee", "op": "d"}),
603        ];
604        let plans = plan_writes_ordered(&page, &spec);
605        let shape: Vec<(usize, usize)> = plans
606            .iter()
607            .map(|p| (p.upserts.len(), p.deletes.len()))
608            .collect();
609        assert_eq!(shape, vec![(0, 1), (2, 0), (0, 1)]);
610        assert!(plans.iter().all(|p| p.failed.is_empty()));
611        assert!(plan_writes_ordered(&[], &spec).is_empty());
612    }
613
614    #[test]
615    fn upsert_extracts_key_and_keeps_row() {
616        let plan = plan_writes(&[json!({"id": 1, "name": "a"})], &upsert_spec(&["id"]));
617        assert_eq!(plan.upserts, vec![json!({"id": 1, "name": "a"})]);
618        assert!(plan.deletes.is_empty());
619        assert!(plan.failed.is_empty());
620    }
621
622    #[test]
623    fn key_to_doc_id_single_key_is_plain() {
624        let k = KeyTuple(vec![("id".into(), json!(7))]);
625        assert_eq!(key_to_doc_id(&k, "_"), "7");
626        let k = KeyTuple(vec![("name".into(), json!("alice"))]);
627        assert_eq!(key_to_doc_id(&k, "_"), "alice");
628    }
629
630    #[test]
631    fn key_to_doc_id_composite_is_injective() {
632        // ["a_", "b"] and ["a", "_b"] must NOT collide (the F13 separator bug).
633        let k1 = KeyTuple(vec![("x".into(), json!("a_")), ("y".into(), json!("b"))]);
634        let k2 = KeyTuple(vec![("x".into(), json!("a")), ("y".into(), json!("_b"))]);
635        let id1 = key_to_doc_id(&k1, "_");
636        let id2 = key_to_doc_id(&k2, "_");
637        assert_ne!(id1, id2, "distinct composite keys must map to distinct ids");
638        // Mixed types also stay distinct.
639        let k3 = KeyTuple(vec![("x".into(), json!(1)), ("y".into(), json!("2"))]);
640        let k4 = KeyTuple(vec![("x".into(), json!("1")), ("y".into(), json!(2))]);
641        assert_ne!(key_to_doc_id(&k3, "_"), key_to_doc_id(&k4, "_"));
642    }
643
644    #[test]
645    fn missing_key_goes_to_failed_with_original_index() {
646        let plan = plan_writes(
647            &[json!({"id": 1}), json!({"name": "no-key"})],
648            &upsert_spec(&["id"]),
649        );
650        assert_eq!(plan.upserts.len(), 1);
651        assert_eq!(plan.failed.len(), 1);
652        assert_eq!(plan.failed[0].0, 1, "failed row keeps its page index");
653    }
654
655    #[test]
656    fn null_key_value_is_a_failure() {
657        let plan = plan_writes(&[json!({"id": null})], &upsert_spec(&["id"]));
658        assert!(plan.upserts.is_empty());
659        assert_eq!(plan.failed.len(), 1);
660    }
661
662    #[test]
663    fn ordered_plans_split_runs_and_skip_unkeyed_rows() {
664        let spec = WriteSpec {
665            write_mode: WriteMode::Upsert,
666            key: vec!["id".into()],
667            delete_marker: Some(DeleteMarker {
668                field: "op".into(),
669                values: vec!["d".into()],
670            }),
671            ..Default::default()
672        };
673        let page = vec![
674            json!({"id": 1, "v": "a"}),
675            json!({"v": "no key"}),
676            json!({"id": 1, "v": "b"}),
677            json!({"id": 2, "op": "d"}),
678            json!({"id": null}),
679            json!({"id": 3, "op": "d"}),
680            json!({"id": 2, "v": "c"}),
681        ];
682        let plans = plan_writes_ordered(&page, &spec);
683        assert_eq!(plans.len(), 3);
684        assert_eq!(plans[0].upserts, vec![json!({"id": 1, "v": "b"})]);
685        assert!(plans[0].deletes.is_empty() && plans[0].failed.is_empty());
686        assert_eq!(plans[1].deletes.len(), 2);
687        assert!(plans[1].upserts.is_empty() && plans[1].failed.is_empty());
688        assert_eq!(plans[2].upserts, vec![json!({"id": 2, "v": "c"})]);
689        let failures = key_failures(&page, &spec);
690        assert_eq!(
691            failures.iter().map(|(i, _)| *i).collect::<Vec<_>>(),
692            plan_writes(&page, &spec)
693                .failed
694                .iter()
695                .map(|(i, _)| *i)
696                .collect::<Vec<_>>()
697        );
698        assert_eq!(failures.len(), 2);
699        assert!(plan_writes_ordered(&[], &spec).is_empty());
700    }
701
702    #[test]
703    fn delete_marker_routes_to_deletes_and_strips_marker() {
704        let spec = WriteSpec {
705            write_mode: WriteMode::Upsert,
706            key: vec!["id".into()],
707            delete_marker: Some(DeleteMarker {
708                field: "__op".into(),
709                values: vec!["d".into()],
710            }),
711            rollback: None,
712        };
713        let plan = plan_writes(
714            &[
715                json!({"id": 1, "name": "a", "__op": "u"}),
716                json!({"id": 2, "__op": "d"}),
717            ],
718            &spec,
719        );
720        assert_eq!(plan.upserts, vec![json!({"id": 1, "name": "a"})]);
721        assert_eq!(plan.deletes.len(), 1);
722        assert_eq!(plan.deletes[0].0, vec![("id".to_string(), json!(2))]);
723    }
724
725    #[test]
726    fn last_write_wins_dedup_keeps_final_upsert() {
727        let plan = plan_writes(
728            &[json!({"id": 1, "v": "old"}), json!({"id": 1, "v": "new"})],
729            &upsert_spec(&["id"]),
730        );
731        assert_eq!(plan.upserts, vec![json!({"id": 1, "v": "new"})]);
732    }
733
734    #[test]
735    fn mixed_json_types_of_one_key_dedup_as_one_key() {
736        let plan = plan_writes(
737            &[
738                json!({"id": 7, "v": "a"}),
739                json!({"id": 7.0, "v": "b"}),
740                json!({"id": "7", "v": "c"}),
741                json!({"id": 7.5, "v": "d"}),
742            ],
743            &upsert_spec(&["id"]),
744        );
745        assert_eq!(
746            plan.upserts,
747            vec![json!({"id": "7", "v": "c"}), json!({"id": 7.5, "v": "d"})]
748        );
749    }
750
751    #[test]
752    fn last_write_wins_delete_after_upsert_is_a_delete() {
753        let spec = WriteSpec {
754            write_mode: WriteMode::Upsert,
755            key: vec!["id".into()],
756            delete_marker: Some(DeleteMarker {
757                field: "__op".into(),
758                values: vec!["d".into()],
759            }),
760            rollback: None,
761        };
762        let plan = plan_writes(
763            &[json!({"id": 1, "__op": "u"}), json!({"id": 1, "__op": "d"})],
764            &spec,
765        );
766        assert!(plan.upserts.is_empty());
767        assert_eq!(plan.deletes.len(), 1);
768    }
769
770    #[test]
771    fn delete_mode_routes_every_row_to_deletes() {
772        let spec = WriteSpec {
773            write_mode: WriteMode::Delete,
774            key: vec!["id".into()],
775            delete_marker: None,
776            rollback: None,
777        };
778        let plan = plan_writes(&[json!({"id": 1}), json!({"id": 2})], &spec);
779        assert!(plan.upserts.is_empty());
780        assert_eq!(plan.deletes.len(), 2);
781    }
782
783    #[test]
784    fn composite_key_tuple_is_ordered() {
785        let plan = plan_writes(
786            &[json!({"a": 1, "b": 2, "v": 9})],
787            &upsert_spec(&["a", "b"]),
788        );
789        assert_eq!(plan.upserts.len(), 1);
790        let plan2 = plan_writes(
791            &[
792                json!({"a": 1, "b": 2, "v": "x"}),
793                json!({"a": 1, "b": 3, "v": "y"}),
794            ],
795            &upsert_spec(&["a", "b"]),
796        );
797        assert_eq!(plan2.upserts.len(), 2, "(1,2) and (1,3) are distinct keys");
798    }
799
800    #[test]
801    fn validate_rejects_upsert_without_key() {
802        let spec = WriteSpec {
803            write_mode: WriteMode::Upsert,
804            key: vec![],
805            delete_marker: None,
806            rollback: None,
807        };
808        assert!(spec.validate().is_err());
809    }
810
811    #[test]
812    fn validate_allows_append_without_key() {
813        assert!(WriteSpec::default().validate().is_ok());
814    }
815
816    #[test]
817    fn dedups_by_key_requires_keyed_upsert_or_delete() {
818        assert!(!WriteSpec::default().dedups_by_key());
819        let upsert = WriteSpec {
820            write_mode: WriteMode::Upsert,
821            key: vec!["id".into()],
822            delete_marker: None,
823            rollback: None,
824        };
825        assert!(upsert.dedups_by_key());
826        let delete = WriteSpec {
827            write_mode: WriteMode::Delete,
828            key: vec!["id".into()],
829            delete_marker: None,
830            rollback: None,
831        };
832        assert!(delete.dedups_by_key());
833        // An (invalid) keyless upsert never claims keyed dedup.
834        let keyless = WriteSpec {
835            write_mode: WriteMode::Upsert,
836            key: vec![],
837            delete_marker: None,
838            rollback: None,
839        };
840        assert!(!keyless.dedups_by_key());
841    }
842
843    #[test]
844    fn last_write_wins_upsert_after_delete_is_an_upsert() {
845        // Inverse of the delete-after-upsert case: [delete, upsert] → upsert wins.
846        let spec = WriteSpec {
847            write_mode: WriteMode::Upsert,
848            key: vec!["id".into()],
849            delete_marker: Some(DeleteMarker {
850                field: "__op".into(),
851                values: vec!["d".into()],
852            }),
853            rollback: None,
854        };
855        let plan = plan_writes(
856            &[
857                json!({"id": 1, "__op": "d"}),
858                json!({"id": 1, "v": 9, "__op": "u"}),
859            ],
860            &spec,
861        );
862        assert!(plan.deletes.is_empty());
863        assert_eq!(plan.upserts, vec![json!({"id": 1, "v": 9})]);
864    }
865
866    #[test]
867    fn overwrite_mode_flags_and_needs_no_key() {
868        let spec = WriteSpec {
869            write_mode: WriteMode::Overwrite,
870            ..Default::default()
871        };
872        assert!(spec.is_overwrite());
873        assert!(!spec.dedups_by_key());
874        // Overwrite is a whole-dataset op — no key required, so validate passes.
875        assert!(spec.validate().is_ok());
876        assert_eq!(WriteMode::Overwrite.as_str(), "overwrite");
877        // Non-overwrite specs report false.
878        assert!(!WriteSpec::default().is_overwrite());
879        assert!(!upsert_spec(&["id"]).is_overwrite());
880    }
881
882    #[test]
883    fn overwrite_scope_window_validates_and_renders() {
884        let scope = OverwriteScope::Window {
885            column: "posting_date".into(),
886            from: json!("2024-06-01"),
887            to: json!("2024-07-01"),
888        };
889        assert_eq!(scope.column(), "posting_date");
890        assert!(scope.validate().is_ok());
891        let whr = scope.render_where_literal("\"posting_date\"");
892        assert_eq!(
893            whr,
894            "\"posting_date\" >= '2024-06-01' AND \"posting_date\" < '2024-07-01'"
895        );
896
897        // Empty column / null bounds are rejected.
898        assert!(
899            OverwriteScope::Window {
900                column: " ".into(),
901                from: json!(1),
902                to: json!(2)
903            }
904            .validate()
905            .is_err()
906        );
907        assert!(
908            OverwriteScope::Window {
909                column: "c".into(),
910                from: json!(null),
911                to: json!(2)
912            }
913            .validate()
914            .is_err()
915        );
916    }
917
918    #[test]
919    fn scope_window_number_bounds() {
920        let scope = OverwriteScope::Window {
921            column: "seq".into(),
922            from: json!(100),
923            to: json!(200),
924        };
925        assert!(scope.validate().is_ok());
926        assert_eq!(
927            scope.render_where_literal("`seq`"),
928            "`seq` >= 100 AND `seq` < 200"
929        );
930    }
931
932    #[test]
933    fn sql_literal_renders_each_scalar_kind_by_the_ansi_rule() {
934        // Strings: single-quoted, embedded quotes doubled — and *only* quotes,
935        // since a backslash is an ordinary character under the ANSI rule.
936        assert_eq!(sql_literal(&json!("plain")), "'plain'");
937        assert_eq!(sql_literal(&json!("O'Brien")), "'O''Brien'");
938        assert_eq!(sql_literal(&json!("a\\b")), "'a\\b'");
939        // Every quote is doubled, not just the first: 5 interior quotes → 10.
940        assert_eq!(sql_literal(&json!("'' ' ''")), "''''' '' '''''");
941        // An empty string stays a valid (empty) literal, not bare quotes-less text.
942        assert_eq!(sql_literal(&json!("")), "''");
943        // Numbers and booleans pass through in canonical form.
944        assert_eq!(sql_literal(&json!(42)), "42");
945        assert_eq!(sql_literal(&json!(-1.5)), "-1.5");
946        assert_eq!(sql_literal(&json!(true)), "true");
947        assert_eq!(sql_literal(&json!(false)), "false");
948        // Null and non-scalars degrade to NULL rather than injecting structure.
949        assert_eq!(sql_literal(&Value::Null), "NULL");
950        assert_eq!(sql_literal(&json!({"a": 1})), "NULL");
951        assert_eq!(sql_literal(&json!([1, 2])), "NULL");
952    }
953
954    #[test]
955    fn sql_literal_neutralises_a_quote_breakout_attempt() {
956        // The whole point of the escaper: a crafted value must stay one literal.
957        let out = sql_literal(&json!("x' OR 1=1 --"));
958        assert_eq!(out, "'x'' OR 1=1 --'");
959        // Exactly one opening and one closing quote delimit the literal: every
960        // interior quote is part of a doubled pair.
961        let interior = &out[1..out.len() - 1];
962        assert!(
963            interior.matches('\'').count().is_multiple_of(2),
964            "unpaired quote inside literal: {out}"
965        );
966    }
967
968    #[test]
969    fn scope_literal_escapes_quotes() {
970        // A crafted bound cannot break out of the string literal.
971        let scope = OverwriteScope::Window {
972            column: "c".into(),
973            from: json!("x' OR '1'='1"),
974            to: json!("z"),
975        };
976        let whr = scope.render_where_literal("\"c\"");
977        assert!(whr.contains("'x'' OR ''1''=''1'"), "{whr}");
978    }
979
980    #[test]
981    fn render_where_with_uses_the_dialect_renderer() {
982        let scope = OverwriteScope::Window {
983            column: "c".into(),
984            from: json!("a"),
985            to: json!(5),
986        };
987        let whr = scope.render_where_with("`c`", |v| format!("<{v}>"));
988        assert_eq!(whr, "`c` >= <\"a\"> AND `c` < <5>");
989    }
990
991    #[test]
992    fn overwrite_deserializes_from_wire() {
993        let spec: WriteSpec = serde_json::from_value(json!({"write_mode": "overwrite"})).unwrap();
994        assert_eq!(spec.write_mode, WriteMode::Overwrite);
995        assert!(spec.is_overwrite());
996    }
997
998    #[test]
999    fn empty_page_produces_empty_plan() {
1000        let plan = plan_writes(&[], &upsert_spec(&["id"]));
1001        assert!(plan.upserts.is_empty());
1002        assert!(plan.deletes.is_empty());
1003        assert!(plan.failed.is_empty());
1004    }
1005
1006    #[test]
1007    fn delete_mode_dedups_repeated_key() {
1008        // Same key deleted twice in one page collapses to a single delete.
1009        let spec = WriteSpec {
1010            write_mode: WriteMode::Delete,
1011            key: vec!["id".into()],
1012            delete_marker: None,
1013            rollback: None,
1014        };
1015        let plan = plan_writes(&[json!({"id": 1}), json!({"id": 1})], &spec);
1016        assert_eq!(plan.deletes.len(), 1);
1017    }
1018}