Skip to main content

uqa_fdw/
lib.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Foreign data wrappers for external sources.
8//!
9//! A [`ForeignServer`] captures the connection metadata for an external
10//! source; a [`ForeignTable`] is a virtual table backed by such a
11//! server. The [`FDWHandler`] trait drives the actual scan, and
12//! [`MemoryHandler`] is the simplest concrete implementation: it
13//! stores pre-loaded rows and applies projection, pushdown predicate
14//! filtering, and a row limit before returning.
15
16// Native handlers are opt-in and unavailable on emscripten.
17// Registry types and the in-memory handler stay available everywhere.
18#[cfg(all(not(target_os = "emscripten"), feature = "arrow"))]
19pub mod arrow_flight;
20#[cfg(all(not(target_os = "emscripten"), feature = "arrow"))]
21pub mod arrow_ipc;
22#[cfg(all(not(target_os = "emscripten"), feature = "duckdb"))]
23pub mod duckdb;
24
25#[cfg(all(not(target_os = "emscripten"), feature = "arrow"))]
26pub use arrow_flight::{
27    build_where_clause as build_arrow_flight_where_clause,
28    prepare_query as arrow_flight_prepare_query, quote_literal as arrow_flight_quote_literal,
29    ArrowFlightPrepareError,
30};
31#[cfg(all(not(target_os = "emscripten"), feature = "arrow"))]
32pub use arrow_ipc::ArrowIpcHandler as ArrowHandler;
33#[cfg(all(not(target_os = "emscripten"), feature = "arrow"))]
34pub use arrow_ipc::{ArrowIpcHandler, ArrowIpcPrepareError};
35#[cfg(all(not(target_os = "emscripten"), feature = "duckdb"))]
36pub use duckdb::{
37    build_where_clause as build_duckdb_where_clause, normalize_source as duckdb_normalize_source,
38    prepare_query as duckdb_prepare_query, DuckDBHandler, DuckDBPrepareError, FILE_READERS,
39};
40
41use std::collections::BTreeMap;
42
43use uqa_core::Value;
44
45#[derive(Debug, Clone)]
46pub struct ColumnDef {
47    pub name: String,
48    pub ty: ColumnType,
49}
50
51#[derive(Debug, Clone, PartialEq, Eq)]
52pub enum RangeSubtype {
53    Integer,
54    BigInteger,
55    Numeric,
56    Date,
57    Timestamp,
58    TimestampTz,
59}
60
61#[derive(Debug, Clone, PartialEq, Eq)]
62pub enum ColumnType {
63    SmallInteger,
64    Integer,
65    BigInteger,
66    Oid,
67    Xid,
68    Real,
69    DoublePrecision,
70    Numeric {
71        precision: Option<u32>,
72        scale: Option<i32>,
73    },
74    Text,
75    RefCursor,
76    Xml,
77    Name,
78    Uuid,
79    Varchar(Option<u32>),
80    Bpchar,
81    Character(u32),
82    Bool,
83    Void,
84    Bytes,
85    /// Fixed bit strings retain packed values and an optional declared logical length.
86    Bit(Option<u32>),
87    /// Varying bit strings retain packed values and an optional maximum length.
88    VarBit(Option<u32>),
89    InternalChar,
90    Regproc,
91    Regprocedure,
92    Regclass,
93    Regnamespace,
94    /// `PostgreSQL` collation object identifier (`pg_catalog.regcollation`).
95    Regcollation,
96    Regrole,
97    Regtype,
98    PgNodeTree,
99    AclItem,
100    Int2Vector,
101    OidVector,
102    AnyArray,
103    Record,
104    Json,
105    JsonB,
106    Date,
107    Time,
108    TimeTz,
109    Timestamp,
110    TimestampTz,
111    Interval,
112    Range(RangeSubtype),
113    Multirange(RangeSubtype),
114    /// Catalog-bound range. Predicates require the SQL-selected endpoint operations.
115    DeclaredRange {
116        schema: String,
117        name: String,
118        oid: u32,
119    },
120    /// Catalog-bound multirange, independently named from its range type.
121    DeclaredMultirange {
122        schema: String,
123        name: String,
124        oid: u32,
125    },
126    Vector(u32),
127    Tensor(u32),
128    Domain {
129        schema: String,
130        name: String,
131        oid: u32,
132        base: Box<ColumnType>,
133    },
134    /// A user-defined enum. Handlers exchange its label text; label order is not lexical, so handlers must not evaluate order predicates on it.
135    Enum {
136        schema: String,
137        name: String,
138        oid: u32,
139    },
140    /// A composite type. Handlers exchange records whose fields carry the type's attribute names in attribute order.
141    Composite {
142        schema: String,
143        name: String,
144        oid: u32,
145    },
146    /// An array whose element metadata is preserved across the SQL/FDW boundary.
147    Array(Box<ColumnType>),
148}
149
150#[derive(Debug, Clone)]
151pub struct ForeignServer {
152    pub name: String,
153    /// Handler type — `"memory"`, `"duckdb_fdw"`, `"arrow_fdw"`, ...
154    pub fdw_type: String,
155    pub options: BTreeMap<String, String>,
156}
157
158#[derive(Debug, Clone)]
159pub struct ForeignTable {
160    pub name: String,
161    pub server_name: String,
162    pub columns: Vec<ColumnDef>,
163    pub options: BTreeMap<String, String>,
164}
165
166/// Pushdown comparison operator. Operators outside this set must be
167/// evaluated above the wrapper; handlers may opt to ignore unknown
168/// values rather than fail.
169#[derive(Debug, Clone, Copy, PartialEq, Eq)]
170pub enum PredicateOp {
171    Eq,
172    NotEq,
173    Lt,
174    LtEq,
175    Gt,
176    GtEq,
177    In,
178    Like,
179    NotLike,
180    ILike,
181    NotILike,
182}
183
184impl PredicateOp {
185    /// Render the operator as the SQL token used in a `WHERE` clause.
186    /// `In` is intentionally absent here -- callers wrap the IN list
187    /// with the column themselves.
188    pub fn sql_token(self) -> &'static str {
189        match self {
190            PredicateOp::Eq => "=",
191            PredicateOp::NotEq => "!=",
192            PredicateOp::Lt => "<",
193            PredicateOp::LtEq => "<=",
194            PredicateOp::Gt => ">",
195            PredicateOp::GtEq => ">=",
196            PredicateOp::In => "IN",
197            PredicateOp::Like => "LIKE",
198            PredicateOp::NotLike => "NOT LIKE",
199            PredicateOp::ILike => "ILIKE",
200            PredicateOp::NotILike => "NOT ILIKE",
201        }
202    }
203}
204
205#[derive(Debug, Clone)]
206pub struct FDWPredicate {
207    pub column: String,
208    pub operator: PredicateOp,
209    /// Scalar literal for comparisons; a `Value::List` for `In`.
210    pub value: Value,
211}
212
213pub type Row = BTreeMap<String, Value>;
214pub type RowStream = Box<dyn Iterator<Item = Result<Row, FDWError>> + Send>;
215
216#[derive(Debug, thiserror::Error)]
217pub enum FDWError {
218    #[cfg(all(not(target_os = "emscripten"), feature = "duckdb"))]
219    #[error(transparent)]
220    DuckDBPrepare(#[from] DuckDBPrepareError),
221    #[cfg(all(not(target_os = "emscripten"), feature = "arrow"))]
222    #[error(transparent)]
223    ArrowIpcPrepare(#[from] ArrowIpcPrepareError),
224    #[cfg(all(not(target_os = "emscripten"), feature = "duckdb"))]
225    #[error(transparent)]
226    DuckDB(#[from] ::duckdb::Error),
227    #[cfg(all(not(target_os = "emscripten"), feature = "arrow"))]
228    #[error(transparent)]
229    Arrow(#[from] arrow_schema::ArrowError),
230    #[error(transparent)]
231    Io(#[from] std::io::Error),
232    #[error("Unsupported FDW value: {0}")]
233    UnsupportedValue(String),
234    #[error("Foreign table `{0}` is not loaded by the memory FDW")]
235    UnknownTable(String),
236    #[error("Invalid FDW predicate: {0}")]
237    InvalidPredicate(String),
238    #[error("{0}")]
239    Other(String),
240}
241
242pub trait FDWHandler: Send + Sync {
243    fn scan(
244        &self,
245        table: &ForeignTable,
246        columns: Option<&[String]>,
247        predicates: &[FDWPredicate],
248        limit: Option<u64>,
249    ) -> Result<Vec<Row>, FDWError>;
250
251    /// Pull-based scan ABI. Existing wrappers inherit the vector adapter;
252    /// wrappers capable of cursor/batch reads can override this method and
253    /// avoid a cardinality-sized intermediate allocation.
254    fn scan_stream(
255        &self,
256        table: &ForeignTable,
257        columns: Option<&[String]>,
258        predicates: &[FDWPredicate],
259        limit: Option<u64>,
260    ) -> Result<RowStream, FDWError> {
261        Ok(Box::new(
262            self.scan(table, columns, predicates, limit)?
263                .into_iter()
264                .map(Ok),
265        ))
266    }
267
268    fn close(&self) {}
269}
270
271/// `MemoryHandler` keeps a flat row vector keyed on `ForeignTable::name`.
272/// `scan` projects, filters, and limits in pure Rust — handy for tests
273/// and as a baseline reference for richer backends.
274#[derive(Debug, Default)]
275pub struct MemoryHandler {
276    tables: BTreeMap<String, Vec<Row>>,
277}
278
279impl MemoryHandler {
280    pub fn new() -> Self {
281        Self::default()
282    }
283
284    pub fn load(&mut self, table_name: impl Into<String>, rows: Vec<Row>) {
285        self.tables.insert(table_name.into(), rows);
286    }
287}
288
289impl FDWHandler for MemoryHandler {
290    fn scan(
291        &self,
292        table: &ForeignTable,
293        columns: Option<&[String]>,
294        predicates: &[FDWPredicate],
295        limit: Option<u64>,
296    ) -> Result<Vec<Row>, FDWError> {
297        let Some(rows) = self.tables.get(&table.name) else {
298            return Err(FDWError::UnknownTable(table.name.clone()));
299        };
300        let mut out: Vec<Row> = Vec::new();
301        for row in rows {
302            if limit.is_some_and(|cap| limit_reached(out.len(), cap)) {
303                break;
304            }
305            if !row_matches_predicates(row, predicates)? {
306                continue;
307            }
308            out.push(project_row(row, columns));
309        }
310        Ok(out)
311    }
312}
313
314pub fn row_matches_predicates(row: &Row, predicates: &[FDWPredicate]) -> Result<bool, FDWError> {
315    for predicate in predicates {
316        if !eval_predicate(predicate, row)? {
317            return Ok(false);
318        }
319    }
320    Ok(true)
321}
322
323pub fn project_row(row: &Row, columns: Option<&[String]>) -> Row {
324    match columns {
325        Some(cols) => {
326            let mut keep = Row::new();
327            for col in cols {
328                keep.insert(col.clone(), row.get(col).cloned().unwrap_or(Value::Null));
329            }
330            keep
331        }
332        None => row.clone(),
333    }
334}
335
336fn eval_predicate(p: &FDWPredicate, row: &Row) -> Result<bool, FDWError> {
337    let lhs = row.get(&p.column);
338    let matches = match p.operator {
339        PredicateOp::Eq if matches!(p.value, Value::Null) => is_null(lhs),
340        PredicateOp::Eq => {
341            lhs.is_some_and(|value| !matches!(value, Value::Null) && value == &p.value)
342        }
343        PredicateOp::NotEq if matches!(p.value, Value::Null) => is_non_null(lhs),
344        PredicateOp::NotEq => {
345            lhs.is_some_and(|value| !matches!(value, Value::Null) && value != &p.value)
346        }
347        PredicateOp::Lt => compare_non_null(lhs, &p.value, p, |lhs, rhs| lhs < rhs)?,
348        PredicateOp::LtEq => compare_non_null(lhs, &p.value, p, |lhs, rhs| lhs <= rhs)?,
349        PredicateOp::Gt => compare_non_null(lhs, &p.value, p, |lhs, rhs| lhs > rhs)?,
350        PredicateOp::GtEq => compare_non_null(lhs, &p.value, p, |lhs, rhs| lhs >= rhs)?,
351        PredicateOp::In => match (&p.value, lhs) {
352            (Value::List(items), Some(value)) if !matches!(value, Value::Null) => items
353                .iter()
354                .any(|item| !matches!(item, Value::Null) && item == value),
355            (Value::List(_), None | Some(Value::Null)) => false,
356            (other, _) => {
357                return Err(FDWError::InvalidPredicate(format!(
358                    "IN on `{}` requires a list, got {other:?}",
359                    p.column
360                )));
361            }
362        },
363        PredicateOp::Like => string_predicate(lhs, &p.value, p, false, false)?,
364        PredicateOp::NotLike => string_predicate(lhs, &p.value, p, false, true)?,
365        PredicateOp::ILike => string_predicate(lhs, &p.value, p, true, false)?,
366        PredicateOp::NotILike => string_predicate(lhs, &p.value, p, true, true)?,
367    };
368    Ok(matches)
369}
370
371fn is_null(value: Option<&Value>) -> bool {
372    match value {
373        None | Some(Value::Null) => true,
374        Some(_) => false,
375    }
376}
377
378fn is_non_null(value: Option<&Value>) -> bool {
379    value.is_some_and(|value| !matches!(value, Value::Null))
380}
381
382fn compare_non_null(
383    lhs: Option<&Value>,
384    rhs: &Value,
385    predicate: &FDWPredicate,
386    compare: impl FnOnce(&Value, &Value) -> bool,
387) -> Result<bool, FDWError> {
388    if matches!(rhs, Value::Null) {
389        return Err(FDWError::InvalidPredicate(format!(
390            "{:?} cannot compare `{}` with NULL",
391            predicate.operator, predicate.column
392        )));
393    }
394    Ok(lhs.is_some_and(|lhs| !matches!(lhs, Value::Null) && compare(lhs, rhs)))
395}
396
397pub(crate) fn limit_reached(len: usize, cap: u64) -> bool {
398    usize::try_from(cap).is_ok_and(|cap| len >= cap)
399}
400
401fn string_predicate(
402    lhs: Option<&Value>,
403    rhs: &Value,
404    predicate: &FDWPredicate,
405    case_insensitive: bool,
406    negate: bool,
407) -> Result<bool, FDWError> {
408    let Value::Str(pattern) = rhs else {
409        return Err(FDWError::InvalidPredicate(format!(
410            "{:?} on `{}` requires a string pattern",
411            predicate.operator, predicate.column
412        )));
413    };
414    let Some(lhs) = lhs else {
415        return Ok(false);
416    };
417    let Value::Str(haystack) = lhs else {
418        if matches!(lhs, Value::Null) {
419            return Ok(false);
420        }
421        return Err(FDWError::InvalidPredicate(format!(
422            "{:?} on `{}` requires string input",
423            predicate.operator, predicate.column
424        )));
425    };
426    let matched = if case_insensitive {
427        sql_like(&haystack.to_lowercase(), &pattern.to_lowercase())?
428    } else {
429        sql_like(haystack, pattern)?
430    };
431    if negate {
432        Ok(!matched)
433    } else {
434        Ok(matched)
435    }
436}
437
438/// SQL `LIKE` matcher with `%` (zero+ chars) and `_` (single char)
439/// wildcards. Greedy backtracking; safe on non-ASCII because we work
440/// in `char`s, not bytes.
441fn sql_like(haystack: &str, pattern: &str) -> Result<bool, FDWError> {
442    let h: Vec<char> = haystack.chars().collect();
443    let p: Vec<char> = pattern.chars().collect();
444    like_match(&h, &p)
445}
446
447fn like_match(h: &[char], p: &[char]) -> Result<bool, FDWError> {
448    let mut hi = 0;
449    let mut pi = 0;
450    let mut star_h: Option<usize> = None;
451    let mut star_p: Option<usize> = None;
452    while hi < h.len() {
453        match p.get(pi) {
454            Some('\\') => {
455                let Some(literal) = p.get(pi + 1) else {
456                    return Err(FDWError::InvalidPredicate(
457                        "LIKE pattern must not end with escape character".into(),
458                    ));
459                };
460                if *literal == h[hi] {
461                    hi += 1;
462                    pi += 2;
463                } else if let (Some(sp), Some(sh)) = (star_p, star_h) {
464                    pi = sp + 1;
465                    star_h = Some(sh + 1);
466                    hi = sh + 1;
467                } else {
468                    return Ok(false);
469                }
470            }
471            Some('%') => {
472                star_p = Some(pi);
473                star_h = Some(hi);
474                pi += 1;
475            }
476            Some('_') => {
477                hi += 1;
478                pi += 1;
479            }
480            Some(c) if *c == h[hi] => {
481                hi += 1;
482                pi += 1;
483            }
484            _ => {
485                if let (Some(sp), Some(sh)) = (star_p, star_h) {
486                    pi = sp + 1;
487                    star_h = Some(sh + 1);
488                    hi = sh + 1;
489                } else {
490                    return Ok(false);
491                }
492            }
493        }
494    }
495    while p.get(pi) == Some(&'%') {
496        pi += 1;
497    }
498    Ok(pi == p.len())
499}
500
501#[cfg(test)]
502mod tests {
503    use super::*;
504
505    fn row(pairs: &[(&str, Value)]) -> Row {
506        pairs
507            .iter()
508            .map(|(k, v)| ((*k).to_string(), v.clone()))
509            .collect()
510    }
511
512    fn books_table() -> ForeignTable {
513        ForeignTable {
514            name: "books".into(),
515            server_name: "mem".into(),
516            columns: vec![
517                ColumnDef {
518                    name: "id".into(),
519                    ty: ColumnType::Integer,
520                },
521                ColumnDef {
522                    name: "title".into(),
523                    ty: ColumnType::Text,
524                },
525                ColumnDef {
526                    name: "year".into(),
527                    ty: ColumnType::Integer,
528                },
529            ],
530            options: BTreeMap::new(),
531        }
532    }
533
534    #[test]
535    fn memory_handler_returns_all_rows_unfiltered() {
536        let mut handler = MemoryHandler::new();
537        handler.load(
538            "books",
539            vec![
540                row(&[
541                    ("id", Value::Int(1)),
542                    ("title", Value::Str("a".into())),
543                    ("year", Value::Int(2024)),
544                ]),
545                row(&[
546                    ("id", Value::Int(2)),
547                    ("title", Value::Str("b".into())),
548                    ("year", Value::Int(2023)),
549                ]),
550            ],
551        );
552        let rows = handler.scan(&books_table(), None, &[], None).unwrap();
553        assert_eq!(rows.len(), 2);
554    }
555
556    #[test]
557    fn memory_handler_does_not_treat_an_unloaded_table_as_empty() {
558        let handler = MemoryHandler::new();
559        let error = handler.scan(&books_table(), None, &[], None).unwrap_err();
560        assert!(matches!(error, FDWError::UnknownTable(name) if name == "books"));
561    }
562
563    #[test]
564    fn pushdown_predicate_filters_rows() {
565        let mut handler = MemoryHandler::new();
566        handler.load(
567            "books",
568            vec![
569                row(&[("id", Value::Int(1)), ("year", Value::Int(2024))]),
570                row(&[("id", Value::Int(2)), ("year", Value::Int(2023))]),
571                row(&[("id", Value::Int(3)), ("year", Value::Int(2024))]),
572            ],
573        );
574        let rows = handler
575            .scan(
576                &books_table(),
577                None,
578                &[FDWPredicate {
579                    column: "year".into(),
580                    operator: PredicateOp::Eq,
581                    value: Value::Int(2024),
582                }],
583                None,
584            )
585            .unwrap();
586        assert_eq!(rows.len(), 2);
587        assert!(rows
588            .iter()
589            .all(|r| r.get("year") == Some(&Value::Int(2024))));
590    }
591
592    #[test]
593    fn projection_keeps_only_requested_columns() {
594        let mut handler = MemoryHandler::new();
595        handler.load(
596            "books",
597            vec![row(&[
598                ("id", Value::Int(1)),
599                ("title", Value::Str("a".into())),
600                ("year", Value::Int(2024)),
601            ])],
602        );
603        let cols = ["title".to_string()];
604        let rows = handler
605            .scan(&books_table(), Some(&cols), &[], None)
606            .unwrap();
607        assert_eq!(rows.len(), 1);
608        assert_eq!(rows[0].len(), 1);
609        assert!(rows[0].contains_key("title"));
610        assert!(!rows[0].contains_key("id"));
611    }
612
613    #[test]
614    fn projection_represents_missing_columns_as_null() {
615        let row = row(&[("id", Value::Int(1))]);
616        let projected = project_row(&row, Some(&["missing".to_string()]));
617        assert_eq!(projected.get("missing"), Some(&Value::Null));
618    }
619
620    #[test]
621    fn malformed_in_predicate_is_an_error_and_not_an_empty_match() {
622        let row = row(&[("id", Value::Int(1))]);
623        let predicate = FDWPredicate {
624            column: "id".into(),
625            operator: PredicateOp::In,
626            value: Value::Int(1),
627        };
628        assert!(row_matches_predicates(&row, &[predicate]).is_err());
629    }
630
631    #[test]
632    fn not_like_does_not_match_missing_values_and_rejects_non_text_values() {
633        let predicate = FDWPredicate {
634            column: "title".into(),
635            operator: PredicateOp::NotLike,
636            value: Value::Str("x%".into()),
637        };
638        assert!(!row_matches_predicates(&Row::new(), std::slice::from_ref(&predicate)).unwrap());
639        assert!(row_matches_predicates(&row(&[("title", Value::Int(7))]), &[predicate]).is_err());
640    }
641
642    #[test]
643    fn null_predicates_match_sql_filter_semantics() {
644        let is_null = FDWPredicate {
645            column: "missing".into(),
646            operator: PredicateOp::Eq,
647            value: Value::Null,
648        };
649        assert!(row_matches_predicates(&Row::new(), std::slice::from_ref(&is_null)).unwrap());
650        assert!(row_matches_predicates(&row(&[("missing", Value::Null)]), &[is_null]).unwrap());
651
652        let not_equal = FDWPredicate {
653            column: "value".into(),
654            operator: PredicateOp::NotEq,
655            value: Value::Int(1),
656        };
657        assert!(!row_matches_predicates(&row(&[("value", Value::Null)]), &[not_equal]).unwrap());
658
659        let invalid_order = FDWPredicate {
660            column: "value".into(),
661            operator: PredicateOp::Lt,
662            value: Value::Null,
663        };
664        assert!(row_matches_predicates(&Row::new(), &[invalid_order]).is_err());
665    }
666
667    #[test]
668    fn limit_caps_result_count() {
669        let mut handler = MemoryHandler::new();
670        let rs: Vec<Row> = (0..10).map(|i| row(&[("id", Value::Int(i))])).collect();
671        handler.load("books", rs);
672        let rows = handler.scan(&books_table(), None, &[], Some(3)).unwrap();
673        assert_eq!(rows.len(), 3);
674    }
675
676    #[test]
677    fn like_pattern_matches_with_percent_and_underscore() {
678        assert!(sql_like("alpha", "a%a").unwrap());
679        assert!(sql_like("alpha", "alp_a").unwrap());
680        assert!(sql_like("alpha", "%pha").unwrap());
681        assert!(!sql_like("alpha", "beta").unwrap());
682        assert!(sql_like("a_b", r"a\_b").unwrap());
683        assert!(!sql_like("a", r"a\").unwrap());
684        assert!(sql_like("ab", r"a\").is_err());
685    }
686
687    #[test]
688    fn in_predicate_filters_by_membership() {
689        let mut handler = MemoryHandler::new();
690        handler.load(
691            "books",
692            vec![
693                row(&[("id", Value::Int(1))]),
694                row(&[("id", Value::Int(2))]),
695                row(&[("id", Value::Int(3))]),
696            ],
697        );
698        let rows = handler
699            .scan(
700                &books_table(),
701                None,
702                &[FDWPredicate {
703                    column: "id".into(),
704                    operator: PredicateOp::In,
705                    value: Value::List(vec![Value::Int(1), Value::Int(3)]),
706                }],
707                None,
708            )
709            .unwrap();
710        assert_eq!(rows.len(), 2);
711    }
712}
713
714pub mod arrays;