#[cfg(doctest)]
#[doc = include_str!("../README.md")]
pub struct ReadmeDoctests;
use fv_mac::{where_sql, Bindings, Taxonomy};
use fv_value::Value;
use tokio_postgres::types::Type;
use tokio_postgres::NoTls;
#[cfg(test)]
pub(crate) static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
pub mod binding;
pub mod changelog;
mod config;
mod ident;
pub mod labels;
pub mod lake;
pub mod log;
mod mysql;
pub mod rest;
pub mod scan;
pub mod source;
pub mod sqlite;
mod write;
pub use binding::{bindings_from_definitions, primary_binding, Binding};
pub use changelog::{diff_snapshots, Change, Snapshot};
pub use config::{secret_env_var, ConnectorConfig};
pub use labels::{normalize_row, LabelBinding};
pub use lake::{Checkpoint, SnapshotStore};
pub use log::{fold_compacted_log, LogMessage};
pub use mysql::read_mysql_with_mac;
pub use scan::{scan_source, CollectSink, LandingSource, MemorySource, Sink, Source};
pub use source::{register_source, registered_kinds, source_for, Caps, Source as ConnectorSource};
pub use write::{write_and_promote_mysql, write_and_promote_postgres};
pub type Row = Vec<(String, Value)>;
pub async fn read_source(
config: &ConnectorConfig,
password: &str,
table: &str,
taxonomy: &Taxonomy,
grants: &Bindings,
label_column: impl Fn(&str) -> String + Copy + Sync,
limit: Option<usize>,
) -> Result<Vec<Row>, ConnectError> {
match source_for(&config.kind) {
Some(s) if s.capabilities().read => {
s.read(config, password, table, taxonomy, grants, &label_column, limit)
.await
}
Some(_) => err(format!("connector type '{}' has no read capability", config.kind)),
None => err(format!(
"connector type '{}' has no registered source (have {})",
config.kind,
registered_kinds().join(", ")
)),
}
}
pub async fn read_source_preferring_landing(
config: &ConnectorConfig,
password: &str,
table: &str,
taxonomy: &Taxonomy,
grants: &Bindings,
label_column: impl Fn(&str) -> String + Copy + Sync,
limit: Option<usize>,
) -> Result<Vec<Row>, ConnectError> {
if !table.is_empty() {
let root = std::env::var("PG_LANDING_DIR")
.or_else(|_| std::env::var("LAKE_DIR"))
.unwrap_or_else(|_| "/tmp/fv-funnel/landing".to_string());
let store = lake::SnapshotStore::new(&root, table);
if !store.list().is_empty() {
let mut rows = store.last_raw().map_err(ConnectError)?;
if let Some(n) = limit {
rows.truncate(n);
}
return Ok(rows);
}
}
read_source(config, password, table, taxonomy, grants, label_column, limit).await
}
pub async fn write_and_promote(
config: &ConnectorConfig,
password: &str,
table: &str,
columns: &[String],
rows: &[Row],
) -> Result<(), ConnectError> {
match source_for(&config.kind) {
Some(s) if s.capabilities().write => s.write_and_promote(config, password, table, columns, rows).await,
Some(_) => err(format!("connector type '{}' has no write capability", config.kind)),
None => err(format!(
"connector type '{}' has no registered source (have {})",
config.kind,
registered_kinds().join(", ")
)),
}
}
#[derive(Debug)]
pub struct ConnectError(pub String);
impl std::fmt::Display for ConnectError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
impl std::error::Error for ConnectError {}
fn err<T>(e: impl std::fmt::Display) -> Result<T, ConnectError> {
Err(ConnectError(e.to_string()))
}
pub async fn read_with_mac(
dsn: &str,
table: &str,
taxonomy: &Taxonomy,
grants: &Bindings,
label_column: impl Fn(&str) -> String,
limit: Option<usize>,
) -> Result<Vec<Row>, ConnectError> {
let (client, connection) = match tokio_postgres::connect(dsn, NoTls).await {
Ok(pair) => pair,
Err(e) => return err(format!("connect: {e}")),
};
let handle = tokio::spawn(async move {
let _ = connection.await;
});
let predicate = where_sql(taxonomy, grants, &label_column);
let mut sql = format!("SELECT * FROM {} WHERE {predicate}", ident::pg_ident(table));
if let Some(n) = limit {
sql.push_str(&format!(" LIMIT {n}"));
}
let result = client.query(&sql, &[]).await;
drop(client);
let _ = handle.await;
let pg_rows = match result {
Ok(r) => r,
Err(e) => {
let cause = e
.as_db_error()
.map(|d| d.message().to_string())
.or_else(|| std::error::Error::source(&e).map(|s| s.to_string()))
.unwrap_or_default();
return err(format!("query `{sql}`: {e} {cause}"));
}
};
let mut out = Vec::with_capacity(pg_rows.len());
for r in &pg_rows {
let mut row: Row = Vec::with_capacity(r.columns().len());
for (i, col) in r.columns().iter().enumerate() {
row.push((col.name().to_string(), cell(r, i, col.type_())));
}
out.push(row);
}
Ok(out)
}
#[allow(clippy::too_many_arguments)] pub async fn read_series(
dsn: &str,
series_id: &str,
from: Option<i64>,
to: Option<i64>,
size: usize,
taxonomy: &Taxonomy,
grants: &Bindings,
column_for: impl Fn(&str) -> String,
) -> Result<Vec<Row>, ConnectError> {
let (client, connection) = match tokio_postgres::connect(dsn, NoTls).await {
Ok(pair) => pair,
Err(e) => return err(format!("connect: {e}")),
};
let handle = tokio::spawn(async move {
let _ = connection.await;
});
let mut clauses = vec![format!("series_id = {}", sql_str(series_id))];
if let Some(f) = from {
clauses.push(format!("ts >= {f}"));
}
if let Some(t) = to {
clauses.push(format!("ts <= {t}"));
}
clauses.push(fv_mac::pg_where(taxonomy, grants, column_for)); let sql = format!(
"SELECT ts, value_num, value_str, value_geo FROM series.observations WHERE {} ORDER BY ts ASC LIMIT {size}",
clauses.join(" AND ")
);
let result = client.query(&sql, &[]).await;
drop(client);
let _ = handle.await;
let pg_rows = match result {
Ok(r) => r,
Err(e) => return err(format!("series query `{sql}`: {e}")),
};
let mut out = Vec::with_capacity(pg_rows.len());
for r in &pg_rows {
let mut row: Row = Vec::with_capacity(r.columns().len());
for (i, col) in r.columns().iter().enumerate() {
row.push((col.name().to_string(), cell(r, i, col.type_())));
}
out.push(row);
}
Ok(out)
}
fn sql_str(s: &str) -> String {
format!("'{}'", s.replace('\'', "''"))
}
fn cell(row: &tokio_postgres::Row, i: usize, ty: &Type) -> Value {
match *ty {
Type::TEXT | Type::VARCHAR | Type::BPCHAR | Type::NAME => row
.try_get::<_, Option<String>>(i)
.ok()
.flatten()
.map(Value::Str)
.unwrap_or(Value::Null),
Type::INT2 => row
.try_get::<_, Option<i16>>(i)
.ok()
.flatten()
.map(|v| Value::Num(v as f64))
.unwrap_or(Value::Null),
Type::INT4 => row
.try_get::<_, Option<i32>>(i)
.ok()
.flatten()
.map(|v| Value::Num(v as f64))
.unwrap_or(Value::Null),
Type::INT8 => row
.try_get::<_, Option<i64>>(i)
.ok()
.flatten()
.map(|v| Value::Num(v as f64))
.unwrap_or(Value::Null),
Type::FLOAT4 => row
.try_get::<_, Option<f32>>(i)
.ok()
.flatten()
.map(|v| Value::Num(v as f64))
.unwrap_or(Value::Null),
Type::FLOAT8 => row
.try_get::<_, Option<f64>>(i)
.ok()
.flatten()
.map(Value::Num)
.unwrap_or(Value::Null),
Type::BOOL => row
.try_get::<_, Option<bool>>(i)
.ok()
.flatten()
.map(Value::Bool)
.unwrap_or(Value::Null),
_ => row
.try_get::<_, Option<String>>(i)
.ok()
.flatten()
.map(Value::Str)
.unwrap_or(Value::Null),
}
}
pub fn get<'a>(row: &'a Row, key: &str) -> &'a Value {
row.iter()
.find(|(k, _)| k == key)
.map(|(_, v)| v)
.unwrap_or(&Value::Null)
}
#[cfg(test)]
mod tee_tests {
use super::*;
use fv_value::Value;
#[test]
fn runner_read_prefers_the_latest_landing_and_falls_back_without_one() {
let dir = std::env::temp_dir().join(format!("fv-landing-tests-{}", std::process::id()));
let store = lake::SnapshotStore::at(dir.join("curated.tee-vessel"));
let row = |rid: &str, speed: f64| -> Row {
vec![
("rid".into(), Value::Str(rid.into())),
("speedMps".into(), Value::Num(speed)),
]
};
store.land(&[row("v1", 1.0)]).unwrap();
store.land(&[row("v1", 2.0), row("v2", 3.0)]).unwrap();
let _env = crate::ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); std::env::set_var("PG_LANDING_DIR", dir.to_str().unwrap());
let config = ConnectorConfig {
kind: "REST_POLL".into(),
host: String::new(),
port: 0,
database: String::new(),
user: String::new(),
options: serde_json::json!({}),
};
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let read = |table: &'static str, limit| {
rt.block_on(read_source_preferring_landing(
&config,
"",
table,
&Taxonomy::default(),
&Bindings::new(),
|c| c.to_string(),
limit,
))
};
let rows = read("curated.tee-vessel", None).unwrap();
assert_eq!(rows.len(), 2);
assert!(rows.iter().any(|r| r
.iter()
.any(|(k, v)| k == "speedMps" && matches!(v, Value::Num(n) if *n == 3.0))));
assert_eq!(read("curated.tee-vessel", Some(1)).unwrap().len(), 1);
assert!(read("curated.never-landed", None)
.unwrap_err()
.0
.contains("config.options.url is required"));
std::env::remove_var("PG_LANDING_DIR"); }
}