use std::collections::BTreeMap;
use async_trait::async_trait;
use fv_mac::{Bindings, Taxonomy};
use fv_value::{compile, json_to_value, Expr, Scope};
use serde_json::Value as J;
use crate::config::ConnectorConfig;
use crate::source::{Caps, LabelColumn, ScanLeg, Source};
use crate::{ConnectError, Row};
fn records_at<'a>(response: &'a J, path: &str) -> Result<&'a Vec<J>, ConnectError> {
let mut node = response;
if !path.is_empty() {
for key in path.split('.') {
node = node
.get(key)
.ok_or_else(|| ConnectError(format!("REST: recordsPath '{path}' has no key '{key}'")))?;
}
}
node.as_array()
.ok_or_else(|| ConnectError(format!("REST: records at '{path}' is not a JSON array")))
}
pub fn extract_rows(
response: &J,
records_path: &str,
fields: &BTreeMap<String, String>,
) -> Result<Vec<Row>, ConnectError> {
let records = records_at(response, records_path)?;
let compiled: Vec<(String, Expr)> = fields
.iter()
.map(|(col, expr)| {
compile(expr)
.map(|a| (col.clone(), a))
.map_err(|e| ConnectError(format!("REST: field '{col}' expr `{expr}`: {e}")))
})
.collect::<Result<_, _>>()?;
let envelope: Option<fv_value::Value> = if fields.values().any(|expr| expr.contains("e.")) {
let mut env = response.clone();
if !records_path.is_empty() {
let parts: Vec<&str> = records_path.split('.').collect();
let mut node = Some(&mut env);
for key in &parts[..parts.len() - 1] {
node = node.and_then(|n| n.get_mut(*key));
}
if let Some(obj) = node.and_then(|n| n.as_object_mut()) {
obj.remove(parts[parts.len() - 1]);
}
} else if let Some(obj) = env.as_object_mut() {
obj.clear(); }
Some(json_to_value(&env))
} else {
None
};
let mut rows = Vec::with_capacity(records.len());
for rec in records {
let mut scope = Scope::new();
scope.insert("r".into(), json_to_value(rec));
if let Some(env) = &envelope {
scope.insert("e".into(), env.clone());
}
let mut row: Row = Vec::with_capacity(compiled.len());
for (col, ast) in &compiled {
row.push((
col.clone(),
ast.eval(&scope)
.map_err(|e| ConnectError(format!("REST: field '{col}': {e}")))?,
));
}
rows.push(row);
}
Ok(rows)
}
pub struct RestSource;
fn opt_str<'a>(config: &'a ConnectorConfig, key: &str) -> Option<&'a str> {
config.options.get(key).and_then(J::as_str)
}
fn parse_fields(options: &J) -> Result<BTreeMap<String, String>, ConnectError> {
let obj = match options.get("fields") {
Some(J::Object(m)) => m.clone(),
Some(J::String(s)) => serde_json::from_str(s)
.map_err(|e| ConnectError(format!("REST: config.options.fields is not valid JSON: {e}")))?,
_ => {
return Err(ConnectError(
"REST: config.options.fields is required (object or JSON string)".into(),
))
}
};
Ok(obj
.iter()
.filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_string())))
.collect())
}
#[async_trait]
impl Source for RestSource {
fn capabilities(&self) -> Caps {
Caps {
read: true,
write: false,
}
}
fn scan_leg(&self) -> ScanLeg {
ScanLeg::Polled }
async fn read(
&self,
config: &ConnectorConfig,
_password: &str,
table: &str,
_taxonomy: &Taxonomy,
_grants: &Bindings,
_label_column: LabelColumn<'_>,
limit: Option<usize>,
) -> Result<Vec<Row>, ConnectError> {
let _ = table;
let url = opt_str(config, "url").ok_or_else(|| ConnectError("REST: config.options.url is required".into()))?;
let records_path = opt_str(config, "recordsPath").unwrap_or("");
let fields = parse_fields(&config.options)?;
let mut req = reqwest::Client::new().get(url);
for (k, v) in config_headers(&config.options) {
req = req.header(&k, &v);
}
let resp = req
.send()
.await
.map_err(|e| ConnectError(format!("REST GET {url}: {e}")))?;
if !resp.status().is_success() {
return Err(ConnectError(format!("REST GET {url} → {}", resp.status())));
}
let json: J = resp
.json()
.await
.map_err(|e| ConnectError(format!("REST parse {url}: {e}")))?;
let mut rows = extract_rows(&json, records_path, &fields)?;
if let Some(n) = limit {
rows.truncate(n);
}
Ok(rows)
}
async fn write_and_promote(
&self,
config: &ConnectorConfig,
_: &str,
_: &str,
_: &[String],
_: &[Row],
) -> Result<(), ConnectError> {
Err(ConnectError(format!(
"connector type '{}' is read-only (a REST feed is a source, not a sink)",
config.kind
)))
}
}
fn config_headers(options: &J) -> Vec<(String, String)> {
let parsed;
let obj = match options.get("headers") {
Some(J::Object(o)) => Some(o),
Some(J::String(raw)) => {
parsed = serde_json::from_str::<J>(raw).ok();
parsed.as_ref().and_then(J::as_object)
}
_ => None,
};
obj.map(|o| {
o.iter()
.filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_string())))
.collect()
})
.unwrap_or_default()
}
#[cfg(test)]
mod tests {
use super::*;
use fv_value::Value;
use serde_json::json;
#[test]
fn extracts_positional_array_records_via_the_value_dialect() {
let response = json!({ "states": [
["ac2320", "DAL2166 ", "United States", null, 1788, -118.23, 33.64, 4701.5, false, 191.3, 89.8, 8.45, null, 4907.2],
]});
let fields = BTreeMap::from([
("icao24".to_string(), "r[0]".to_string()),
("callsign".to_string(), "trim(r[1])".to_string()),
(
"position".to_string(),
"concat(toString(r[6]), ',', toString(r[5]))".to_string(),
),
("altitudeM".to_string(), "coalesce(r[13], r[7])".to_string()),
]);
let rows = extract_rows(&response, "states", &fields).unwrap();
assert_eq!(rows.len(), 1);
let row: BTreeMap<_, _> = rows[0].iter().cloned().collect();
assert_eq!(row["icao24"], Value::Str("ac2320".into()));
assert_eq!(row["callsign"], Value::Str("DAL2166".into())); assert_eq!(row["position"], Value::Str("33.64,-118.23".into())); assert_eq!(row["altitudeM"], Value::Num(4907.2)); }
#[test]
fn extracts_keyed_object_records_too() {
let response = json!({ "data": { "items": [{ "id": "s-1", "tempC": 21.5 }] } });
let fields = BTreeMap::from([
("id".to_string(), "r.id".to_string()),
("tempC".to_string(), "r.tempC".to_string()),
]);
let rows = extract_rows(&response, "data.items", &fields).unwrap();
let row: BTreeMap<_, _> = rows[0].iter().cloned().collect();
assert_eq!(row["id"], Value::Str("s-1".into()));
assert_eq!(row["tempC"], Value::Num(21.5));
}
#[test]
fn extracts_adsblol_keyed_shape_including_the_ground_string() {
let alt = "(r.alt_baro == 'ground' ? r.alt_geom : coalesce(r.alt_geom, r.alt_baro)) * 0.3048";
let fields = BTreeMap::from([
("icao24".to_string(), "r.hex".to_string()),
("callsign".to_string(), "trim(coalesce(r.flight, r.hex))".to_string()),
("altitudeM".to_string(), alt.to_string()),
("onGround".to_string(), "r.alt_baro == 'ground'".to_string()),
("velocityMps".to_string(), "coalesce(r.gs, 0) * 0.514444".to_string()),
]);
let response = json!({ "ac": [
{ "hex": "a1b2c3", "flight": "UAL123 ", "lat": 33.9, "lon": -118.4, "alt_baro": 1600, "alt_geom": null, "gs": 200, "track": 90 },
{ "hex": "d4e5f6", "flight": null, "lat": 33.7, "lon": -117.2, "alt_baro": "ground", "alt_geom": null, "gs": null },
]});
let rows = extract_rows(&response, "ac", &fields).unwrap();
let air: BTreeMap<_, _> = rows[0].iter().cloned().collect();
assert_eq!(air["icao24"], Value::Str("a1b2c3".into()));
assert_eq!(air["callsign"], Value::Str("UAL123".into()));
assert_eq!(air["altitudeM"], Value::Num(1600.0 * 0.3048)); assert_eq!(air["onGround"], Value::Bool(false));
let gnd: BTreeMap<_, _> = rows[1].iter().cloned().collect();
assert_eq!(gnd["callsign"], Value::Str("d4e5f6".into())); assert_eq!(gnd["altitudeM"], Value::Null); assert_eq!(gnd["onGround"], Value::Bool(true));
assert_eq!(gnd["velocityMps"], Value::Num(0.0)); }
#[test]
fn extracts_geojson_feature_records_with_nested_indexing() {
let fields = BTreeMap::from([
("quakeId".to_string(), "r.id".to_string()),
("magnitude".to_string(), "r.properties.mag".to_string()),
(
"position".to_string(),
"concat(toString(r.geometry.coordinates[1]), ',', toString(r.geometry.coordinates[0]))".to_string(),
),
("depthKm".to_string(), "r.geometry.coordinates[2]".to_string()),
(
"altitudeM".to_string(),
"0 - (r.geometry.coordinates[2] * 1000)".to_string(),
),
("tsunami".to_string(), "r.properties.tsunami == 1".to_string()),
]);
let response = json!({ "features": [
{ "id": "ci41539688", "properties": { "mag": 1.7, "tsunami": 0 },
"geometry": { "coordinates": [-115.4, 32.8, 10.2] } },
]});
let rows = extract_rows(&response, "features", &fields).unwrap();
let row: BTreeMap<_, _> = rows[0].iter().cloned().collect();
assert_eq!(row["quakeId"], Value::Str("ci41539688".into()));
assert_eq!(row["position"], Value::Str("32.8,-115.4".into())); assert_eq!(row["depthKm"], Value::Num(10.2));
assert_eq!(row["altitudeM"], Value::Num(-10200.0)); assert_eq!(row["tsunami"], Value::Bool(false));
}
#[test]
fn extracts_ais_shape_nulling_sentinels_and_stringifying_numeric_pks() {
let fields = BTreeMap::from([
("mmsi".to_string(), "toString(r.mmsi)".to_string()),
(
"speedMps".to_string(),
"r.properties.sog == 102.3 ? null : r.properties.sog * 0.514444".to_string(),
),
(
"headingDeg".to_string(),
"r.properties.heading == 511 ? null : r.properties.heading".to_string(),
),
("observedAt".to_string(), "r.properties.timestampExternal".to_string()),
]);
let response = json!({ "features": [
{ "mmsi": 352001492, "properties": { "sog": 11.1, "heading": 227, "timestampExternal": 1788181961818u64 } },
{ "mmsi": 230983250, "properties": { "sog": 102.3, "heading": 511, "timestampExternal": 1788181961000u64 } },
]});
let rows = extract_rows(&response, "features", &fields).unwrap();
let moving: BTreeMap<_, _> = rows[0].iter().cloned().collect();
assert_eq!(moving["mmsi"], Value::Str("352001492".into())); assert_eq!(moving["speedMps"], Value::Num(11.1 * 0.514444));
assert_eq!(moving["headingDeg"], Value::Num(227.0));
let idle: BTreeMap<_, _> = rows[1].iter().cloned().collect();
assert_eq!(idle["speedMps"], Value::Null); assert_eq!(idle["headingDeg"], Value::Null);
}
#[test]
fn extracts_omm_element_sets_with_derived_orbit_scalars() {
let semi_major = "cbrt(398600.4418 / pow(r.MEAN_MOTION * 0.00007272205216643, 2))";
let fields = BTreeMap::from([
("noradId".to_string(), "toString(r.NORAD_CAT_ID)".to_string()),
("epoch".to_string(), "epochMs(concat(r.EPOCH, 'Z'))".to_string()),
("periodMin".to_string(), "1440.0 / r.MEAN_MOTION".to_string()),
(
"apogeeKm".to_string(),
format!("{semi_major} * (1 + r.ECCENTRICITY) - 6378.137"),
),
(
"perigeeKm".to_string(),
format!("{semi_major} * (1 - r.ECCENTRICITY) - 6378.137"),
),
]);
let response = json!([
{ "NORAD_CAT_ID": 25544, "OBJECT_NAME": "ISS (ZARYA)", "EPOCH": "2026-08-30T18:23:45.123456",
"MEAN_MOTION": 15.5, "ECCENTRICITY": 0.0004 },
]);
let rows = extract_rows(&response, "", &fields).unwrap();
let row: BTreeMap<_, _> = rows[0].iter().cloned().collect();
assert_eq!(row["noradId"], Value::Str("25544".into()));
assert_eq!(row["epoch"], Value::Num(1788114225123.0)); let period = match row["periodMin"] {
Value::Num(n) => n,
_ => panic!("periodMin not numeric"),
};
assert!((period - 92.9).abs() < 0.1, "period {period}");
let apogee = match row["apogeeKm"] {
Value::Num(n) => n,
_ => panic!("apogeeKm not numeric"),
};
let perigee = match row["perigeeKm"] {
Value::Num(n) => n,
_ => panic!("perigeeKm not numeric"),
};
assert!(apogee > 400.0 && apogee < 460.0, "apogee {apogee}"); assert!(perigee > 390.0 && perigee < apogee, "perigee {perigee}");
}
#[test]
fn errors_on_missing_path_or_non_array() {
let r = json!({ "x": 1 });
assert!(extract_rows(&r, "nope", &BTreeMap::new()).is_err());
assert!(extract_rows(&r, "x", &BTreeMap::new()).is_err());
}
#[test]
fn driver_is_read_only_polled() {
assert_eq!(
RestSource.capabilities(),
Caps {
read: true,
write: false
}
);
assert_eq!(RestSource.scan_leg(), ScanLeg::Polled);
}
#[test]
fn parse_fields_accepts_object_or_json_string() {
let obj = json!({ "fields": { "a": "r[0]", "b": "r[1]" } });
assert_eq!(parse_fields(&obj).unwrap().len(), 2);
let str_form = json!({ "fields": "{\"a\":\"r[0]\",\"b\":\"r[1]\"}" });
let f = parse_fields(&str_form).unwrap();
assert_eq!(f["a"], "r[0]");
assert!(parse_fields(&json!({})).is_err()); }
#[test]
fn ingestion_read_never_touches_landings_even_when_one_exists() {
let dir = std::env::temp_dir().join(format!("fv-landing-tests-{}", std::process::id()));
let store = crate::lake::SnapshotStore::at(dir.join("curated.guard"));
store.land(&[vec![("rid".into(), Value::Str("v1".into()))]]).unwrap();
let _env = crate::ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); std::env::set_var("PG_LANDING_DIR", dir.to_str().unwrap());
let config = ConnectorConfig {
kind: "REST_POLL".into(),
host: String::new(),
port: 0,
database: String::new(),
user: String::new(),
options: json!({}),
};
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let out = rt.block_on(RestSource.read(
&config,
"",
"curated.guard",
&Taxonomy::default(),
&Bindings::new(),
&|c: &str| c.to_string(),
None,
));
assert!(out.unwrap_err().0.contains("config.options.url is required"));
std::env::remove_var("PG_LANDING_DIR"); }
#[test]
fn headers_parse_from_object_and_json_string_shapes() {
use serde_json::json;
let as_string = json!({ "headers": "{\"User-Agent\":\"fv-demo\",\"X-N\":\"1\"}" });
let mut got = config_headers(&as_string);
got.sort();
assert_eq!(
got,
vec![
("User-Agent".to_string(), "fv-demo".to_string()),
("X-N".to_string(), "1".to_string())
]
);
let as_object = json!({ "headers": { "User-Agent": "fv-demo" } });
assert_eq!(
config_headers(&as_object),
vec![("User-Agent".to_string(), "fv-demo".to_string())]
);
assert!(config_headers(&json!({})).is_empty());
assert!(config_headers(&json!({ "headers": "not-json" })).is_empty());
}
#[test]
fn mappings_can_read_the_response_envelope_as_e() {
use serde_json::json;
let response = json!({ "now": 1_000_000, "total": 2, "ac": [
{ "hex": "abc", "seen_pos": 2.0 },
{ "hex": "def", "seen_pos": 0.5 },
]});
let mut fields = BTreeMap::new();
fields.insert("icao".to_string(), "r.hex".to_string());
fields.insert("observedAt".to_string(), "e.now - r.seen_pos * 1000".to_string());
let rows = extract_rows(&response, "ac", &fields).unwrap();
let obs = |i: usize| rows[i].iter().find(|(k, _)| k == "observedAt").map(|(_, v)| v.clone());
assert_eq!(obs(0), Some(Value::Num(998_000.0)));
assert_eq!(obs(1), Some(Value::Num(999_500.0)));
let mut plain = BTreeMap::new();
plain.insert("icao".to_string(), "r.hex".to_string());
assert_eq!(extract_rows(&response, "ac", &plain).unwrap().len(), 2);
}
}