use crate::config::SnowflakeSourceConfig;
use crate::convert::{ColumnMeta, row_to_json};
use async_trait::async_trait;
use faucet_common_snowflake::{
SnowflakeAuth, authorization_header, credential_to_auth, snowflake_token_type,
};
use faucet_core::util::substitute_context_bind_params;
use faucet_core::{AuthSpec, FaucetError, SharedAuthProvider, Stream, StreamPage};
use reqwest::Client;
use serde::Deserialize;
use serde_json::{Map, Value, json};
use std::collections::HashMap;
use std::pin::Pin;
use std::time::Duration;
#[derive(Debug, Deserialize)]
struct StatementResponse {
#[serde(default)]
code: Option<String>,
#[serde(default)]
message: Option<String>,
#[serde(rename = "statementHandle", default)]
statement_handle: Option<String>,
#[serde(rename = "resultSetMetaData", default)]
result_set_metadata: Option<ResultSetMetadata>,
#[serde(default)]
data: Option<Vec<Vec<Value>>>,
}
#[derive(Debug, Deserialize)]
struct ResultSetMetadata {
#[serde(rename = "rowType", default)]
row_type: Vec<ColumnMeta>,
#[serde(rename = "partitionInfo", default)]
partition_info: Vec<PartitionInfo>,
}
#[derive(Debug, Deserialize)]
#[allow(dead_code)] struct PartitionInfo {
#[serde(rename = "rowCount", default)]
row_count: u64,
}
pub struct SnowflakeSource {
config: SnowflakeSourceConfig,
client: Client,
endpoint_base: Option<String>,
auth_provider: Option<SharedAuthProvider>,
}
impl SnowflakeSource {
pub fn new(config: SnowflakeSourceConfig) -> Result<Self, FaucetError> {
faucet_core::validate_batch_size(config.batch_size)?;
Ok(Self {
config,
client: Client::new(),
endpoint_base: None,
auth_provider: None,
})
}
pub fn with_auth_provider(mut self, provider: SharedAuthProvider) -> Self {
self.auth_provider = Some(provider);
self
}
pub fn with_endpoint_base(mut self, base: impl Into<String>) -> Self {
self.endpoint_base = Some(base.into());
self
}
fn base_url(&self) -> String {
match &self.endpoint_base {
Some(b) => b.trim_end_matches('/').to_owned(),
None => format!("https://{}.snowflakecomputing.com", self.config.account),
}
}
fn statements_url(&self) -> String {
format!("{}/api/v2/statements", self.base_url())
}
fn partition_url(&self, handle: &str, partition: usize) -> String {
format!(
"{}/api/v2/statements/{}?partition={}",
self.base_url(),
handle,
partition
)
}
async fn resolve_auth(&self) -> Result<SnowflakeAuth, FaucetError> {
if let Some(p) = &self.auth_provider {
return credential_to_auth(p.credential().await?);
}
match &self.config.auth {
AuthSpec::Inline(a) => Ok(a.clone()),
AuthSpec::Reference(r) => Err(FaucetError::Auth(format!(
"auth references provider '{}' but no provider was supplied",
r.name
))),
}
}
fn build_request_body(&self, bindings: &[Value]) -> Value {
let mut body = json!({
"statement": self.config.query,
"timeout": self.config.statement_timeout.as_secs(),
"database": self.config.database,
"schema": self.config.schema,
"warehouse": self.config.warehouse,
});
if let Some(role) = &self.config.role {
body["role"] = json!(role);
}
if !bindings.is_empty() {
let mut map = Map::with_capacity(bindings.len());
for (i, v) in bindings.iter().enumerate() {
let (ty, value) = match v {
Value::Null => ("TEXT", Value::Null),
Value::Bool(b) => ("BOOLEAN", Value::String(b.to_string())),
Value::Number(n) => {
let ty = if n.is_i64() || n.is_u64() {
"FIXED"
} else {
"REAL"
};
(ty, Value::String(n.to_string()))
}
Value::String(s) => ("TEXT", Value::String(s.clone())),
other => ("TEXT", Value::String(other.to_string())),
};
map.insert((i + 1).to_string(), json!({"type": ty, "value": value}));
}
body["bindings"] = Value::Object(map);
}
body
}
fn resolve_query(&self, context: &HashMap<String, Value>) -> (String, Vec<Value>) {
let mut bindings = self.config.params.clone();
let (rewritten, context_values) = if context.is_empty() {
(self.config.query.clone(), Vec::new())
} else {
substitute_context_bind_params(&self.config.query, context, bindings.len() + 1, |_| {
"?".to_string()
})
};
bindings.extend(context_values);
(rewritten, bindings)
}
async fn submit_statement(
&self,
context: &HashMap<String, Value>,
) -> Result<StatementResponse, FaucetError> {
let (query, bindings) = self.resolve_query(context);
let mut body = self.build_request_body(&bindings);
body["statement"] = Value::String(query);
let url = self.statements_url();
let effective = self.resolve_auth().await?;
let auth = authorization_header(&effective, &self.config.account)?;
let token_type = snowflake_token_type(&effective);
let resp = self
.client
.post(&url)
.header("Authorization", &auth)
.header("Content-Type", "application/json")
.header("Accept", "application/json")
.header("X-Snowflake-Authorization-Token-Type", token_type)
.json(&body)
.send()
.await
.map_err(|e| FaucetError::Source(format!("Snowflake request failed: {e}")))?;
let status = resp.status();
let async_pending = status.as_u16() == 202;
if !status.is_success() && !async_pending {
let text = resp.text().await.unwrap_or_default();
return Err(FaucetError::Source(format!(
"Snowflake SQL API returned HTTP {status}: {text}"
)));
}
let parsed: StatementResponse = resp
.json()
.await
.map_err(|e| FaucetError::Source(format!("failed to parse Snowflake response: {e}")))?;
if async_pending {
let handle = parsed.statement_handle.clone().ok_or_else(|| {
FaucetError::Source(
"Snowflake returned 202 without a statementHandle to poll".into(),
)
})?;
self.poll_until_ready(&handle, &auth, token_type).await
} else {
check_code(&parsed)?;
Ok(parsed)
}
}
async fn poll_until_ready(
&self,
handle: &str,
auth: &str,
token_type: &'static str,
) -> Result<StatementResponse, FaucetError> {
let url = format!("{}/api/v2/statements/{}", self.base_url(), handle);
let poll_timeout = self.config.poll_timeout;
let started = std::time::Instant::now();
loop {
let resp = self
.client
.get(&url)
.header("Authorization", auth)
.header("Accept", "application/json")
.header("X-Snowflake-Authorization-Token-Type", token_type)
.send()
.await
.map_err(|e| FaucetError::Source(format!("Snowflake poll request failed: {e}")))?;
let status = resp.status();
if status.as_u16() == 202 {
if !poll_timeout.is_zero() && started.elapsed() >= poll_timeout {
return Err(FaucetError::Source(format!(
"Snowflake statement '{handle}' did not finish within poll_timeout ({}s); still HTTP 202",
poll_timeout.as_secs()
)));
}
tokio::time::sleep(Duration::from_millis(500)).await;
continue;
}
if !status.is_success() {
let text = resp.text().await.unwrap_or_default();
return Err(FaucetError::Source(format!(
"Snowflake poll returned HTTP {status}: {text}"
)));
}
let parsed: StatementResponse = resp.json().await.map_err(|e| {
FaucetError::Source(format!("failed to parse Snowflake poll response: {e}"))
})?;
check_code(&parsed)?;
return Ok(parsed);
}
}
async fn fetch_partition(
&self,
handle: &str,
partition: usize,
auth: &str,
token_type: &'static str,
) -> Result<Vec<Vec<Value>>, FaucetError> {
let url = self.partition_url(handle, partition);
let resp = self
.client
.get(&url)
.header("Authorization", auth)
.header("Accept", "application/json")
.header("X-Snowflake-Authorization-Token-Type", token_type)
.send()
.await
.map_err(|e| {
FaucetError::Source(format!(
"Snowflake partition fetch failed (partition {partition}): {e}"
))
})?;
let status = resp.status();
if !status.is_success() {
let text = resp.text().await.unwrap_or_default();
return Err(FaucetError::Source(format!(
"Snowflake partition fetch returned HTTP {status} (partition {partition}): {text}"
)));
}
let parsed: StatementResponse = resp.json().await.map_err(|e| {
FaucetError::Source(format!(
"failed to parse Snowflake partition response (partition {partition}): {e}"
))
})?;
check_code(&parsed)?;
Ok(parsed.data.unwrap_or_default())
}
}
fn check_code(resp: &StatementResponse) -> Result<(), FaucetError> {
if let Some(code) = &resp.code
&& code != "090001"
{
return Err(FaucetError::Source(format!(
"Snowflake error {}: {}",
code,
resp.message.clone().unwrap_or_default()
)));
}
Ok(())
}
#[async_trait]
impl faucet_core::Source for SnowflakeSource {
fn connector_name(&self) -> &'static str {
"snowflake"
}
fn config_schema(&self) -> Value {
serde_json::to_value(faucet_core::schema_for!(SnowflakeSourceConfig))
.expect("schema serialization")
}
fn dataset_uri(&self) -> String {
format!(
"snowflake://{}/{}/{}?query={}",
self.config.account, self.config.database, self.config.schema, self.config.query
)
}
async fn check(
&self,
ctx: &faucet_core::check::CheckContext,
) -> Result<faucet_core::check::CheckReport, FaucetError> {
use faucet_core::check::{CheckReport, Probe};
let start = std::time::Instant::now();
let probe = async {
let mut body = self.build_request_body(&[]);
body["statement"] = Value::String("SELECT 1".to_string());
let url = self.statements_url();
let effective = self.resolve_auth().await.map_err(|e| {
Probe::fail_hint(
"auth",
start.elapsed(),
e.to_string(),
"verify the Snowflake credentials / shared auth provider",
)
})?;
let auth = authorization_header(&effective, &self.config.account)
.map_err(|e| Probe::fail("auth", start.elapsed(), e.to_string()))?;
let token_type = snowflake_token_type(&effective);
let resp = self
.client
.post(&url)
.header("Authorization", &auth)
.header("Content-Type", "application/json")
.header("Accept", "application/json")
.header("X-Snowflake-Authorization-Token-Type", token_type)
.json(&body)
.send()
.await
.map_err(|e| {
Probe::fail_hint(
"query",
start.elapsed(),
format!("Snowflake request failed: {e}"),
"verify the account endpoint is reachable",
)
})?;
let status = resp.status();
if status.is_success() || status.as_u16() == 202 {
Ok::<Probe, Probe>(Probe::pass("query", start.elapsed()))
} else {
let text = resp.text().await.unwrap_or_default();
Err(Probe::fail_hint(
"query",
start.elapsed(),
format!("Snowflake SQL API returned HTTP {status}: {text}"),
"verify credentials, warehouse, database/schema, and role",
))
}
};
let probe = match tokio::time::timeout(ctx.timeout, probe).await {
Ok(Ok(p)) | Ok(Err(p)) => p,
Err(_elapsed) => Probe::fail_hint(
"query",
start.elapsed(),
"Snowflake probe timed out",
"Snowflake did not respond within the check timeout",
),
};
Ok(CheckReport::single(probe))
}
async fn fetch_with_context(
&self,
context: &HashMap<String, Value>,
) -> Result<Vec<Value>, FaucetError> {
let initial = self.submit_statement(context).await?;
let columns = initial
.result_set_metadata
.as_ref()
.map(|m| m.row_type.clone())
.unwrap_or_default();
if columns.is_empty() {
tracing::info!(
rows = 0,
query = %self.config.query,
"Snowflake source fetch returned no schema (likely no rows)",
);
return Ok(Vec::new());
}
let mut rows: Vec<Value> = initial
.data
.unwrap_or_default()
.iter()
.map(|r| row_to_json(r, &columns))
.collect();
let partition_count = initial
.result_set_metadata
.as_ref()
.map(|m| m.partition_info.len())
.unwrap_or(0);
if partition_count > 1 {
let handle = initial.statement_handle.ok_or_else(|| {
FaucetError::Source(
"Snowflake reported >1 partition without a statementHandle to fetch them"
.into(),
)
})?;
let effective = self.resolve_auth().await?;
let auth = authorization_header(&effective, &self.config.account)?;
let token_type = snowflake_token_type(&effective);
for i in 1..partition_count {
let raw = self.fetch_partition(&handle, i, &auth, token_type).await?;
for r in raw {
rows.push(row_to_json(&r, &columns));
}
}
}
tracing::info!(
rows = rows.len(),
query = %self.config.query,
"Snowflake source fetch complete",
);
Ok(rows)
}
fn stream_pages<'a>(
&'a self,
context: &'a HashMap<String, Value>,
_batch_size: usize,
) -> Pin<Box<dyn Stream<Item = Result<StreamPage, FaucetError>> + Send + 'a>> {
let batch_size = self.config.batch_size;
Box::pin(async_stream::try_stream! {
let initial = self.submit_statement(context).await?;
let columns = initial
.result_set_metadata
.as_ref()
.map(|m| m.row_type.clone())
.unwrap_or_default();
if columns.is_empty() {
return;
}
let partition_count = initial
.result_set_metadata
.as_ref()
.map(|m| m.partition_info.len())
.unwrap_or(1);
let chunk = if batch_size == 0 { usize::MAX } else { batch_size };
let initial_capacity = if batch_size == 0 {
initial
.result_set_metadata
.as_ref()
.map(|m| m.partition_info.iter().map(|p| p.row_count as usize).sum())
.unwrap_or(1024)
} else {
batch_size
};
let mut buffer: Vec<Value> = Vec::with_capacity(initial_capacity);
let mut total = 0usize;
for raw in initial.data.unwrap_or_default() {
buffer.push(row_to_json(&raw, &columns));
if buffer.len() >= chunk {
let page = std::mem::replace(&mut buffer, Vec::with_capacity(initial_capacity));
total += page.len();
yield StreamPage { records: page, bookmark: None };
}
}
if partition_count > 1 {
let handle = initial.statement_handle.ok_or_else(|| {
FaucetError::Source(
"Snowflake reported >1 partition without a statementHandle".into(),
)
})?;
let effective = self.resolve_auth().await?;
let auth = authorization_header(&effective, &self.config.account)?;
let token_type = snowflake_token_type(&effective);
for i in 1..partition_count {
let raw = self.fetch_partition(&handle, i, &auth, token_type).await?;
for r in raw {
buffer.push(row_to_json(&r, &columns));
if buffer.len() >= chunk {
let page = std::mem::replace(
&mut buffer,
Vec::with_capacity(initial_capacity),
);
total += page.len();
yield StreamPage { records: page, bookmark: None };
}
}
}
}
if !buffer.is_empty() {
total += buffer.len();
yield StreamPage { records: buffer, bookmark: None };
}
tracing::info!(
rows = total,
batch_size,
query = %self.config.query,
"Snowflake source stream complete",
);
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::SnowflakeAuth;
fn cfg() -> SnowflakeSourceConfig {
SnowflakeSourceConfig::new(
"xy12345.us-east-1",
"WH",
"DB",
"PUBLIC",
SnowflakeAuth::OAuth { token: "t".into() },
"SELECT 1",
)
}
#[test]
fn new_rejects_out_of_range_batch_size() {
let mut config = cfg();
config.batch_size = faucet_core::MAX_BATCH_SIZE + 1;
match SnowflakeSource::new(config) {
Err(FaucetError::Config(m)) => assert!(m.contains("batch_size"), "got: {m}"),
_ => panic!("expected a batch_size Config error"),
}
}
#[test]
fn statements_url_uses_account_when_no_override() {
let src = SnowflakeSource::new(cfg()).unwrap();
assert_eq!(
src.statements_url(),
"https://xy12345.us-east-1.snowflakecomputing.com/api/v2/statements"
);
}
#[test]
fn statements_url_uses_endpoint_override() {
let src = SnowflakeSource::new(cfg())
.unwrap()
.with_endpoint_base("http://127.0.0.1:9999");
assert_eq!(
src.statements_url(),
"http://127.0.0.1:9999/api/v2/statements"
);
}
#[test]
fn partition_url_includes_handle_and_index() {
let src = SnowflakeSource::new(cfg())
.unwrap()
.with_endpoint_base("http://srv");
assert_eq!(
src.partition_url("abc-123", 2),
"http://srv/api/v2/statements/abc-123?partition=2"
);
}
#[test]
fn build_request_body_minimal() {
let src = SnowflakeSource::new(cfg()).unwrap();
let body = src.build_request_body(&[]);
assert_eq!(body["statement"], "SELECT 1");
assert_eq!(body["timeout"], 60);
assert_eq!(body["database"], "DB");
assert_eq!(body["schema"], "PUBLIC");
assert_eq!(body["warehouse"], "WH");
assert!(body.get("bindings").is_none());
assert!(body.get("role").is_none());
}
#[test]
fn build_request_body_includes_role_when_set() {
let mut c = cfg();
c.role = Some("ANALYST".into());
let src = SnowflakeSource::new(c).unwrap();
let body = src.build_request_body(&[]);
assert_eq!(body["role"], "ANALYST");
}
#[test]
fn build_request_body_infers_binding_types() {
let src = SnowflakeSource::new(cfg()).unwrap();
let body = src.build_request_body(&[
Value::String("alice".into()),
json!(42),
json!(true),
json!(3.5),
]);
let b = &body["bindings"];
assert_eq!(b["1"]["type"], "TEXT");
assert_eq!(b["1"]["value"], "alice");
assert_eq!(b["2"]["type"], "FIXED");
assert_eq!(b["2"]["value"], "42");
assert_eq!(b["3"]["type"], "BOOLEAN");
assert_eq!(b["3"]["value"], "true");
assert_eq!(b["4"]["type"], "REAL");
assert_eq!(b["4"]["value"], "3.5");
}
#[test]
fn build_request_body_array_and_object_bindings_fall_back_to_text() {
let src = SnowflakeSource::new(cfg()).unwrap();
let body = src.build_request_body(&[json!([1, 2, 3]), json!({"k": "v"})]);
let b = &body["bindings"];
assert_eq!(b["1"]["type"], "TEXT");
assert_eq!(b["1"]["value"], "[1,2,3]");
assert_eq!(b["2"]["type"], "TEXT");
assert_eq!(b["2"]["value"], r#"{"k":"v"}"#);
}
#[test]
fn connector_name_is_snowflake() {
use faucet_core::Source;
let src = SnowflakeSource::new(cfg()).unwrap();
assert_eq!(src.connector_name(), "snowflake");
}
#[test]
fn config_schema_reports_required_fields() {
use faucet_core::Source;
let src = SnowflakeSource::new(cfg()).unwrap();
let schema = src.config_schema();
assert!(schema["properties"]["account"].is_object());
assert!(schema["properties"]["query"].is_object());
let required = schema["required"].as_array().expect("required array");
assert!(required.iter().any(|v| v == "account"));
assert!(required.iter().any(|v| v == "query"));
}
#[test]
fn build_request_body_null_binding_preserves_positional_alignment() {
let src = SnowflakeSource::new(cfg()).unwrap();
let body = src.build_request_body(&[Value::Null, json!(42)]);
let b = &body["bindings"];
assert_eq!(b["1"]["type"], "TEXT");
assert_eq!(
b["1"]["value"],
Value::Null,
"position 1 must be a NULL binding"
);
assert_eq!(b["2"]["value"], "42", "position 2 must still be 42");
}
#[test]
fn resolve_query_with_no_context_returns_input() {
let src = SnowflakeSource::new(cfg().with_params(vec![json!(7)])).unwrap();
let (q, binds) = src.resolve_query(&HashMap::new());
assert_eq!(q, "SELECT 1");
assert_eq!(binds, vec![json!(7)]);
}
#[test]
fn resolve_query_substitutes_context_with_question_mark_markers() {
let mut c = cfg();
c.query = "SELECT * FROM t WHERE id = {parent.id}".into();
let src = SnowflakeSource::new(c).unwrap();
let mut ctx = HashMap::new();
ctx.insert("parent.id".to_string(), json!(7));
let (q, binds) = src.resolve_query(&ctx);
assert_eq!(q, "SELECT * FROM t WHERE id = ?");
assert_eq!(binds, vec![json!(7)]);
}
#[test]
fn dataset_uri_includes_account_db_schema_and_query() {
use faucet_core::Source;
let src = SnowflakeSource::new(cfg()).unwrap();
assert_eq!(
src.dataset_uri(),
"snowflake://xy12345.us-east-1/DB/PUBLIC?query=SELECT 1"
);
}
}