use futures_util::{StreamExt, TryStreamExt};
use globset::GlobSet;
use crate::postgres_source::models::SourceTable;
use crate::snowflake_source::_helpers::catalog::{
catalog_tables, described_columns, listed_tables, table_matcher,
};
use crate::snowflake_source::_helpers::type_map::quote_identifier;
use crate::snowflake_source::_helpers::unload_query::subquery;
use crate::snowflake_source::constants::{
CATALOG_SQL, DESCRIBED_TABLES_MAX, PARALLEL_DESCRIBES, SHOW_TABLES_SQL, SHOW_VIEWS_SQL,
};
use crate::snowflake_source::errors::SnowflakeSourceError;
use crate::snowflake_source::models::SnowflakeReader;
use crate::sql_api::main::execute_script_rows::execute_script_rows;
use crate::sql_api::main::execute_sql::execute_sql;
pub async fn read_catalog(
reader: &SnowflakeReader,
(include, exclude): (&[String], &[String]),
queries: &[(String, String)],
) -> Result<Vec<SourceTable>, SnowflakeSourceError> {
let catalog = async {
match include.is_empty() {
true => Ok(Vec::new()),
false => {
matching_tables(reader, &table_matcher(include)?, &table_matcher(exclude)?).await
}
}
};
let described = futures_util::stream::iter(queries)
.map(|(name, sql)| query_table(reader, name, sql))
.buffered(PARALLEL_DESCRIBES)
.try_collect::<Vec<SourceTable>>();
let (mut tables, described) = tokio::try_join!(catalog, described)?;
tables.extend(described);
Ok(tables)
}
async fn matching_tables(
reader: &SnowflakeReader,
include: &GlobSet,
exclude: &GlobSet,
) -> Result<Vec<SourceTable>, SnowflakeSourceError> {
let (shown_tables, shown_views) = tokio::try_join!(
execute_sql(&reader.sql, SHOW_TABLES_SQL),
execute_sql(&reader.sql, SHOW_VIEWS_SQL)
)?;
let listed = match listed_tables(&[shown_tables, shown_views], include, exclude) {
Some(listed) if listed.len() <= DESCRIBED_TABLES_MAX => listed,
_ => {
let rows = execute_sql(&reader.sql, CATALOG_SQL).await?;
return Ok(catalog_tables(&rows, include, exclude));
}
};
futures_util::stream::iter(listed)
.map(|mut table| async move {
let statement = format!(
"DESCRIBE TABLE {}.{}",
quote_identifier(&table.schema),
quote_identifier(&table.name)
);
table.columns = described_columns(&execute_sql(&reader.sql, &statement).await?);
match table.columns.is_empty() {
true => Err(SnowflakeSourceError::Invalid(format!(
"{}.{} has no columns, as for a view whose underlying table was dropped",
table.schema, table.name
))),
false => Ok(table),
}
})
.buffered(PARALLEL_DESCRIBES)
.try_collect()
.await
}
async fn query_table(
reader: &SnowflakeReader,
name: &str,
sql: &str,
) -> Result<SourceTable, SnowflakeSourceError> {
let statements = vec![
format!("SELECT * FROM {} LIMIT 0", subquery(sql)),
"DESCRIBE RESULT LAST_QUERY_ID()".to_owned(),
];
let described = execute_script_rows(&reader.sql, &statements)
.await
.map_err(|error| SnowflakeSourceError::Invalid(format!("query table {name}: {error}")))?;
Ok(SourceTable {
schema: String::new(),
name: name.to_owned(),
columns: described_columns(&described.last().cloned().unwrap_or_default()),
primary_key: Vec::new(),
indexed_columns: Vec::new(),
estimated_rows: -1,
estimated_bytes: -1,
comment: None,
listed_keys: Vec::new(),
})
}