use futures_util::{StreamExt, TryStreamExt, stream};
use globset::{GlobBuilder, GlobSet, GlobSetBuilder};
use serde_json::Value;
use crate::bigquery_api::main::list_datasets::list_datasets;
use crate::bigquery_api::main::list_tables::list_tables;
use crate::bigquery_api::main::table_metadata::table_metadata;
use crate::bigquery_source::_helpers::type_map::source_column;
use crate::bigquery_source::constants::{READ_API_TABLES, TABLES_AT_ONCE};
use crate::bigquery_source::errors::BigQuerySourceError;
use crate::bigquery_source::models::BigQueryReader;
use crate::postgres_source::models::{SourceColumn, SourceTable};
pub(crate) async fn catalog_tables(
reader: &BigQueryReader,
(include, exclude): (&[String], &[String]),
) -> Result<Vec<SourceTable>, BigQuerySourceError> {
let included = matcher(include)?;
let excluded = matcher(exclude)?;
let datasets = matcher(
&include
.iter()
.map(|pattern| {
pattern
.split_once('.')
.map_or(pattern.as_str(), |(dataset, _)| dataset)
.to_owned()
})
.collect::<Vec<String>>(),
)?;
let mut names: Vec<(String, String)> = Vec::new();
for dataset in list_datasets(&reader.client, &reader.project).await? {
if !datasets.is_match(&dataset) {
continue;
}
for (table, _) in list_tables(&reader.client, (&reader.project, &dataset)).await? {
let qualified = format!("{dataset}.{table}");
if included.is_match(&qualified) && !excluded.is_match(&qualified) {
names.push((dataset.clone(), table));
}
}
}
names.sort();
stream::iter(names)
.map(|name| described(reader, name))
.buffered(TABLES_AT_ONCE)
.try_filter_map(|table| async move { Ok(table) })
.try_collect()
.await
}
async fn described(
reader: &BigQueryReader,
(dataset, table): (String, String),
) -> Result<Option<SourceTable>, BigQuerySourceError> {
let found = table_metadata(&reader.client, (&reader.project, &dataset, &table)).await?;
Ok(found.map(|found| listed_table((dataset, table), &found)))
}
fn matcher(patterns: &[String]) -> Result<GlobSet, BigQuerySourceError> {
let mut builder = GlobSetBuilder::new();
for pattern in patterns {
let glob = GlobBuilder::new(pattern).build().map_err(|error| {
BigQuerySourceError::Invalid(format!("invalid table pattern {pattern:?}: {error}"))
})?;
builder.add(glob);
}
builder
.build()
.map_err(|error| BigQuerySourceError::Invalid(error.to_string()))
}
pub(crate) fn listed_table((dataset, table): (String, String), found: &Value) -> SourceTable {
let mut columns: Vec<SourceColumn> = Vec::new();
for field in found["schema"]["fields"].as_array().into_iter().flatten() {
columns.push(source_column(field));
}
let mut primary_key: Vec<String> = Vec::new();
for column in found["tableConstraints"]["primaryKey"]["columns"]
.as_array()
.into_iter()
.flatten()
{
primary_key.extend(column.as_str().map(str::to_owned));
}
let direct = READ_API_TABLES.contains(&found["type"].as_str().unwrap_or_default());
SourceTable {
schema: dataset,
name: table,
columns,
primary_key,
indexed_columns: Vec::new(),
estimated_rows: count(&found["numRows"]),
estimated_bytes: match direct {
true => count(&found["numBytes"]).max(0),
false => -1,
},
comment: found["description"].as_str().map(str::to_owned),
listed_keys: Vec::new(),
}
}
fn count(value: &Value) -> i64 {
match value.as_str().map(str::parse::<i64>) {
Some(Ok(number)) => number,
_ => -1,
}
}