use crate::adapters::contract::models::{TableName, WriteMode, WriteTarget};
use crate::auth::models::Authenticator;
use crate::column_types::models::ColumnType;
use crate::config::models::{AuthConfig, DestinationConfig};
use crate::file_store::models::OpenFolder;
use crate::files_source::models::{
BatchFooters, DataFile, FilesEncoding, FilesRead, SourceFormat, StoredRead,
};
use crate::loader::errors::LoaderError;
use crate::loader::models::TableSpec;
use crate::parquet_parts::models::PartWriter;
use crate::project::models::MaskMethod;
use crate::schema_plan::models::{DesiredColumn, LiveColumn};
use crate::sql_api::models::SqlClient;
use crate::stage_api::models::DriverSession;
use crate::stage_api::models::StageInfo;
use crate::store_paths::models::Location;
use crate::streaming_api::models::StreamingClient;
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
#[derive(Debug, Clone)]
pub struct SnowflakeDestination {
pub sql: SqlClient,
pub driver: Arc<DriverSession>,
pub http: reqwest::Client,
pub database: String,
pub ddl: Arc<tokio::sync::Mutex<()>>,
pub stage: ForcedStage,
pub stage_checks: Arc<Mutex<BTreeMap<String, Result<bool, StepFailure>>>>,
pub auth: Arc<Authenticator>,
pub config: DestinationConfig,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ForcedStage {
User,
Table,
Streaming,
Insert,
Internal { name: String },
External(ExternalStage),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ExternalStage {
pub location: Location,
pub object: StageObject,
pub home: TableName,
pub keep_staged_files: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum StageObject {
Managed { integration: String },
Existing { name: String },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum StageRoute {
External(ExternalStage),
Internal {
name: String,
},
User,
TargetTable,
StagingTable,
Streaming,
Insert,
}
#[derive(Debug, Clone)]
pub struct SnowflakeAccount {
pub account: String,
pub user: String,
pub auth: AuthConfig,
pub role: Option<String>,
pub warehouse: Option<String>,
pub database: String,
pub base_url: Option<String>,
pub stage: ForcedStage,
}
#[derive(Debug, Clone, Copy)]
pub struct CopyStaged<'a> {
pub table: &'a TableName,
pub stage: &'a str,
pub folder: &'a str,
pub names: &'a [String],
pub selected: Option<&'a [DesiredColumn]>,
}
#[derive(Debug, Clone, Copy)]
pub struct StagedFile<'a> {
pub stage: &'a str,
pub key: &'a str,
pub body: &'a [u8],
}
#[derive(Debug, Clone, Copy, Default)]
pub struct SnowflakeSql;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CopyValue {
File {
file_type: Option<ColumnType>,
stored: StoredRead,
short: bool,
},
Partition,
SourceFile,
SourceVersion,
SourceRow,
Literal(String),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CopyColumn {
pub name: String,
pub column_type: ColumnType,
pub value: CopyValue,
}
#[derive(Debug, Clone, Copy)]
pub struct DirectCopy<'a> {
pub target: &'a TableName,
pub stage: &'a str,
pub folder: &'a str,
pub filename_prefix: &'a str,
pub source: &'a Location,
pub files: &'a [DataFile],
pub columns: &'a [CopyColumn],
pub encoding: &'a FilesEncoding,
pub inferred: &'a [String],
pub headers: &'a std::collections::BTreeMap<String, Vec<String>>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CopyBatch {
pub sql: String,
pub keys: Vec<String>,
}
#[derive(Debug, Clone, Copy)]
pub struct DirectCheck<'a> {
pub stage_url: Option<&'a Location>,
pub source: &'a Location,
pub format: SourceFormat,
pub files: &'a [DataFile],
pub columns: &'a [CopyColumn],
pub refused: &'a [(String, String)],
pub masks: &'a [(String, MaskMethod)],
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StageLookup {
pub query: String,
pub name: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ShownStage {
pub name: String,
pub url: String,
pub storage_integration: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PlannedStage {
pub name: String,
pub url: String,
pub statements: Vec<String>,
}
#[derive(Debug, Clone, Copy)]
pub struct RouteRequest<'a> {
pub destination: &'a TableName,
pub table_exists: bool,
pub direct_append: bool,
pub snowflake: &'a SnowflakeDestination,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum StepFailure {
Unusable(String),
Failed(String),
}
#[derive(Debug)]
pub struct RowStream {
pub client: StreamingClient,
pub pipe: String,
pub channel: String,
pub continuation: String,
pub offset: i64,
}
#[derive(Debug)]
pub struct SnowflakeWriter {
pub destination: SnowflakeDestination,
pub route: Option<StageRoute>,
pub external_folder: Option<OpenFolder>,
pub stream: Option<RowStream>,
pub parts: PartWriter,
pub staged: Vec<String>,
pub stage: Option<StageInfo>,
pub uploads: tokio::task::JoinSet<Result<(), LoaderError>>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CopyPlace {
pub stage: String,
pub folder: String,
pub filename_prefix: String,
pub uploaded: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DirectLoaded {
Committed(u64),
Staged(u64),
}
#[derive(Debug, Clone, Copy)]
pub struct DirectBatch<'a> {
pub spec: &'a TableSpec,
pub read: &'a FilesRead,
pub write_columns: &'a [DesiredColumn],
pub target: WriteTarget<'a>,
pub loaded_at: &'a str,
pub files: &'a [DataFile],
pub footers: &'a BatchFooters,
pub mode: &'a WriteMode,
pub live: Option<&'a [LiveColumn]>,
pub source_folder: Option<&'a OpenFolder>,
}
#[derive(Debug)]
pub(in crate::adapters) struct HashedScript {
pub(in crate::adapters) create: String,
pub(in crate::adapters) copies: Vec<String>,
pub(in crate::adapters) write: Vec<String>,
pub(in crate::adapters) drop_staging: String,
pub(in crate::adapters) create_delta: String,
pub(in crate::adapters) hash_into_delta: Vec<String>,
pub(in crate::adapters) finish: Vec<String>,
pub(in crate::adapters) drop_delta: String,
}