use std::collections::BTreeMap;
use toml::{Table, Value};
use crate::loader::constants::PRIMARY_KEY_PLACEHOLDER;
use crate::copy::_helpers::endpoints::endpoint;
use crate::copy::constants::COPY_PREFIX;
use crate::copy::models::{CopyProject, CopyRequest, Endpoint, Role};
const FILE_KINDS: &[&str] = &["s3", "filesystem"];
pub(crate) fn project(
request: &CopyRequest,
connections: &Table,
taken: &[String],
) -> Result<CopyProject, String> {
let mut source = endpoint(&request.source, Role::Source, connections)?;
let mut destination = endpoint(&request.destination, Role::Destination, connections)?;
let table = source_table(&source)?;
let base = free_base(&format!("{COPY_PREFIX}{}", identifier(&table)), taken);
for (end, role) in [
(&mut source, Role::Source),
(&mut destination, Role::Destination),
] {
if !end.named {
end.connection = format!("{base}_{}", role.label());
}
}
let (target, schema, name) = target(&destination, &table)?;
let mut source_table = source_settings(request, &source, schema, &name)?;
source_table.insert("destination".to_owned(), text_value(&base));
let document = |with_named: bool| {
let mut text = Table::new();
text.insert(
"connections".to_owned(),
Value::Table(connection_tables([&source, &destination], with_named)),
);
text.insert(
"targets".to_owned(),
Value::Table(single(&base, target.clone())),
);
text.insert(
"sources".to_owned(),
Value::Table(single(&base, source_table.clone())),
);
toml::to_string(&text).map_err(|error| format!("cannot write the copy's project: {error}"))
};
let variables: BTreeMap<String, String> = [&source.secret, &destination.secret]
.into_iter()
.flatten()
.cloned()
.collect();
let warnings: Vec<String> = [&source, &destination]
.iter()
.filter(|end| end.secret.is_some())
.map(|end| format!("{} has a literal password; use ${{ENV:NAME}} in the URI so it stays out of shell history", end.shown))
.collect();
Ok(CopyProject {
text: document(true)?,
printed: document(false)?,
variables,
warnings,
})
}
fn free_base(base: &str, taken: &[String]) -> String {
let used = |name: &str| {
[
name.to_owned(),
format!("{name}_source"),
format!("{name}_destination"),
]
.iter()
.any(|candidate| taken.contains(candidate))
};
std::iter::once(base.to_owned())
.chain((2_u32..).map(|n| format!("{base}_{n}")))
.find(|name| !used(name))
.unwrap_or_default()
}
fn identifier(name: &str) -> String {
name.chars()
.map(|character| match character.is_ascii_alphanumeric() {
true => character.to_ascii_lowercase(),
false => '_',
})
.collect()
}
fn source_table(source: &Endpoint) -> Result<String, String> {
let name = source
.locator
.trim_end_matches('/')
.rsplit(['.', '/'])
.next()
.filter(|name| !name.is_empty())
.or_else(|| {
source
.settings
.get("root")
.and_then(Value::as_str)
.and_then(|root| {
root.trim_end_matches(['/', '\\'])
.rsplit(['/', '\\'])
.next()
})
})
.unwrap_or_default();
match name.is_empty() {
true => Err(format!(
"the source {} names no table; add one, as connection:schema.table or a URI ending #schema.table",
source.shown
)),
false => Ok(name.to_owned()),
}
}
fn connection_tables(ends: [&Endpoint; 2], with_named: bool) -> Table {
ends.into_iter()
.filter(|end| with_named || !end.named)
.map(|end| (end.connection.clone(), Value::Table(end.settings.clone())))
.collect()
}
fn target(destination: &Endpoint, table: &str) -> Result<(Table, Option<String>, String), String> {
let mut target = Table::new();
target.insert("connection".to_owned(), text_value(&destination.connection));
match FILE_KINDS.contains(&kind(destination)) {
true => {
let folder = destination.locator.trim_matches('/');
let path = match folder.is_empty() {
true => "{table}/".to_owned(),
false => format!("{folder}/{{table}}/"),
};
target.insert("path".to_owned(), Value::String(path));
Ok((target, None, table.to_owned()))
}
false => {
let (schema, name) = destination.locator.split_once('.').ok_or_else(|| {
format!(
"the destination {} names no table; end it with SCHEMA.TABLE",
destination.shown
)
})?;
target.insert("loader_schema".to_owned(), text_value(schema));
Ok((target, Some(schema.to_owned()), name.to_owned()))
}
}
}
fn source_settings(
request: &CopyRequest,
source: &Endpoint,
schema: Option<String>,
name: &str,
) -> Result<Table, String> {
let mut settings = Table::new();
settings.insert("connection".to_owned(), text_value(&source.connection));
settings.insert(
"description".to_owned(),
Value::String(format!("Copied from {}.", source.described)),
);
settings.extend(schema.map(|schema| ("schema".to_owned(), Value::String(schema))));
settings.extend(table_settings(request, !FILE_KINDS.contains(&kind(source))));
match kind(source) {
"postgres" => {
settings.insert("kind".to_owned(), text_value("postgres"));
settings.insert(
"include".to_owned(),
Value::Array(vec![text_value(&source.locator)]),
);
let mut named = Table::new();
named.insert("name".to_owned(), text_value(name));
named.insert(
"description".to_owned(),
Value::String(format!("Copied from {}.", source.described)),
);
settings.insert(
"tables".to_owned(),
Value::Table(single(&source.locator, named)),
);
}
files if FILE_KINDS.contains(&files) => {
settings.insert("kind".to_owned(), text_value("files"));
settings.insert("path".to_owned(), text_value(&source.locator));
settings.insert("table_name".to_owned(), text_value(name));
settings.extend(
request
.format
.clone()
.map(|format| ("format".to_owned(), Value::String(format))),
);
}
other => {
return Err(format!(
"the source {} is a {other} connection; copy reads Postgres tables and files (s3://, file://, a folder)",
source.shown
));
}
}
Ok(settings)
}
fn table_settings(request: &CopyRequest, reports_keys: bool) -> Vec<(String, Value)> {
let mut settings: Vec<(String, Value)> = Vec::new();
let keyed = !request.unique_key.is_empty() || reports_keys;
let strategy =
request
.write_strategy
.clone()
.or_else(|| match (&request.cursor_column, keyed) {
(None, _) => Some("replace".to_owned()),
(Some(_), true) => Some("merge".to_owned()),
(Some(_), false) => None,
});
let key: Vec<Value> = match (request.unique_key.is_empty(), reports_keys) {
(false, _) => request
.unique_key
.iter()
.map(|key| text_value(key))
.collect(),
(true, true) => vec![text_value(PRIMARY_KEY_PLACEHOLDER)],
(true, false) => Vec::new(),
};
let merges = strategy.as_deref() == Some("merge");
settings
.extend(strategy.map(|strategy| ("write_strategy".to_owned(), Value::String(strategy))));
settings.extend(
((merges || !request.unique_key.is_empty()) && !key.is_empty())
.then(|| ("unique_key".to_owned(), Value::Array(key))),
);
settings.extend(
request
.cursor_column
.clone()
.map(|column| ("cursor_column".to_owned(), Value::String(column))),
);
settings
}
fn kind(end: &Endpoint) -> &str {
end.settings
.get("kind")
.and_then(Value::as_str)
.unwrap_or_default()
}
fn single(key: &str, table: Table) -> Table {
let mut outer = Table::new();
outer.insert(key.to_owned(), Value::Table(table));
outer
}
fn text_value(text: &str) -> Value {
Value::String(text.to_owned())
}