1use std::collections::BTreeMap;
16use std::collections::HashSet;
17use std::fmt;
18use std::fmt::Display;
19use std::fmt::Formatter;
20use std::str::FromStr;
21
22use derive_visitor::Drive;
23use derive_visitor::DriveMut;
24use itertools::Itertools;
25use percent_encoding::percent_decode_str;
26use url::Url;
27
28use crate::ast::quote::QuotedString;
29use crate::ast::write_comma_separated_map;
30use crate::ast::write_comma_separated_string_list;
31use crate::ast::write_comma_separated_string_map;
32use crate::ast::Hint;
33use crate::ast::Identifier;
34use crate::ast::Query;
35use crate::ast::TableRef;
36use crate::ast::With;
37use crate::ParseError;
38use crate::Result;
39
40#[derive(Debug, Clone, PartialEq, Drive, DriveMut)]
48pub struct CopyIntoTableStmt {
49 pub with: Option<With>,
50 pub src: CopyIntoTableSource,
51 pub dst: TableRef,
52 pub dst_columns: Option<Vec<Identifier>>,
53
54 pub hints: Option<Hint>,
55
56 pub file_format: FileFormatOptions,
57
58 pub files: Option<Vec<String>>,
60 pub pattern: Option<LiteralStringOrVariable>,
61
62 pub options: CopyIntoTableOptions,
63}
64
65impl CopyIntoTableStmt {
66 pub fn apply_option(
67 &mut self,
68 opt: CopyIntoTableOption,
69 ) -> std::result::Result<(), &'static str> {
70 match opt {
71 CopyIntoTableOption::Files(v) => self.files = Some(v),
72 CopyIntoTableOption::Pattern(v) => self.pattern = Some(v),
73 CopyIntoTableOption::FileFormat(v) => self.file_format = v,
74 CopyIntoTableOption::SizeLimit(v) => self.options.size_limit = v,
75 CopyIntoTableOption::MaxFiles(v) => self.options.max_files = v,
76 CopyIntoTableOption::SplitSize(v) => self.options.split_size = v,
77 CopyIntoTableOption::Purge(v) => self.options.purge = v,
78 CopyIntoTableOption::Force(v) => self.options.force = v,
79 CopyIntoTableOption::DisableVariantCheck(v) => self.options.disable_variant_check = v,
80 CopyIntoTableOption::ReturnFailedOnly(v) => self.options.return_failed_only = v,
81 CopyIntoTableOption::OnError(v) => self.options.on_error = OnErrorMode::from_str(&v)?,
82 CopyIntoTableOption::ColumnMatchMode(v) => {
83 self.options.column_match_mode = Some(ColumnMatchMode::from_str(&v)?)
84 }
85 }
86 Ok(())
87 }
88}
89
90impl Display for CopyIntoTableStmt {
91 fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
92 if let Some(cte) = &self.with {
93 write!(f, "WITH {} ", cte)?;
94 }
95 write!(f, "COPY")?;
96 if let Some(hints) = &self.hints {
97 write!(f, "{} ", hints)?;
98 }
99 write!(f, " INTO {}", self.dst)?;
100 if let Some(columns) = &self.dst_columns {
101 write!(f, "({})", columns.iter().map(|c| c.to_string()).join(","))?;
102 }
103 write!(f, " FROM {}", self.src)?;
104
105 if let Some(files) = &self.files {
106 write!(f, " FILES = (")?;
107 write_comma_separated_string_list(f, files)?;
108 write!(f, " )")?;
109 }
110
111 if let Some(pattern) = &self.pattern {
112 write!(f, " PATTERN = {}", pattern)?;
113 }
114
115 if !self.file_format.is_empty() {
116 write!(f, " FILE_FORMAT = ({})", self.file_format)?;
117 }
118 write!(f, " {}", self.options)?;
119 Ok(())
120 }
121}
122
123#[derive(
124 serde::Serialize, serde::Deserialize, Debug, Clone, Default, PartialEq, Drive, DriveMut, Eq,
125)]
126pub struct CopyIntoTableOptions {
127 pub on_error: OnErrorMode,
128 pub size_limit: usize,
129 pub max_files: usize,
130 pub split_size: usize,
131 pub force: bool,
132 pub purge: bool,
133 pub disable_variant_check: bool,
134 pub return_failed_only: bool,
135 pub validation_mode: String,
136 pub column_match_mode: Option<ColumnMatchMode>,
137}
138
139impl CopyIntoTableOptions {
140 fn parse_uint(k: &str, v: &String) -> std::result::Result<usize, String> {
141 usize::from_str(v).map_err(|e| format!("can not parse {}={} as uint: {}", k, v, e))
142 }
143 fn parse_bool(k: &str, v: &String) -> std::result::Result<bool, String> {
144 bool::from_str(v).map_err(|e| format!("can not parse {}={} as bool: {}", k, v, e))
145 }
146
147 pub fn set_column_match_mode(&mut self, mode: ColumnMatchMode) {
148 self.column_match_mode = Some(mode);
149 }
150
151 pub fn apply(
152 &mut self,
153 opts: &BTreeMap<String, String>,
154 ignore_unknown: bool,
155 ) -> std::result::Result<(), String> {
156 if opts.is_empty() {
157 return Ok(());
158 }
159 for (k, v) in opts.iter() {
160 match k.as_str() {
161 "on_error" => {
162 let on_error = OnErrorMode::from_str(v)?;
163 self.on_error = on_error;
164 }
165 "column_match_mode" => {
166 let column_match_mode = ColumnMatchMode::from_str(v)?;
167 self.column_match_mode = Some(column_match_mode);
168 }
169 "size_limit" => {
170 self.size_limit = Self::parse_uint(k, v)?;
171 }
172 "max_files" => {
173 self.max_files = Self::parse_uint(k, v)?;
174 }
175 "split_size" => {
176 self.split_size = Self::parse_uint(k, v)?;
177 }
178 "purge" => {
179 self.purge = Self::parse_bool(k, v)?;
180 }
181 "disable_variant_check" => {
182 self.disable_variant_check = Self::parse_bool(k, v)?;
183 }
184 "return_failed_only" => {
185 self.return_failed_only = Self::parse_bool(k, v)?;
186 }
187 _ => {
188 if !ignore_unknown {
189 return Err(format!("Unknown stage copy option {}", k));
190 }
191 }
192 }
193 }
194 Ok(())
195 }
196}
197
198impl Display for CopyIntoTableOptions {
199 fn fmt(&self, f: &mut Formatter) -> fmt::Result {
200 if !self.validation_mode.is_empty() {
201 write!(f, "VALIDATION_MODE = {}", self.validation_mode)?;
202 }
203
204 if self.size_limit != 0 {
205 write!(f, " SIZE_LIMIT = {}", self.size_limit)?;
206 }
207
208 if self.max_files != 0 {
209 write!(f, " MAX_FILES = {}", self.max_files)?;
210 }
211
212 if self.split_size != 0 {
213 write!(f, " SPLIT_SIZE = {}", self.split_size)?;
214 }
215
216 write!(f, " PURGE = {}", self.purge)?;
217 write!(f, " FORCE = {}", self.force)?;
218 write!(f, " DISABLE_VARIANT_CHECK = {}", self.disable_variant_check)?;
219 write!(f, " ON_ERROR = {}", self.on_error)?;
220 write!(f, " RETURN_FAILED_ONLY = {}", self.return_failed_only)?;
221 if let Some(mode) = &self.column_match_mode {
222 write!(f, " COLUMN_MATCH_MODE = {}", mode)?;
223 }
224 Ok(())
225 }
226}
227
228#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Drive, DriveMut, Eq)]
229pub struct CopyIntoLocationOptions {
230 pub single: bool,
231 pub max_file_size: usize,
232 pub detailed_output: bool,
233 pub use_raw_path: bool,
234 pub include_query_id: bool,
235 pub overwrite: bool,
236}
237
238impl Default for CopyIntoLocationOptions {
239 fn default() -> Self {
240 Self {
241 single: Default::default(),
242 max_file_size: Default::default(),
243 detailed_output: false,
244 use_raw_path: false,
245 include_query_id: true,
246 overwrite: false,
247 }
248 }
249}
250
251#[derive(Debug, Clone, PartialEq, Drive, DriveMut)]
253pub struct CopyIntoLocationStmt {
254 pub with: Option<With>,
255 pub hints: Option<Hint>,
256 pub src: CopyIntoLocationSource,
257 pub dst: FileLocation,
258 pub file_format: FileFormatOptions,
259 pub options: CopyIntoLocationOptions,
260}
261
262impl Display for CopyIntoLocationStmt {
263 fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
264 if let Some(cte) = &self.with {
265 write!(f, "WITH {} ", cte)?;
266 }
267 write!(f, "COPY")?;
268 if let Some(hints) = &self.hints {
269 write!(f, "{} ", hints)?;
270 }
271 write!(f, " INTO {}", self.dst)?;
272 write!(f, " FROM {}", self.src)?;
273
274 if !self.file_format.is_empty() {
275 write!(f, " FILE_FORMAT = ({})", self.file_format)?;
276 }
277 write!(f, " SINGLE = {}", self.options.single)?;
278 write!(f, " MAX_FILE_SIZE = {}", self.options.max_file_size)?;
279 write!(f, " DETAILED_OUTPUT = {}", self.options.detailed_output)?;
280 write!(f, " INCLUDE_QUERY_ID = {}", self.options.include_query_id)?;
281 write!(f, " USE_RAW_PATH = {}", self.options.use_raw_path)?;
282 write!(f, " OVERWRITE = {}", self.options.overwrite)?;
283
284 Ok(())
285 }
286}
287
288impl CopyIntoLocationStmt {
289 pub fn apply_option(&mut self, opt: CopyIntoLocationOption) {
290 match opt {
291 CopyIntoLocationOption::FileFormat(v) => self.file_format = v,
292 CopyIntoLocationOption::Single(v) => self.options.single = v,
293 CopyIntoLocationOption::MaxFileSize(v) => self.options.max_file_size = v,
294 CopyIntoLocationOption::DetailedOutput(v) => self.options.detailed_output = v,
295 CopyIntoLocationOption::IncludeQueryID(v) => self.options.include_query_id = v,
296 CopyIntoLocationOption::UseRawPath(v) => self.options.use_raw_path = v,
297 CopyIntoLocationOption::OverWrite(v) => self.options.overwrite = v,
298 }
299 }
300}
301
302#[derive(Debug, Clone, PartialEq, Drive, DriveMut)]
303pub enum CopyIntoTableSource {
304 Location(FileLocation),
305 Query(Box<Query>),
308}
309
310impl Display for CopyIntoTableSource {
311 fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
312 match self {
313 CopyIntoTableSource::Location(location) => write!(f, "{location}"),
314 CopyIntoTableSource::Query(query) => {
315 write!(f, "({query})")
316 }
317 }
318 }
319}
320
321#[derive(Debug, Clone, PartialEq, Drive, DriveMut)]
322pub enum CopyIntoLocationSource {
323 Query(Box<Query>),
324 Table(TableRef),
326}
327
328impl Display for CopyIntoLocationSource {
329 fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
330 match self {
331 CopyIntoLocationSource::Query(query) => {
332 write!(f, "({query})")
333 }
334 CopyIntoLocationSource::Table(table) => {
335 write!(f, "{}", table)
336 }
337 }
338 }
339}
340
341#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)]
342pub struct Connection {
343 #[drive(skip)]
344 visited_keys: HashSet<String>,
345 pub conns: BTreeMap<String, String>,
346}
347
348impl Connection {
349 pub fn new(conns: BTreeMap<String, String>) -> Self {
350 Self {
351 visited_keys: HashSet::new(),
352 conns,
353 }
354 }
355
356 pub fn mask(&self) -> Self {
357 let mut conns = BTreeMap::new();
358 for (k, v) in &self.conns {
359 conns.insert(k.to_string(), mask_string(v, 3));
360 }
361 Self {
362 visited_keys: self.visited_keys.clone(),
363 conns,
364 }
365 }
366
367 pub fn get(&mut self, key: &str) -> Option<&String> {
368 self.visited_keys.insert(key.to_string());
369 self.conns.get(key)
370 }
371
372 pub fn check(&self) -> Result<()> {
373 let conn_keys = HashSet::from_iter(self.conns.keys().cloned());
374 let diffs: Vec<String> = conn_keys
375 .difference(&self.visited_keys)
376 .map(|x| x.to_string())
377 .collect();
378
379 if !diffs.is_empty() {
380 return Err(ParseError(
381 None,
382 format!(
383 "connection params invalid: expected [{}], got [{}]",
384 self.visited_keys
385 .iter()
386 .cloned()
387 .collect::<Vec<_>>()
388 .join(","),
389 diffs.join(",")
390 ),
391 ));
392 }
393 Ok(())
394 }
395}
396
397impl Display for Connection {
398 fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
399 if !self.conns.is_empty() {
400 write!(f, " CONNECTION = ( ")?;
401 write_comma_separated_string_map(f, &self.conns)?;
402 write!(f, " )")?;
403 }
404 Ok(())
405 }
406}
407
408fn mask_string(s: &str, unmask_len: usize) -> String {
410 if s.len() <= unmask_len {
411 s.to_string()
412 } else {
413 let mut ret = "******".to_string();
414 ret.push_str(&s[(s.len() - unmask_len)..]);
415 ret
416 }
417}
418
419#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)]
423pub struct UriLocation {
424 pub protocol: String,
425 pub name: String,
426 pub path: String,
427 pub connection: Connection,
428}
429
430impl UriLocation {
431 pub fn new(
432 protocol: String,
433 name: String,
434 path: String,
435 conns: BTreeMap<String, String>,
436 ) -> Self {
437 Self {
438 protocol,
439 name,
440 path,
441 connection: Connection::new(conns),
442 }
443 }
444
445 pub fn from_uri(uri: String, conns: BTreeMap<String, String>) -> Result<Self> {
446 if let Some(path) = uri.strip_prefix("fs://") {
448 if !path.starts_with('/') {
449 return Err(ParseError(
450 None,
451 format!("Invalid uri: {}. fs location must start with 'fs:///'", uri),
452 ));
453 }
454 return Ok(UriLocation::new(
455 "fs".to_string(),
456 "".to_string(),
457 path.to_string(),
458 BTreeMap::default(),
459 ));
460 }
461
462 let parsed =
463 Url::parse(&uri).map_err(|e| ParseError(None, format!("invalid uri {}", e)))?;
464
465 let protocol = parsed.scheme().to_string();
466
467 let name = parsed
468 .host_str()
469 .map(|hostname| {
470 if let Some(port) = parsed.port() {
471 format!("{}:{}", hostname, port)
472 } else {
473 hostname.to_string()
474 }
475 })
476 .ok_or_else(|| ParseError(None, "invalid uri".to_string()))?;
477
478 let path = if parsed.path().is_empty() {
479 "/".to_string()
480 } else {
481 percent_decode_str(parsed.path())
482 .decode_utf8_lossy()
483 .to_string()
484 };
485
486 Ok(Self {
487 protocol,
488 name,
489 path,
490 connection: Connection::new(conns),
491 })
492 }
493
494 pub fn mask(&self) -> Self {
495 Self {
496 connection: self.connection.mask(),
497 ..self.clone()
498 }
499 }
500}
501
502impl Display for UriLocation {
503 fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
504 write!(f, "'{}://{}{}'", self.protocol, self.name, self.path)?;
505 write!(f, "{}", self.connection)?;
506 Ok(())
507 }
508}
509
510#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)]
522pub enum FileLocation {
523 Stage(String),
524 Uri(UriLocation),
525}
526
527impl Display for FileLocation {
528 fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
529 match self {
530 FileLocation::Uri(loc) => {
531 write!(f, "{}", loc)
532 }
533 FileLocation::Stage(loc) => {
534 write!(f, "'@{}'", loc)
535 }
536 }
537 }
538}
539
540#[derive(Debug, Clone, PartialEq, Drive, DriveMut)]
547pub enum LiteralStringOrVariable {
548 Literal(String),
549 Variable(String),
550}
551
552impl Display for LiteralStringOrVariable {
553 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
554 match self {
555 LiteralStringOrVariable::Literal(s) => {
556 write!(f, "'{s}'")
557 }
558 LiteralStringOrVariable::Variable(s) => {
559 write!(f, "${s}")
560 }
561 }
562 }
563}
564
565pub enum CopyIntoTableOption {
566 Files(Vec<String>),
567 Pattern(LiteralStringOrVariable),
568 FileFormat(FileFormatOptions),
569 SizeLimit(usize),
570 MaxFiles(usize),
571 SplitSize(usize),
572 Purge(bool),
573 Force(bool),
574 DisableVariantCheck(bool),
575 ReturnFailedOnly(bool),
576 OnError(String),
577 ColumnMatchMode(String),
578}
579
580pub enum CopyIntoLocationOption {
581 FileFormat(FileFormatOptions),
582 MaxFileSize(usize),
583 Single(bool),
584 IncludeQueryID(bool),
585 UseRawPath(bool),
586 DetailedOutput(bool),
587 OverWrite(bool),
588}
589
590#[derive(Clone, Debug, PartialEq, Eq, Default, Drive, DriveMut)]
591pub struct FileFormatOptions {
592 pub options: BTreeMap<String, FileFormatValue>,
593}
594
595impl FileFormatOptions {
596 pub fn is_empty(&self) -> bool {
597 self.options.is_empty()
598 }
599}
600
601impl Display for FileFormatOptions {
602 fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
603 write_comma_separated_map(f, &self.options)
604 }
605}
606
607#[derive(Clone, Debug, PartialEq, Eq, Drive, DriveMut)]
608pub enum FileFormatValue {
609 Keyword(String),
610 Bool(bool),
611 U64(u64),
612 String(String),
613 StringList(Vec<String>),
614}
615
616impl FileFormatValue {
617 pub fn to_meta_value(&self) -> String {
618 match self {
619 FileFormatValue::Keyword(v) => v.clone(),
620 FileFormatValue::Bool(v) => v.to_string(),
621 FileFormatValue::U64(v) => v.to_string(),
622 FileFormatValue::String(v) => v.clone(),
623 FileFormatValue::StringList(v) => serde_json::to_string(v).unwrap(),
624 }
625 }
626}
627
628impl Display for FileFormatValue {
629 fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
630 match self {
631 FileFormatValue::Keyword(v) => write!(f, "{v}"),
632 FileFormatValue::Bool(v) => write!(f, "{v}"),
633 FileFormatValue::U64(v) => write!(f, "{v}"),
634 FileFormatValue::String(v) => {
635 write!(f, "{}", QuotedString(v, '\''))
636 }
637 FileFormatValue::StringList(v) => {
638 write!(f, "(")?;
639 for (i, s) in v.iter().enumerate() {
640 if i > 0 {
641 write!(f, ", ")?;
642 }
643 write!(f, "{}", QuotedString(s, '\''))?;
644 }
645 write!(f, ")")
646 }
647 }
648 }
649}
650
651#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Drive, DriveMut, Eq)]
652pub enum OnErrorMode {
653 Continue,
654 SkipFileNum(u64),
655 AbortNum(u64),
656}
657
658impl Default for OnErrorMode {
659 fn default() -> Self {
660 Self::AbortNum(1)
661 }
662}
663
664impl Display for OnErrorMode {
665 fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
666 match self {
667 OnErrorMode::Continue => {
668 write!(f, "continue")
669 }
670 OnErrorMode::SkipFileNum(n) => {
671 if *n <= 1 {
672 write!(f, "skipfile")
673 } else {
674 write!(f, "skipfile_{}", n)
675 }
676 }
677 OnErrorMode::AbortNum(n) => {
678 if *n <= 1 {
679 write!(f, "abort")
680 } else {
681 write!(f, "abort_{}", n)
682 }
683 }
684 }
685 }
686}
687
688const ERROR_MODE_MSG: &str =
689 "OnError must one of {{ CONTINUE | SKIP_FILE | SKIP_FILE_<num> | ABORT | ABORT_<num> }}";
690impl FromStr for OnErrorMode {
691 type Err = &'static str;
692
693 fn from_str(s: &str) -> std::result::Result<Self, &'static str> {
694 match s.to_uppercase().as_str() {
695 "" | "ABORT" => Ok(OnErrorMode::AbortNum(1)),
696 "CONTINUE" => Ok(OnErrorMode::Continue),
697 "SKIP_FILE" => Ok(OnErrorMode::SkipFileNum(1)),
698 v => {
699 if v.starts_with("ABORT_") {
700 let num_str = v.replace("ABORT_", "");
701 let nums = num_str.parse::<u64>();
702 match nums {
703 Ok(n) if n < 1 => Err(ERROR_MODE_MSG),
704 Ok(n) => Ok(OnErrorMode::AbortNum(n)),
705 Err(_) => Err(ERROR_MODE_MSG),
706 }
707 } else {
708 let num_str = v.replace("SKIP_FILE_", "");
709 let nums = num_str.parse::<u64>();
710 match nums {
711 Ok(n) if n < 1 => Err(ERROR_MODE_MSG),
712 Ok(n) => Ok(OnErrorMode::SkipFileNum(n)),
713 Err(_) => Err(ERROR_MODE_MSG),
714 }
715 }
716 }
717 }
718 }
719}
720
721#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Drive, DriveMut, Eq)]
722pub enum ColumnMatchMode {
723 CaseSensitive,
724 CaseInsensitive,
725 Position,
726}
727
728impl Display for ColumnMatchMode {
729 fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
730 match self {
731 ColumnMatchMode::CaseSensitive => write!(f, "CASE_SENSITIVE"),
732 ColumnMatchMode::CaseInsensitive => write!(f, "CASE_INSENSITIVE"),
733 ColumnMatchMode::Position => write!(f, "POSITION"),
734 }
735 }
736}
737
738const COLUMN_MATCH_MODE_MSG: &str =
739 "ColumnMatchMode must be one of {{ CASE_SENSITIVE | CASE_INSENSITIVE | POSITION }}";
740impl FromStr for ColumnMatchMode {
741 type Err = &'static str;
742
743 fn from_str(s: &str) -> std::result::Result<Self, &'static str> {
744 match s.to_uppercase().as_str() {
745 "CASE_SENSITIVE" => Ok(Self::CaseSensitive),
746 "CASE_INSENSITIVE" => Ok(Self::CaseInsensitive),
747 "POSITION" => Ok(Self::Position),
748 _ => Err(COLUMN_MATCH_MODE_MSG),
749 }
750 }
751}