1use polars::prelude::PolarsError;
7use std::io;
8use std::path::{Path, PathBuf};
9
10#[derive(Debug)]
13pub struct FileError {
14 path: PathBuf,
15 at: Option<(usize, usize)>,
17 what: String,
18 source: Option<color_eyre::eyre::Report>,
20}
21
22impl FileError {
23 pub fn new(path: &Path, what: impl Into<String>) -> Self {
24 Self {
25 path: path.to_path_buf(),
26 at: None,
27 what: what.into(),
28 source: None,
29 }
30 }
31
32 pub fn at(path: &Path, line: usize, column: usize, what: impl Into<String>) -> Self {
35 Self {
36 at: (line > 0).then_some((line, column)),
37 ..Self::new(path, what)
38 }
39 }
40}
41
42impl std::fmt::Display for FileError {
43 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
44 f.write_str(&located_message(Some(&self.path), self.at, &self.what))
45 }
46}
47
48impl std::error::Error for FileError {
49 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
50 self.source
51 .as_ref()
52 .map(|e| &**e as &(dyn std::error::Error + 'static))
53 }
54}
55
56pub fn file_message(path: &Path, what: &str) -> String {
60 located_message(Some(path), None, what)
61}
62
63fn names_a_file(what: &str) -> bool {
67 let Some(quoted) = what.strip_prefix('"') else {
68 return false;
69 };
70 let Some((_, after)) = quoted.split_once("\":") else {
71 return false;
72 };
73 after.starts_with(' ') || after.starts_with(|c: char| c.is_ascii_digit())
74}
75
76pub fn located_message(path: Option<&Path>, at: Option<(usize, usize)>, what: &str) -> String {
80 let mut said = String::with_capacity(what.len() + 16);
81 let mut what = what.trim();
82 if let Some(path) = path {
83 if names_a_file(what) {
84 return what.to_string();
85 }
86 let named = path.display().to_string();
87 what = what.strip_prefix(&format!("{named}: ")).unwrap_or(what);
89 said.push('"');
90 said.push_str(&named);
91 said.push('"');
92 said.push(':');
93 if at.is_none() {
94 said.push(' ');
95 }
96 }
97 if let Some((line, column)) = at {
98 said.push_str(&format!("{line}:{column}: "));
99 }
100 said.push_str(&sentence(what));
101 said
102}
103
104pub fn sentence(what: &str) -> String {
107 let what = what.trim();
108 let (first, rest) = what.split_once('\n').unwrap_or((what, ""));
109 let first = first.trim_end();
110 let mut said = String::with_capacity(what.len() + 1);
111 let mut chars = first.chars();
112 if let Some(c) = chars.next() {
113 if starts_with_a_key(first) {
115 said.push(c);
116 } else {
117 said.extend(c.to_uppercase());
118 }
119 said.push_str(chars.as_str());
120 }
121 if !first.ends_with(['.', '?', '!']) {
122 said.push('.');
123 }
124 if !rest.is_empty() {
125 said.push('\n');
126 said.push_str(rest);
127 }
128 said
129}
130
131pub fn starts_with_a_key(what: &str) -> bool {
135 let word = what.split_whitespace().next().unwrap_or_default();
136 word.ends_with(':')
137 || word.contains(|c: char| matches!(c, '.' | '_' | '=' | '"' | '`') || c.is_ascii_digit())
138}
139
140#[cfg(feature = "cloud")]
143pub fn store_message(err: &object_store::Error) -> String {
144 use object_store::Error as E;
145 match err {
146 E::NotFound { .. } => "No object there. Check the URL.".to_string(),
147 E::PermissionDenied { .. } => "Access denied. Check the credentials.".to_string(),
148 E::Unauthenticated { .. } => {
149 "The store did not accept the credentials. Check them.".to_string()
150 }
151 e => format!(
152 "Could not read it: {}. Check the credentials and the URL.",
153 e.to_string().trim_end_matches('.')
154 ),
155 }
156}
157
158#[cfg(any(feature = "http", feature = "cloud"))]
161pub fn http_message(url: &str, err: &ureq::Error) -> String {
162 let host = url_host(url);
163 match err {
164 ureq::Error::StatusCode(code @ (404 | 410)) => {
165 format!("The server at {host} returned {code}: the file may have moved.")
166 }
167 ureq::Error::StatusCode(code @ (401 | 403)) => {
168 format!("The server at {host} refused it ({code}). Check the URL and its access.")
169 }
170 ureq::Error::StatusCode(code @ 500..=599) => {
171 format!("The server at {host} returned {code}: try again later.")
172 }
173 ureq::Error::StatusCode(code) => format!("The server at {host} returned {code}."),
174 _ if http_unanswered(err) => format!("No answer from {host}."),
175 e => format!(
176 "Could not read it: {}.",
177 e.to_string().trim_end_matches('.')
178 ),
179 }
180}
181
182#[derive(Debug, Clone, PartialEq, Eq)]
184pub struct HttpGone {
185 pub cell: String,
187 pub message: String,
189}
190
191#[cfg(any(feature = "http", feature = "cloud"))]
195pub fn http_gone(url: &str, err: &ureq::Error) -> Option<HttpGone> {
196 let cell = match err {
197 ureq::Error::StatusCode(code @ (404 | 410)) => format!("HTTP {code}"),
198 _ if http_unanswered(err) => "no answer".to_string(),
199 _ => return None,
200 };
201 Some(HttpGone {
202 cell,
203 message: http_message(url, err),
204 })
205}
206
207#[cfg(any(feature = "http", feature = "cloud"))]
210fn http_unanswered(err: &ureq::Error) -> bool {
211 matches!(
212 err,
213 ureq::Error::HostNotFound
214 | ureq::Error::ConnectionFailed
215 | ureq::Error::Timeout(_)
216 | ureq::Error::Io(_)
217 )
218}
219
220#[cfg(any(feature = "http", feature = "cloud"))]
223fn url_host(url: &str) -> String {
224 url::Url::parse(url)
225 .ok()
226 .and_then(|u| {
227 let host = u.host_str()?.to_string();
228 Some(match u.port() {
229 Some(port) => format!("{host}:{port}"),
230 None => host,
231 })
232 })
233 .unwrap_or_else(|| url.to_string())
234}
235
236pub fn in_file(path: &Path, err: color_eyre::eyre::Report) -> color_eyre::eyre::Report {
238 if err.downcast_ref::<FileError>().is_some() {
239 return err;
240 }
241 let what = report_message(cfg!(windows), &err, None);
242 color_eyre::eyre::Report::new(FileError {
243 source: Some(err),
244 ..FileError::new(path, what)
245 })
246}
247
248pub fn user_message_from_polars(err: &PolarsError) -> String {
250 let said = polars_words(err);
255 if is_union_schema_error(&said) {
256 return union_schema_message(&said);
257 }
258 if let Some(differ) = files_columns_differ(&said) {
259 return differ;
260 }
261 if let Some(none) = nothing_matched(&said) {
262 return none;
263 }
264 let said = without_the_query_plan(&said);
265 let (first, rest) = said.split_once('\n').unwrap_or((&said, ""));
266 match rust_names_said_plainly(first) {
267 Some(plain) if rest.is_empty() => plain,
268 Some(plain) => format!("{plain}\n{rest}"),
269 None => said,
270 }
271}
272
273fn rust_names_said_plainly(msg: &str) -> Option<String> {
276 let words = |name: &str| {
278 let mut out = String::new();
279 for (i, c) in name.chars().enumerate() {
280 if c.is_uppercase() && i > 0 {
281 out.push(' ');
282 }
283 out.extend(c.to_lowercase());
284 }
285 out
286 };
287 let msg = msg.trim();
288 let is_name = |s: &str| !s.is_empty() && s.chars().all(|c| c.is_ascii_alphanumeric());
289 let damaged = |what: Option<String>| {
290 let what = what.map(|w| format!(" ({w})")).unwrap_or_default();
291 format!(
292 "The file is not laid out as its format says{what}: it is damaged, or another \
293 format. --format names the format to read it as."
294 )
295 };
296 if msg == "OutOfSpec" {
297 return Some(damaged(None));
298 }
299 const SPEC: &str = "out-of-spec: ";
300 if let Some(name) = msg
301 .get(..SPEC.len())
302 .filter(|p| p.eq_ignore_ascii_case(SPEC))
303 .map(|_| &msg[SPEC.len()..])
304 && is_name(name)
305 {
306 return Some(damaged(Some(words(name))));
307 }
308 let at = msg.strip_prefix("InvalidUtf8")?;
309 Some(format!("The file is not UTF-8 text{at}."))
310}
311
312fn nothing_matched(msg: &str) -> Option<String> {
317 let (_, input) = msg.split_once("expanded paths were empty")?;
318 let pattern = input.contains("glob: true")
321 && input.split("inner: \"").skip(1).any(|p| {
322 p.split('"')
323 .next()
324 .is_some_and(|p| p.contains(['*', '?', '[']))
325 });
326 Some(if pattern {
327 "No files match this pattern.".to_string()
328 } else {
329 "No files found there.".to_string()
330 })
331}
332
333fn polars_words(err: &PolarsError) -> String {
334 use polars::prelude::PolarsError as PE;
335
336 match err {
337 PE::ColumnNotFound(msg) => format!(
338 "Column not found: {}. Check spelling and that the column exists.",
339 msg
340 ),
341 PE::Duplicate(msg) => format!(
342 "Duplicate column in result: {}. Use aliases to rename columns, e.g. `select my_date: timestamp.date`",
343 msg
344 ),
345 PE::IO { error, msg } => {
346 user_message_from_io(error.as_ref(), msg.as_ref().map(|m| m.as_ref()))
347 }
348 PE::NoData(msg) => format!("No data: {}", msg),
349 PE::SchemaMismatch(msg) => format!("Schema mismatch: {}", msg),
350 PE::ShapeMismatch(msg) => format!("Row shape mismatch: {}", msg),
351 PE::InvalidOperation(msg) => format!("Operation not allowed: {}", msg),
352 PE::OutOfBounds(msg) => format!("Index or row out of bounds: {}", msg),
353 PE::SchemaFieldNotFound(msg) => format!("Schema field not found: {}", msg),
354 PE::StructFieldNotFound(msg) => format!("Struct field not found: {}", msg),
355 PE::ComputeError(msg) => simplify_compute_message(msg),
356 PE::AssertionError(msg) => format!("Assertion failed: {}", msg),
357 PE::StringCacheMismatch(msg) => format!("String cache mismatch: {}", msg),
358 PE::SQLInterface(msg) | PE::SQLSyntax(msg) => msg.to_string(),
359 PE::Context { error, msg } => {
360 let inner = user_message_from_polars(error);
361 format!("{}: {}", msg, inner)
362 }
363 #[allow(unreachable_patterns)]
364 _ => err.to_string(),
365 }
366}
367
368#[derive(Debug, Clone, PartialEq, Eq)]
372pub struct ConversionFailure {
373 pub column: String,
374 pub to: String,
376 pub failed: usize,
377 pub checked: usize,
380 pub examples: Vec<String>,
382 pub parsing: bool,
384}
385
386pub fn conversion_failure(err: &PolarsError) -> Option<ConversionFailure> {
388 let mut parsing = false;
389 let mut err = err;
390 loop {
391 match err {
392 PolarsError::ExprContext { error, expr } => {
393 parsing |= expr.contains("strptime") || expr.contains("to_date");
394 err = error;
395 }
396 PolarsError::Context { error, .. } => err = error,
397 _ => break,
398 }
399 }
400 let PolarsError::InvalidOperation(msg) = err else {
401 return None;
402 };
403 parse_conversion(msg, parsing)
404}
405
406fn parse_conversion(msg: &str, parsing: bool) -> Option<ConversionFailure> {
409 let rest = msg.strip_prefix("conversion from `")?;
410 let (_, rest) = rest.split_once("` to `")?;
411 let (to, rest) = rest.split_once("` failed in column '")?;
412 let (column, rest) = rest.split_once("' for ")?;
413 let (failed, rest) = rest.split_once(" out of ")?;
414 let (checked, rest) = rest.split_once(" values: ")?;
415 let list = rest.lines().next().unwrap_or("");
416 let list = list.strip_prefix('[').unwrap_or(list);
417 let list = list.strip_suffix(']').unwrap_or(list);
418 let mut examples: Vec<String> = Vec::new();
419 for item in list.split(", ") {
420 let item = item.trim_start_matches('…').trim();
422 let item = item
423 .strip_prefix('"')
424 .and_then(|i| i.strip_suffix('"'))
425 .unwrap_or(item);
426 if !item.is_empty() && !examples.iter().any(|e| e == item) && examples.len() < 3 {
427 examples.push(item.to_string());
428 }
429 }
430 Some(ConversionFailure {
431 column: column.to_string(),
432 to: to.to_string(),
433 failed: failed.trim().parse().ok()?,
434 checked: checked.trim().parse().ok()?,
435 examples,
436 parsing,
437 })
438}
439
440impl ConversionFailure {
441 pub fn sql_message(&self, rows: Option<usize>) -> String {
446 let column = crate::sql_assist::sql_name(&self.column);
447 let exact = rows == Some(self.checked);
448 let one = !exact && self.failed == 1;
449 let temporal = self.to == "date" || self.to == "time" || self.to.starts_with("datetime");
450 let (many, single) = if temporal && self.parsing {
451 ("do not match the format", "does not match the format")
452 } else if self.to == "date" {
453 (
454 "are not dates written YYYY-MM-DD",
455 "is not a date written YYYY-MM-DD",
456 )
457 } else if self.to == "time" {
458 (
459 "are not times written HH:MM:SS",
460 "is not a time written HH:MM:SS",
461 )
462 } else if temporal {
463 (
464 "are not timestamps written YYYY-MM-DD HH:MM:SS",
465 "is not a timestamp written YYYY-MM-DD HH:MM:SS",
466 )
467 } else if self.to.starts_with('i') || self.to.starts_with('u') {
468 ("are not whole numbers", "is not a whole number")
469 } else if self.to.starts_with('f') || self.to.starts_with("decimal") {
470 ("are not numbers", "is not a number")
471 } else if self.to == "bool" {
472 ("are not true or false", "is not true or false")
473 } else {
474 ("cannot be converted", "cannot be converted")
475 };
476 let quoted: Vec<String> = self.examples.iter().map(|e| format!("\"{e}\"")).collect();
477 let such_as = match quoted.as_slice() {
478 [] => String::new(),
479 [one] => format!(", such as {one}"),
480 [init @ .., last] => format!(", such as {} and {last}", init.join(", ")),
481 };
482 let lead = if exact {
483 format!(
484 "{column}: {} of {} values {many}{such_as}.",
485 crate::numfmt::group_chrome(self.failed),
486 crate::numfmt::group_chrome(self.checked)
487 )
488 } else if one {
489 format!("At least 1 value in {column} {single}{such_as}.")
490 } else {
491 format!(
492 "At least {} values in {column} {many}{such_as}.",
493 crate::numfmt::group_chrome(self.failed)
494 )
495 };
496 let hint = if temporal && self.parsing {
497 format!(
498 "Try: a format that fits them all, or trim the text first with \
499 SUBSTR({column}, 1, n) or REPLACE({column}, 'text', '')."
500 )
501 } else if temporal {
502 format!(
503 "Try: STRPTIME({column}, '%d/%m/%Y') with the format the values are \
504 written in."
505 )
506 } else {
507 let sql_type = sql_type_for(&self.to);
508 format!(
509 "Try: TRY_CAST({column} AS {sql_type}) to read them as null, or clean \
510 the text first with REPLACE or SUBSTR."
511 )
512 };
513 format!("{lead}\n{hint}")
514 }
515}
516
517fn sql_type_for(polars: &str) -> &'static str {
519 match polars {
520 "i8" => "TINYINT",
521 "i16" => "SMALLINT",
522 "i32" => "INT",
523 "i64" => "BIGINT",
524 "i128" => "HUGEINT",
525 "u8" => "UTINYINT",
526 "u16" => "USMALLINT",
527 "u32" => "UINTEGER",
528 "u64" => "UBIGINT",
529 "f32" => "REAL",
530 "f64" => "DOUBLE",
531 "bool" => "BOOLEAN",
532 t if t.starts_with("decimal") => "DECIMAL",
533 _ => "VARCHAR",
534 }
535}
536
537pub fn sql_error_message(err: &PolarsError, rows: Option<usize>) -> String {
540 match conversion_failure(err) {
541 Some(failure) => failure.sql_message(rows),
542 None => user_message_from_polars(err),
543 }
544}
545
546const HELD: &str =
548 "is open in another program that does not allow reading it; close it there and reopen";
549
550pub fn held_by_another_program(err: &io::Error) -> bool {
556 held_on(cfg!(windows), err)
557}
558
559fn held_on(windows: bool, err: &io::Error) -> bool {
560 windows && (matches!(err.raw_os_error(), Some(32 | 33)) || says_held(&err.to_string()))
561}
562
563fn says_held(text: &str) -> bool {
565 text.contains("(os error 32)") || text.contains("(os error 33)")
566}
567
568fn report_held(windows: bool, report: &color_eyre::eyre::Report) -> bool {
571 fn polars_held(windows: bool, err: &PolarsError) -> bool {
572 match err {
573 PolarsError::IO { error, .. } => held_on(windows, error),
574 PolarsError::Context { error, .. } => polars_held(windows, error),
575 _ => false,
576 }
577 }
578 windows
579 && report.chain().any(|cause| {
580 cause
581 .downcast_ref::<io::Error>()
582 .is_some_and(|e| held_on(windows, e))
583 || cause
584 .downcast_ref::<PolarsError>()
585 .is_some_and(|e| polars_held(windows, e))
586 || says_held(&cause.to_string())
587 })
588}
589
590fn held_message(path: Option<&Path>) -> String {
592 match path {
593 Some(path) => file_message(path, &format!("the file {HELD}")),
594 None => format!("The file {HELD}."),
595 }
596}
597
598pub fn user_message_from_io(err: &io::Error, context: Option<&str>) -> String {
600 use std::io::ErrorKind;
601
602 if held_by_another_program(err) {
603 return held_message(None);
604 }
605 let base: String = match err.kind() {
606 ErrorKind::NotFound => "File or directory not found.".to_string(),
607 ErrorKind::PermissionDenied => "Permission denied. Check read access.".to_string(),
608 ErrorKind::ConnectionRefused => "Connection refused.".to_string(),
609 ErrorKind::ConnectionReset => "Connection reset.".to_string(),
610 ErrorKind::InvalidData | ErrorKind::InvalidInput => {
611 "Invalid or corrupted data.".to_string()
612 }
613 ErrorKind::UnexpectedEof => "Unexpected end of file.".to_string(),
614 ErrorKind::WouldBlock => "Operation would block.".to_string(),
615 ErrorKind::Interrupted => "Operation interrupted.".to_string(),
616 ErrorKind::OutOfMemory => "Out of memory.".to_string(),
617 ErrorKind::Other => {
618 let msg = err.to_string();
619 if msg.contains("No space left") || msg.contains("space left") {
620 return "No space left on device. Free up disk space and try again.".to_string();
621 }
622 if msg.contains("Is a directory") {
623 return "Path is a directory, not a file.".to_string();
624 }
625 return if context.is_some() {
626 format!("I/O error: {}", msg)
627 } else {
628 msg
629 };
630 }
631 _ => err.to_string(),
632 };
633
634 if let Some(ctx) = context {
635 if !ctx.is_empty() {
636 format!("{} {}", base, ctx)
637 } else {
638 base
639 }
640 } else {
641 base
642 }
643}
644
645#[derive(Debug, Clone, Copy)]
648pub enum ErrorKindForPython {
649 FileNotFound,
650 PermissionDenied,
651 Other,
652}
653
654pub fn error_for_python(report: &color_eyre::eyre::Report) -> (ErrorKindForPython, String) {
657 use std::io::ErrorKind;
658 let named = report.downcast_ref::<FileError>().map(ToString::to_string);
659 for cause in report.chain() {
660 if let Some(io_err) = cause.downcast_ref::<io::Error>() {
661 let kind = match io_err.kind() {
662 ErrorKind::NotFound => ErrorKindForPython::FileNotFound,
663 ErrorKind::PermissionDenied => ErrorKindForPython::PermissionDenied,
664 _ => ErrorKindForPython::Other,
665 };
666 let msg = named.unwrap_or_else(|| io_err.to_string());
667 return (kind, msg);
668 }
669 }
670 if let Some(named) = named {
671 return (ErrorKindForPython::Other, named);
672 }
673 let display = report.to_string();
674 let msg = display
675 .lines()
676 .next()
677 .map(str::trim)
678 .unwrap_or("An error occurred")
679 .to_string();
680 (ErrorKindForPython::Other, msg)
681}
682
683pub fn named_by_source(message: &str, file: &Path, source: &Path) -> String {
692 let file = file.to_string_lossy();
693 if file.is_empty() {
694 return message.to_string();
695 }
696 let source = source.to_string_lossy();
697 let in_a_name = |c: char| c.is_alphanumeric() || matches!(c, '_' | '-' | '~');
698 let in_a_path = |c: char| in_a_name(c) || matches!(c, '.' | '/' | '\\');
699 let mut named = String::with_capacity(message.len());
700 let mut copied = 0;
701 for (at, _) in message.match_indices(file.as_ref()) {
702 let end = at + file.len();
703 let mut after = message[end..].chars();
704 let whole = !message[..at].chars().next_back().is_some_and(in_a_path)
705 && match after.next() {
706 None => true,
707 Some('.') => !after.next().is_some_and(in_a_path),
708 Some(c) => !in_a_path(c),
709 };
710 if whole {
711 named.push_str(&message[copied..at]);
712 named.push_str(&source);
713 copied = end;
714 }
715 }
716 named.push_str(&message[copied..]);
717 named
718}
719
720pub fn user_message_from_report(report: &color_eyre::eyre::Report, path: Option<&Path>) -> String {
723 report_message(cfg!(windows), report, path)
724}
725
726fn report_message(windows: bool, report: &color_eyre::eyre::Report, path: Option<&Path>) -> String {
727 if let Some(named) = report.downcast_ref::<FileError>() {
729 return named.to_string();
730 }
731 if report_held(windows, report) {
732 return held_message(path);
733 }
734 let named = |msg: String| match path {
735 Some(p) => file_message(p, &msg),
736 None => msg,
737 };
738 for cause in report.chain() {
739 if let Some(named_file) = cause.downcast_ref::<FileError>() {
740 return named_file.to_string();
741 }
742 if let Some(pe) = cause.downcast_ref::<PolarsError>() {
743 return named(user_message_from_polars(pe));
744 }
745 if let Some(io_err) = cause.downcast_ref::<io::Error>() {
746 return named(user_message_from_io(io_err, None));
747 }
748 }
749
750 let display = report.to_string();
752 let first_line = display.lines().next().unwrap_or("An error occurred").trim();
753 named(rust_names_said_plainly(first_line).unwrap_or_else(|| first_line.to_string()))
754}
755
756fn without_the_query_plan(msg: &str) -> String {
766 let Some(cut) = msg.find("Resolved plan until failure:") else {
767 return msg.to_string();
768 };
769 let (said, plan) = msg.split_at(cut);
770 let said = said.trim_end();
771 match file_in_plan(plan) {
772 Some(file) => format!("{said}\nIt stopped at {file}."),
773 None => said.to_string(),
774 }
775}
776
777fn file_in_plan(plan: &str) -> Option<&str> {
784 const MARKER: &str = "FAILED HERE";
785 let from = plan.find(MARKER).map_or(0, |at| at + MARKER.len());
786 let at = plan[from..].find(" SCAN [")? + from + " SCAN [".len();
787 let rest = &plan[at..];
788 let end = rest.find(']')?;
789 let file = rest[..end].trim();
790 (!file.is_empty()).then_some(file)
791}
792
793fn is_union_schema_error(msg: &str) -> bool {
802 msg.contains("'union'/'concat' inputs should all have the same schema")
808}
809
810fn union_schema_message(msg: &str) -> String {
811 let mut said = "These files cannot be read as one table: they disagree on a column \
812 in a way datui cannot reconcile by widening its type."
813 .to_string();
814 if let Some(file) = file_in_plan(msg) {
815 said.push_str(&format!("\nIt stopped at {file}."));
816 }
817 said.push_str(
818 "\nTry: open one file on its own, or --format / --infer-rows to settle \
819 the types.",
820 );
821 said
822}
823
824fn files_columns_differ(msg: &str) -> Option<String> {
830 let how = if let Some(rest) = msg.split("schema names differ: got ").nth(1) {
831 let (got, expected) = rest.split_once(", expected ")?;
832 let expected = expected.lines().next().unwrap_or(expected).trim();
833 format!(
834 "one has a column named \"{}\" where another has \"{expected}\"",
835 got.trim()
836 )
837 } else if msg.contains("schema lengths differ") {
838 "they have different numbers of columns".to_string()
839 } else {
840 return None;
841 };
842 Some(format!(
843 "These files cannot be read as one table: their columns differ — {how}.\n\
844 Try: open one file on its own. If the files have no header row, --no-header \
845 reads their columns by position."
846 ))
847}
848
849fn simplify_compute_message(msg: &str) -> String {
851 if is_csv_parse_type_error(msg) {
852 return short_csv_parse_error_message(msg);
853 }
854 crate::query::sanitize_query_error(msg)
855}
856
857fn is_csv_parse_type_error(msg: &str) -> bool {
859 let m = msg.to_lowercase();
860 (m.contains("could not parse") && m.contains("as dtype"))
861 || m.contains("invalid primitive value")
862}
863
864fn extract_csv_parse_column(msg: &str) -> Option<String> {
866 let m = msg.to_lowercase();
867 for (needle, quote) in [("at column '", '\''), ("at column \"", '"')] {
868 if let Some(start) = m.find(needle) {
869 let after = &msg[start + needle.len()..];
870 let end = after.find(quote)?;
871 let name = after[..end].trim();
872 if !name.is_empty() {
873 return Some(name.to_string());
874 }
875 }
876 }
877 None
878}
879
880fn short_csv_parse_error_message(raw: &str) -> String {
881 let col = extract_csv_parse_column(raw);
882 let first = match &col {
883 Some(c) => format!(
884 "CSV parse error in column \"{}\": a value didn't match the inferred type.",
885 c
886 ),
887 None => "CSV parse error: a value didn't match the inferred column type.".to_string(),
888 };
889 format!(
890 "{}\n\
891 Try: --infer-rows 1000\n\
892 --null <value> (treat as null)\n\
893 --ignore-errors (skip bad rows)",
894 first
895 )
896}
897
898#[cfg(test)]
899mod tests {
900 use super::*;
901
902 #[test]
905 fn keys_keep_their_case() {
906 for (what, said) in [
907 ("not a WAV file", "Not a WAV file."),
908 ("could not read it: gone", "Could not read it: gone."),
909 ("type: expected u1", "type: expected u1."),
910 (
911 "tags.nine: expected a tag number",
912 "tags.nine: expected a tag number.",
913 ),
914 (
915 "header_rows: missing `name`",
916 "header_rows: missing `name`.",
917 ),
918 (
919 "\"day\" is made from \"date\"",
920 "\"day\" is made from \"date\".",
921 ),
922 ("u9 is not a type", "u9 is not a type."),
923 ("`colour` is not a key", "`colour` is not a key."),
924 ] {
925 assert_eq!(sentence(what), said, "{what}");
926 }
927 assert_eq!(
928 located_message(
929 Some(Path::new("d.toml")),
930 Some((3, 1)),
931 "tags.x: expected a tag number"
932 ),
933 "\"d.toml\":3:1: tags.x: expected a tag number."
934 );
935 }
936
937 #[test]
939 fn a_file_error_names_the_file_once() {
940 let path = Path::new("/d/a.wav");
941 for what in [
942 "not a WAV file",
943 "Not a WAV file.",
944 "/d/a.wav: not a WAV file",
945 "\"/d/a.wav\": Not a WAV file.",
946 ] {
947 assert_eq!(file_message(path, what), "\"/d/a.wav\": Not a WAV file.");
948 }
949 assert_eq!(
950 file_message(path, "bad\nTry: --format csv"),
951 "\"/d/a.wav\": Bad.\nTry: --format csv"
952 );
953 let err = in_file(path, color_eyre::eyre::eyre!("too short"));
955 let err = in_file(Path::new("/d"), err);
956 assert_eq!(err.to_string(), "\"/d/a.wav\": Too short.");
957 assert_eq!(
958 user_message_from_report(&err, Some(Path::new("/elsewhere"))),
959 "\"/d/a.wav\": Too short."
960 );
961 let missing = in_file(path, io::Error::new(io::ErrorKind::NotFound, "gone").into());
962 assert!(matches!(
963 error_for_python(&missing),
964 (ErrorKindForPython::FileNotFound, ref m) if m.starts_with("\"/d/a.wav\": ")
965 ));
966 assert_eq!(
967 user_message_from_report(&color_eyre::eyre::eyre!("bad header"), Some(path)),
968 "\"/d/a.wav\": Bad header."
969 );
970 }
971
972 #[test]
974 fn rust_names_are_said_plainly() {
975 let said = |m: &str| rust_names_said_plainly(m);
976 assert!(
977 said("out-of-spec: InvalidFooter")
978 .unwrap()
979 .contains("(invalid footer)")
980 );
981 assert!(said("OutOfSpec").unwrap().contains("--format"));
982 assert_eq!(
983 said("InvalidUtf8 at character 0").unwrap(),
984 "The file is not UTF-8 text at character 0."
985 );
986 assert_eq!(said("out-of-spec: the footer is short"), None);
987 assert_eq!(said("bad header"), None);
988 }
989
990 #[test]
992 fn a_pattern_that_matched_nothing_says_so() {
993 let csv = "failed to retrieve file schemas (csv): expanded paths were empty \
994 (path expansion input: 'paths: [PlRefPath { inner: \"/d/x?.csv\" }]', \
995 glob: true).";
996 let err = PolarsError::ComputeError(csv.into());
997 assert_eq!(
998 user_message_from_polars(&err),
999 "No files match this pattern."
1000 );
1001 let dir = "failed to retrieve first file schema (parquet): expanded paths were \
1002 empty (path expansion input: 'paths: [PlRefPath { inner: \"/d/empty\" }]', \
1003 glob: true). Hint: passing a schema can allow this scan to succeed.";
1004 let err = PolarsError::ComputeError(dir.into());
1005 assert_eq!(user_message_from_polars(&err), "No files found there.");
1006 }
1007
1008 #[test]
1011 fn a_temp_copy_is_named_by_its_source() {
1012 let file = Path::new("/home/u/tmp/.tmp9tY5X2.parquet");
1013 let url = Path::new("http://host/broken.parquet");
1014 let said = named_by_source(
1015 "\"/home/u/tmp/.tmp9tY5X2.parquet\": Bad.\nIt stopped at /home/u/tmp/.tmp9tY5X2.parquet.",
1016 file,
1017 url,
1018 );
1019 assert_eq!(
1020 said,
1021 "\"http://host/broken.parquet\": Bad.\nIt stopped at http://host/broken.parquet."
1022 );
1023 assert_eq!(named_by_source("no path here", file, url), "no path here");
1024 assert_eq!(named_by_source("x", Path::new(""), url), "x");
1025 assert_eq!(
1026 named_by_source("'/home/u/tmp/.tmp9tY5X2.parquet' (os error 2)", file, url),
1027 "'http://host/broken.parquet' (os error 2)"
1028 );
1029 }
1030
1031 #[test]
1034 fn a_similar_path_is_not_renamed() {
1035 let copy = Path::new("/home/u/tmp/.tmpAb12Cd");
1036 let gz = Path::new("/data/rows.csv.gz");
1037 for other in [
1038 "/home/u/tmp/.tmpAb12Cd.csv",
1039 "/home/u/tmp/.tmpAb12Cd2",
1040 "/home/u/tmp/.tmpAb12Cd_old",
1041 "/home/u/tmp/.tmpAb12Cd/part-0.csv",
1042 "/mnt/home/u/tmp/.tmpAb12Cd",
1043 "x/home/u/tmp/.tmpAb12Cd",
1044 ] {
1045 let message = format!("\"{other}\": Bad.");
1046 assert_eq!(named_by_source(&message, copy, gz), message, "{other}");
1047 }
1048 assert_eq!(
1049 named_by_source(
1050 "/home/u/tmp/.tmpAb12Cd.csv is not /home/u/tmp/.tmpAb12Cd.",
1051 copy,
1052 gz
1053 ),
1054 "/home/u/tmp/.tmpAb12Cd.csv is not /data/rows.csv.gz.",
1055 );
1056 }
1057
1058 #[test]
1061 fn files_whose_columns_differ_are_said_as_files() {
1062 let said = user_message_from_polars(&PolarsError::ComputeError(
1063 "schema names differ: got 39, expected 25".into(),
1064 ));
1065 assert!(said.contains("cannot be read as one table"), "{said}");
1066 assert!(
1067 said.contains("a column named \"39\" where another has \"25\""),
1068 "{said}"
1069 );
1070 assert!(said.contains("--no-header"), "{said}");
1071 assert!(!said.contains("schema"), "no Polars words left: {said}");
1072
1073 let said =
1074 user_message_from_polars(&PolarsError::ComputeError("schema lengths differ".into()));
1075 assert!(said.contains("different numbers of columns"), "{said}");
1076
1077 let other =
1078 user_message_from_polars(&PolarsError::ComputeError("something else entirely".into()));
1079 assert!(!other.contains("one table"), "{other}");
1080 }
1081
1082 #[test]
1088 fn a_polars_query_plan_is_not_shown_to_the_user() {
1089 let raw = "Operation not allowed: 'union'/'concat' inputs should all have the \
1090 same schema,got\nSchema { fields: {\"a\": Int64, \"b\": Int64} } and \
1091 \nSchema { fields: {\"q\": Int64} }\n\nResolved plan until failure:\n\n\
1092 \t---> FAILED HERE RESOLVING THIS_NODE <---\nCsv SCAN \
1093 [/data/mixed/two.csv]\nPROJECT */3 COLUMNS\nESTIMATED ROWS: 2";
1094
1095 let said = union_schema_message(raw);
1096 assert!(
1097 !said.contains("FAILED HERE") && !said.contains("PROJECT"),
1098 "the plan is gone: {said:?}"
1099 );
1100 assert!(
1101 !said.contains("Schema {"),
1102 "and so are two schemas printed in full: {said:?}"
1103 );
1104 assert!(
1105 said.contains("/data/mixed/two.csv"),
1106 "but the file it stopped at is kept: {said:?}"
1107 );
1108
1109 let other = "Column not found: region\n\nResolved plan until failure:\n\n\
1111 \t---> FAILED HERE RESOLVING THIS_NODE <---\nParquet SCAN \
1112 [/data/events/part-7.parquet]\nPROJECT 3/9 COLUMNS";
1113 let tidied = without_the_query_plan(other);
1114 assert_eq!(
1115 tidied, "Column not found: region\nIt stopped at /data/events/part-7.parquet.",
1116 "got {tidied:?}"
1117 );
1118
1119 assert_eq!(
1121 without_the_query_plan("Column not found: region"),
1122 "Column not found: region"
1123 );
1124
1125 let two_scans = "Column not found: region\n\nResolved plan until failure:\n\n Parquet SCAN [/data/a.parquet]\nUNION\n \t---> FAILED HERE RESOLVING THIS_NODE <---\n Csv SCAN [/data/b.csv]";
1127 assert!(
1128 without_the_query_plan(two_scans).ends_with("It stopped at /data/b.csv."),
1129 "got {:?}",
1130 without_the_query_plan(two_scans)
1131 );
1132
1133 assert!(
1136 !is_union_schema_error("unable to vstack, column names don't match: \"a\" and \"b\""),
1137 "a single-file vstack must keep its own message"
1138 );
1139 }
1140
1141 #[test]
1142 fn test_user_message_from_io_not_found() {
1143 let err = io::Error::new(io::ErrorKind::NotFound, "No such file");
1144 let msg = user_message_from_io(&err, None);
1145 assert!(
1146 msg.contains("not found"),
1147 "expected 'not found', got: {}",
1148 msg
1149 );
1150 }
1151
1152 #[test]
1156 fn a_file_another_program_holds_says_so() {
1157 let path = Path::new(r"C:\data\book.xlsx");
1158 let held = format!(
1159 "\"{}\": The file is open in another program that does not allow reading it; \
1160 close it there and reopen.",
1161 path.display()
1162 );
1163 let raw = || io::Error::from_raw_os_error(32);
1164 let rewrapped = || {
1165 io::Error::other(format!(
1166 "The process cannot access the file because it is being used by another \
1167 process. (os error 32): {}",
1168 path.display()
1169 ))
1170 };
1171 for report in [
1172 color_eyre::eyre::Report::new(raw()),
1173 color_eyre::eyre::Report::new(PolarsError::from(rewrapped())),
1174 color_eyre::eyre::Report::new(PolarsError::from(raw()).context("scan".into())),
1175 color_eyre::eyre::eyre!("{}", PolarsError::from(rewrapped())),
1176 ] {
1177 assert_eq!(
1178 report_message(true, &report, Some(path)),
1179 held,
1180 "{report:?}"
1181 );
1182 if !cfg!(windows) {
1185 let elsewhere = report_message(false, &report, Some(path));
1186 assert!(!elsewhere.contains("another program"), "{elsewhere}");
1187 }
1188 }
1189 assert!(held_on(true, &raw()));
1190 assert!(held_on(true, &io::Error::from_raw_os_error(33)));
1191 assert!(!held_on(true, &io::Error::from_raw_os_error(5)));
1192 assert!(!held_on(false, &raw()));
1194 }
1195
1196 #[test]
1197 fn test_user_message_from_io_permission_denied() {
1198 let err = io::Error::new(io::ErrorKind::PermissionDenied, "Permission denied");
1199 let msg = user_message_from_io(&err, None);
1200 assert!(
1201 msg.to_lowercase().contains("permission"),
1202 "expected 'permission', got: {}",
1203 msg
1204 );
1205 }
1206
1207 #[test]
1208 fn test_user_message_from_polars_column_not_found() {
1209 use polars::prelude::PolarsError;
1210 let err = PolarsError::ColumnNotFound("foo".into());
1211 let msg = user_message_from_polars(&err);
1212 assert!(msg.contains("foo"), "expected 'foo', got: {}", msg);
1213 assert!(
1214 msg.contains("Column not found"),
1215 "expected column not found, got: {}",
1216 msg
1217 );
1218 }
1219
1220 #[test]
1221 fn test_user_message_from_polars_duplicate() {
1222 use polars::prelude::PolarsError;
1223 let err = PolarsError::Duplicate("bar".into());
1224 let msg = user_message_from_polars(&err);
1225 assert!(
1226 msg.contains("Duplicate"),
1227 "expected 'Duplicate', got: {}",
1228 msg
1229 );
1230 assert!(msg.contains("alias"), "expected alias hint, got: {}", msg);
1231 }
1232
1233 #[test]
1234 fn test_simplify_compute_message_alias_hint() {
1235 let raw = "projections contained duplicate: 'x'. Try renaming with .alias(\"name\")";
1236 let msg = simplify_compute_message(raw);
1237 assert!(
1238 !msg.contains(".alias("),
1239 "should strip .alias( hint: {}",
1240 msg
1241 );
1242 assert!(
1243 msg.contains("Use aliases"),
1244 "expected alias suggestion: {}",
1245 msg
1246 );
1247 }
1248
1249 #[test]
1250 fn test_simplify_compute_message_csv_parse_error() {
1251 let raw = "could not parse `N/A` as dtype `i64` at column 'column' (column number 1)\n\n\
1252 The current offset in the file is 292 bytes.\n\n\
1253 You might want to try: ...\n\
1254 Original error: ```invalid primitive value found during CSV parsing```";
1255 let msg = simplify_compute_message(raw);
1256 assert!(
1257 msg.contains("CSV parse error"),
1258 "expected short CSV message: {}",
1259 msg
1260 );
1261 assert!(
1262 msg.contains("column \"column\""),
1263 "expected offending column in message: {}",
1264 msg
1265 );
1266 assert!(msg.contains("--infer-rows"), "expected CLI hint: {}", msg);
1267 assert!(msg.contains("--null"), "expected null-value hint: {}", msg);
1268 assert!(
1269 !msg.contains("Original error"),
1270 "should not regurgitate Polars: {}",
1271 msg
1272 );
1273 }
1274
1275 #[cfg(feature = "sql")]
1277 fn sql_failure(sql: &str, df: polars::prelude::DataFrame) -> PolarsError {
1278 use polars::prelude::IntoLazy;
1279 let mut ctx = polars_sql::SQLContext::new();
1280 ctx.register("df", df.lazy());
1281 ctx.execute(sql)
1282 .expect("plans")
1283 .collect()
1284 .expect_err("fails at run time")
1285 }
1286
1287 #[cfg(feature = "sql")]
1289 fn matches() -> polars::prelude::DataFrame {
1290 let dates: Vec<String> = (0..380)
1291 .map(|i| match i % 30 {
1292 0 => "Tue Jan 12 2021(P)".to_string(),
1293 10 => "Sat Feb 20 2021(P)".to_string(),
1294 _ => "Sun Sep 13 2020".to_string(),
1295 })
1296 .collect();
1297 let scores: Vec<String> = (0..380)
1298 .map(|i| if i % 50 == 0 { "n/a" } else { "3" }.to_string())
1299 .collect();
1300 polars::prelude::df!("Date" => dates, "Team 1" => scores).unwrap()
1301 }
1302
1303 #[cfg(feature = "sql")]
1304 #[test]
1305 fn a_date_that_does_not_parse_is_said_in_sql_terms() {
1306 let err = sql_failure(
1307 "SELECT STRPTIME(Date, '%a %b %d %Y') AS d FROM df",
1308 matches(),
1309 );
1310 let failure = conversion_failure(&err).expect("a conversion");
1311 assert_eq!(failure.column, "Date");
1312 assert!(failure.parsing);
1313 assert_eq!(failure.failed, 26);
1314 let msg = sql_error_message(&err, Some(380));
1315 assert!(
1316 msg.starts_with(
1317 "Date: 26 of 380 values do not match the format, such as \"Tue Jan 12 2021(P)\""
1318 ),
1319 "{msg}"
1320 );
1321 assert!(msg.contains("SUBSTR(Date, 1, n)"), "{msg}");
1322 assert!(
1323 !msg.contains("strict=False") && !msg.contains("str.strptime"),
1324 "{msg}"
1325 );
1326 let msg = sql_error_message(&err, Some(1000));
1328 assert!(
1329 msg.starts_with("At least 26 values in Date do not match"),
1330 "{msg}"
1331 );
1332 }
1333
1334 #[cfg(feature = "sql")]
1335 #[test]
1336 fn a_cast_that_fails_names_the_column_and_suggests_try_cast() {
1337 let err = sql_failure(
1338 "SELECT CAST(\"Team 1\" AS INT) + 1 AS goals FROM df ORDER BY goals",
1339 matches(),
1340 );
1341 let msg = sql_error_message(&err, Some(380));
1342 assert!(
1343 msg.starts_with("\"Team 1\": 8 of 380 values are not whole numbers, such as \"n/a\"."),
1344 "{msg}"
1345 );
1346 assert!(msg.contains("TRY_CAST(\"Team 1\" AS INT)"), "{msg}");
1347 }
1348
1349 #[test]
1351 fn a_count_short_of_the_table_is_a_lower_bound() {
1352 let failure = ConversionFailure {
1353 column: "FT".to_string(),
1354 to: "i32".to_string(),
1355 failed: 1,
1356 checked: 1,
1357 examples: vec!["0–3".to_string()],
1358 parsing: false,
1359 };
1360 let lead = |rows| {
1361 failure
1362 .sql_message(rows)
1363 .lines()
1364 .next()
1365 .unwrap()
1366 .to_string()
1367 };
1368 assert_eq!(
1369 lead(None),
1370 "At least 1 value in FT is not a whole number, such as \"0–3\"."
1371 );
1372 assert_eq!(
1373 lead(Some(1)),
1374 "FT: 1 of 1 values are not whole numbers, such as \"0–3\"."
1375 );
1376 }
1377
1378 #[test]
1379 fn anything_else_is_said_as_polars_says_it() {
1380 let err = PolarsError::InvalidOperation("something else".into());
1381 assert_eq!(conversion_failure(&err), None);
1382 assert_eq!(
1383 sql_error_message(&err, None),
1384 user_message_from_polars(&err)
1385 );
1386 }
1387}