datui_lib/formats/gps/
mod.rs

1//! GPS logs (NMEA 0183, GPX) as a table. Neither scans in place, so an open reads the
2//! file once in pieces, writing rows a batch at a time to a temp Arrow IPC file (claimed
3//! via the open's `Writer`) that the dataset scans lazily. A GPX extension field
4//! appearing late starts a new segment file with the wider schema, joined with nulls;
5//! NMEA is always one segment. Readers are incremental, so a growing log can be
6//! followed later.
7
8pub mod gpx;
9pub mod nmea;
10
11use std::path::{Path, PathBuf};
12use std::sync::atomic::AtomicU64;
13
14use color_eyre::Result;
15use color_eyre::eyre::eyre;
16use polars::prelude::*;
17
18use crate::cloud::download::TempDownload;
19use crate::formats::segments::Converted;
20use crate::formats::text_formats::{count, note, read_through};
21use crate::loading::unfinished::Writer;
22use crate::notes::Note;
23use crate::numfmt::group_chrome;
24use crate::{FileFormat, OpenOptions};
25
26/// What datui does with an NMEA log: see [`crate::formats::readers`].
27pub(crate) const NMEA: crate::formats::readers::Reader = crate::formats::readers::Reader {
28    // Several logs are one table.
29    convert: Some(|input| {
30        let (converted, detail) = convert(
31            input.files,
32            input.display,
33            input.format,
34            input.options,
35            input.writer,
36            input.read,
37        )?;
38        Ok((converted, Some(std::sync::Arc::new(detail))))
39    }),
40    scan: crate::formats::readers::read_into,
41    signatures: &[crate::formats::readers::Signature {
42        says: |head, _| nmea::looks_like(head),
43        kind: crate::formats::readers::Kind::Text,
44        trusted: crate::formats::readers::Trusted {
45            listing: false,
46            tables: true,
47            ..crate::formats::readers::EVERYWHERE
48        },
49    }],
50    // The tables are the sentence types read, whichever the log holds: listing them
51    // reads nothing.
52    tables: Some(|_| Ok(nmea_tables())),
53    ..crate::formats::readers::BASE
54};
55
56/// The tables of an NMEA log, as the home screen lists them and `--table` names them.
57fn nmea_tables() -> Vec<crate::formats::sqlite::Table> {
58    nmea::Table::ALL
59        .into_iter()
60        .map(|t| {
61            let columns = t.columns().into_iter().map(|(name, _)| name);
62            crate::formats::sqlite::Table::plain(t.name(), "table", columns)
63        })
64        .collect()
65}
66
67/// What datui does with a GPX file: see [`crate::formats::readers`].
68pub(crate) const GPX: crate::formats::readers::Reader = crate::formats::readers::Reader {
69    // Several logs are one table.
70    convert: Some(|input| {
71        let (converted, detail) = convert(
72            input.files,
73            input.display,
74            input.format,
75            input.options,
76            input.writer,
77            input.read,
78        )?;
79        Ok((converted, Some(std::sync::Arc::new(detail))))
80    }),
81    scan: crate::formats::readers::read_into,
82    signatures: &[crate::formats::readers::Signature {
83        says: |head, _| gpx::looks_like(head),
84        kind: crate::formats::readers::Kind::Text,
85        trusted: crate::formats::readers::Trusted {
86            listing: false,
87            ..crate::formats::readers::EVERYWHERE
88        },
89    }],
90    ..crate::formats::readers::BASE
91};
92
93/// Read `files` (shown as `display`) as `format` into temp IPC files via `writer`,
94/// counting bytes read in `read`. Several files form one table with a leading `file`
95/// column; read notes are summed for Info's GPS tab.
96pub(crate) fn convert(
97    files: &[PathBuf],
98    display: &Path,
99    format: FileFormat,
100    options: &OpenOptions,
101    writer: &Writer,
102    read: &AtomicU64,
103) -> Result<(Converted, crate::formats::text_formats::Detail)> {
104    let [file] = files else {
105        return convert_many(files, format, options, writer, read);
106    };
107    let one = convert_one(file, display, format, options, writer, read)?;
108    let (notes, other_tables) = one.stats.notes(None, options)?;
109    let detail = one.stats.detail(None, &one.extent, options)?;
110    Ok((
111        Converted {
112            lf: one.lf,
113            files: one.files,
114            notes,
115            other_tables,
116        },
117        detail,
118    ))
119}
120
121/// Each of `files` read as [`convert_one`] reads it, then stacked under a `file`
122/// column. A GPX file's fields are typed in that file, and stacked as one type where
123/// two files disagree.
124fn convert_many(
125    files: &[PathBuf],
126    format: FileFormat,
127    options: &OpenOptions,
128    writer: &Writer,
129    read: &AtomicU64,
130) -> Result<(Converted, crate::formats::text_formats::Detail)> {
131    let mut frames = Vec::with_capacity(files.len());
132    let mut written = Vec::new();
133    let mut stats: Option<Stats> = None;
134    let mut extent = Extent::default();
135    for file in files {
136        let one = convert_one(file, file, format, options, writer, read)
137            .map_err(|e| crate::error_display::in_file(file, e))?;
138        let name = file
139            .file_name()
140            .map(|n| n.to_string_lossy().into_owned())
141            .unwrap_or_else(|| file.display().to_string());
142        frames.push(one.lf.select([lit(name).alias("file"), all().as_expr()]));
143        written.extend(one.files);
144        extent.merge(&one.extent);
145        match &mut stats {
146            None => stats = Some(one.stats),
147            Some(stats) => stats.absorb(one.stats),
148        }
149    }
150    let stats = stats.ok_or_else(|| eyre!("No GPS logs to read."))?;
151    let lf = concat(
152        &frames,
153        UnionArgs {
154            diagonal: true,
155            to_supertypes: true,
156            ..Default::default()
157        },
158    )?;
159    let (notes, other_tables) = stats.notes(Some(files.len()), options)?;
160    let detail = stats.detail(Some(files.len()), &extent, options)?;
161    Ok((
162        Converted {
163            lf,
164            files: written,
165            notes,
166            other_tables,
167        },
168        detail,
169    ))
170}
171
172/// What one log's read noticed, of either format.
173enum Stats {
174    Gpx {
175        stats: gpx::Stats,
176        /// Files that end inside an element.
177        truncated: usize,
178    },
179    Nmea {
180        stats: nmea::Stats,
181        /// Files with rows and no sentence that gives the date.
182        undated: usize,
183    },
184}
185
186impl Stats {
187    /// Count `other`, another file's, in with these.
188    fn absorb(&mut self, other: Stats) {
189        match (self, other) {
190            (
191                Stats::Gpx { stats, truncated },
192                Stats::Gpx {
193                    stats: more,
194                    truncated: also,
195                },
196            ) => {
197                stats.points += more.points;
198                stats.tracks += more.tracks;
199                stats.routes += more.routes;
200                stats.waypoints += more.waypoints;
201                stats.fields_dropped += more.fields_dropped;
202                stats.long_names += more.long_names;
203                stats.bad_times += more.bad_times;
204                stats.truncated |= more.truncated;
205                *truncated += also;
206            }
207            (
208                Stats::Nmea { stats, undated },
209                Stats::Nmea {
210                    stats: more,
211                    undated: also,
212                },
213            ) => {
214                stats.absorb(&more);
215                *undated += also;
216            }
217            // One format per open.
218            _ => {}
219        }
220    }
221
222    /// The notes for the Notes tab, and the other tables for the Schema tab, of `of`
223    /// files (`None` for one).
224    fn notes(&self, of: Option<usize>, options: &OpenOptions) -> Result<(Vec<Note>, Vec<String>)> {
225        Ok(match self {
226            Stats::Gpx { stats, truncated } => (gpx_notes(stats, of, *truncated), Vec::new()),
227            Stats::Nmea { stats, undated } => {
228                let table = nmea_table(options)?;
229                (
230                    nmea_notes(stats, of, *undated),
231                    nmea_other_tables(stats, table),
232                )
233            }
234        })
235    }
236}
237
238impl Stats {
239    /// The GPS tab of the Info panel, for `of` files (`None` for one) whose rows
240    /// covered `extent`.
241    fn detail(
242        &self,
243        of: Option<usize>,
244        extent: &Extent,
245        options: &OpenOptions,
246    ) -> Result<crate::formats::text_formats::Detail> {
247        use crate::formats::model_files::MetaValue;
248        let middot = crate::glyphs::get().middot;
249        let mut lines = Vec::new();
250        let (list_title, list) = match self {
251            Stats::Gpx { stats, .. } => {
252                lines.push(
253                    [
254                        count(stats.points, "point", "points"),
255                        count(stats.tracks, "track", "tracks"),
256                        count(stats.routes, "route", "routes"),
257                        count(stats.waypoints, "waypoint", "waypoints"),
258                    ]
259                    .join(&format!(" {middot} ")),
260                );
261                ("", Vec::new())
262            }
263            Stats::Nmea { stats, .. } => {
264                let table = nmea_table(options)?;
265                lines.push(format!(
266                    "{} of {} {middot} {} of {}",
267                    count(stats.rows, "row", "rows"),
268                    table.name(),
269                    count(stats.sentences, "sentence", "sentences"),
270                    count(stats.lines, "line", "lines"),
271                ));
272                let mut list: Vec<(String, MetaValue)> = stats
273                    .types
274                    .iter()
275                    .map(|(t, n)| (t.clone(), MetaValue::Text(group_chrome(*n as usize))))
276                    .collect();
277                if stats.other_types > 0 {
278                    list.push((
279                        "other".to_string(),
280                        MetaValue::Text(group_chrome(stats.other_types as usize)),
281                    ));
282                }
283                ("Sentences", list)
284            }
285        };
286        if let Some(n) = of {
287            lines.push(count(n as u64, "file", "files"));
288        }
289        lines.extend(extent.lines());
290        Ok(crate::formats::text_formats::Detail {
291            tab: crate::formats::text_formats::tab(FileFormat::Nmea),
292            lines,
293            list_title,
294            list,
295            ..Default::default()
296        })
297    }
298}
299
300/// When and where a log's rows are: the least and most of their `time`, `lat` and
301/// `lon`, taken from each batch as it is written, so nothing is read twice.
302#[derive(Debug, Clone, Default, PartialEq)]
303struct Extent {
304    /// Milliseconds since the epoch, UTC.
305    time: Option<(i64, i64)>,
306    lat: Option<(f64, f64)>,
307    lon: Option<(f64, f64)>,
308}
309
310impl Extent {
311    fn absorb(&mut self, df: &DataFrame) {
312        let time = df
313            .column("time")
314            .ok()
315            .and_then(|c| c.as_materialized_series().datetime().ok().cloned())
316            .and_then(|t| Some((t.physical().min()?, t.physical().max()?)));
317        let span = |name: &str| {
318            let c = df.column(name).ok()?;
319            let v = c.as_materialized_series().f64().ok()?;
320            Some((v.min()?, v.max()?))
321        };
322        self.merge(&Extent {
323            time,
324            lat: span("lat"),
325            lon: span("lon"),
326        });
327    }
328
329    fn merge(&mut self, other: &Extent) {
330        fn wider<T: PartialOrd + Copy>(a: Option<(T, T)>, b: Option<(T, T)>) -> Option<(T, T)> {
331            match (a, b) {
332                (Some((a0, a1)), Some((b0, b1))) => {
333                    Some((if b0 < a0 { b0 } else { a0 }, if b1 > a1 { b1 } else { a1 }))
334                }
335                (a, b) => a.or(b),
336            }
337        }
338        self.time = wider(self.time, other.time);
339        self.lat = wider(self.lat, other.lat);
340        self.lon = wider(self.lon, other.lon);
341    }
342
343    /// `Time: 2024-05-01 12:00:00 to 13:05:12 UTC (1:05:12.000)` and
344    /// `Bounds: 51.40000 to 51.60000 N, -0.20000 to 0.10000 E`.
345    fn lines(&self) -> Vec<String> {
346        let mut lines = Vec::new();
347        if let Some((first, last)) = self.time {
348            let at = |ms: i64| chrono::DateTime::from_timestamp_millis(ms);
349            if let (Some(a), Some(b)) = (at(first), at(last)) {
350                let end = if a.date_naive() == b.date_naive() {
351                    b.format("%H:%M:%S").to_string()
352                } else {
353                    b.format("%Y-%m-%d %H:%M:%S").to_string()
354                };
355                lines.push(format!(
356                    "Time: {} to {end} UTC ({})",
357                    a.format("%Y-%m-%d %H:%M:%S"),
358                    crate::widgets::info::clock((last - first) as f64 / 1000.0)
359                ));
360            }
361        }
362        if let (Some((lat0, lat1)), Some((lon0, lon1))) = (self.lat, self.lon) {
363            lines.push(format!(
364                "Bounds: latitude {lat0:.5} to {lat1:.5}, longitude {lon0:.5} to {lon1:.5}"
365            ));
366        }
367        lines
368    }
369}
370
371/// The NMEA table `--table` names, the fixes when it names none.
372fn nmea_table(options: &OpenOptions) -> Result<nmea::Table> {
373    match options.table.as_deref() {
374        None => Ok(nmea::Table::Fixes),
375        Some(name) => nmea::Table::from_name(name).ok_or_else(|| {
376            eyre!(
377                "no table \"{name}\". --table names one of an NMEA log's tables: {}.",
378                nmea::Table::ALL.map(nmea::Table::name).join(", ")
379            )
380        }),
381    }
382}
383
384/// One log read: the frame over its segments, the files they are, and its counts.
385struct One {
386    lf: LazyFrame,
387    files: Vec<TempDownload>,
388    stats: Stats,
389    extent: Extent,
390}
391
392/// Read `file` (named `display` to the user) as `format` into temporary IPC files.
393fn convert_one(
394    file: &Path,
395    display: &Path,
396    format: FileFormat,
397    options: &OpenOptions,
398    writer: &Writer,
399    read: &AtomicU64,
400) -> Result<One> {
401    let mut extent = Extent::default();
402    match format {
403        FileFormat::Gpx => {
404            let mut gpx = gpx::GpxReader::new();
405            let (lf, files) = read_through(file, options, writer, read, &mut gpx, |df| {
406                extent.absorb(df)
407            })?;
408            let stats = gpx.stats().clone();
409            Ok(One {
410                lf: type_gpx_fields(lf, gpx.fields()),
411                files,
412                extent,
413                stats: Stats::Gpx {
414                    truncated: usize::from(stats.truncated),
415                    stats,
416                },
417            })
418        }
419        FileFormat::Nmea => {
420            let mut log = nmea::NmeaReader::new(nmea_table(options)?);
421            let (lf, files) = read_through(file, options, writer, read, &mut log, |df| {
422                extent.absorb(df)
423            })?;
424            if log.stats().sentences == 0 {
425                return Err(crate::error_display::FileError::new(
426                    display,
427                    "no NMEA sentences: no line starts with $ and a sentence address.",
428                )
429                .into());
430            }
431            let stats = log.stats().clone();
432            Ok(One {
433                lf,
434                files,
435                extent,
436                stats: Stats::Nmea {
437                    undated: usize::from(!stats.dated && stats.rows > 0),
438                    stats,
439                },
440            })
441        }
442        other => Err(eyre!("{} is not a GPS format.", other.name())),
443    }
444}
445
446impl crate::formats::text_formats::BatchReader for gpx::GpxReader {
447    fn push(&mut self, piece: &[u8]) -> Result<()> {
448        self.push(piece).map_err(|e| eyre!(e))
449    }
450
451    fn take_batch(&mut self) -> PolarsResult<Option<DataFrame>> {
452        self.take_batch()
453    }
454
455    fn finish(&mut self) -> Result<DataFrame> {
456        self.finish().map_err(|e| eyre!(e))
457    }
458}
459
460impl crate::formats::text_formats::BatchReader for nmea::NmeaReader {
461    fn push(&mut self, piece: &[u8]) -> Result<()> {
462        self.push(piece);
463        Ok(())
464    }
465
466    fn take_batch(&mut self) -> PolarsResult<Option<DataFrame>> {
467        self.take_batch()
468    }
469
470    fn finish(&mut self) -> Result<DataFrame> {
471        Ok(self.finish()?)
472    }
473}
474
475/// A GPX field column as numbers when every value in it was one.
476fn type_gpx_fields(lf: LazyFrame, fields: &[gpx::FieldColumn]) -> LazyFrame {
477    let casts: Vec<Expr> = fields
478        .iter()
479        .filter_map(|f| {
480            let to = if f.integers {
481                DataType::Int64
482            } else if f.numbers {
483                DataType::Float64
484            } else {
485                return None;
486            };
487            Some(col(f.name.as_str()).cast(to))
488        })
489        .collect();
490    if casts.is_empty() {
491        lf
492    } else {
493        lf.with_columns(casts)
494    }
495}
496
497/// What reading an NMEA log noticed; of `of` logs, `undated` of them without a date.
498fn nmea_notes(stats: &nmea::Stats, of: Option<usize>, undated: usize) -> Vec<Note> {
499    let mut notes = Vec::new();
500    let of_lines = match of {
501        None => format!("of {}", count(stats.lines, "line", "lines")),
502        Some(n) => format!(
503            "of {} in {}",
504            count(stats.lines, "line", "lines"),
505            count(n as u64, "log", "logs")
506        ),
507    };
508    if stats.skipped > 0 {
509        notes.push(note(
510            format!(
511                "{} left out: not NMEA",
512                count(stats.skipped, "line", "lines")
513            ),
514            of_lines.clone(),
515        ));
516    }
517    if stats.bad_checksums > 0 {
518        notes.push(note(
519            format!(
520                "checksum failed: {} {} checksum_ok false",
521                count(stats.bad_checksums, "sentence", "sentences"),
522                crate::glyphs::get().middot
523            ),
524            format!("of {}", count(stats.sentences, "sentence", "sentences")),
525        ));
526    }
527    if undated > 0 {
528        let summary = match of {
529            None => format!(
530                "no RMC or ZDA date {} time empty",
531                crate::glyphs::get().middot
532            ),
533            Some(n) => format!(
534                "{undated} of {n} logs without an RMC or ZDA date {} time empty",
535                crate::glyphs::get().middot
536            ),
537        };
538        notes.push(note(summary, of_lines.clone()));
539    }
540    notes
541}
542
543/// An NMEA log's other tables (as `--table` names them, with their sentences) when one
544/// holds rows `table` lacks (GSA, GSV, ZDA, or another type's table); empty for a
545/// fixes-only log.
546fn nmea_other_tables(stats: &nmea::Stats, table: nmea::Table) -> Vec<String> {
547    use nmea::Table;
548    let typed = |t: Table| !matches!(t, Table::Fixes | Table::Sentences);
549    let merged = |t: Table| matches!(t, Table::Gga | Table::Rmc | Table::Vtg | Table::Gll);
550    let worth = Table::ALL.into_iter().any(|t| {
551        t != table && typed(t) && stats.of(t.name()) > 0 && !(table == Table::Fixes && merged(t))
552    });
553    if !worth {
554        return Vec::new();
555    }
556    Table::ALL
557        .into_iter()
558        .filter(|t| *t != table)
559        .filter_map(|t| match typed(t) {
560            false => Some(t.name().to_string()),
561            true => {
562                let n = stats.of(t.name());
563                (n > 0).then(|| format!("{} {}", t.name(), group_chrome(n as usize)))
564            }
565        })
566        .collect()
567}
568
569/// What reading a GPX file noticed; of `of` files, `truncated` of them cut short.
570fn gpx_notes(stats: &gpx::Stats, of: Option<usize>, truncated: usize) -> Vec<Note> {
571    let mut notes = Vec::new();
572    let of_points = match of {
573        None => format!("of {}", count(stats.points, "point", "points")),
574        Some(n) => format!(
575            "of {} in {}",
576            count(stats.points, "point", "points"),
577            count(n as u64, "file", "files")
578        ),
579    };
580    if truncated > 0 {
581        let summary = match of {
582            None => format!(
583                "file cut short inside an element {} points before it shown",
584                crate::glyphs::get().middot
585            ),
586            Some(n) => format!(
587                "{truncated} of {n} files cut short inside an element {} points before it shown",
588                crate::glyphs::get().middot
589            ),
590        };
591        notes.push(note(summary, of_points.clone()));
592    }
593    if stats.bad_times > 0 {
594        notes.push(note(
595            format!(
596                "{} left empty: not ISO 8601",
597                count(stats.bad_times, "time", "times")
598            ),
599            of_points.clone(),
600        ));
601    }
602    if stats.fields_dropped > 0 {
603        notes.push(note(
604            crate::limits::left_out(
605                &count(stats.fields_dropped, "field value", "field values"),
606                crate::limits::get().gpx_fields,
607                "gpx_fields",
608            ),
609            of_points.clone(),
610        ));
611    }
612    if stats.long_names > 0 {
613        notes.push(note(
614            format!(
615                "{} left out: field name too long",
616                count(stats.long_names, "field value", "field values"),
617            ),
618            of_points,
619        ));
620    }
621    notes
622}
623
624#[cfg(test)]
625mod tests;
datui_lib/formats/gps/mod.rs

datui_lib/formats/gps/
mod.rs
