gistools/readers/
mod.rs

1/// CSV Reader
2pub mod csv;
3/// GBFS Readers
4pub mod gbfs;
5/// GeoTIFF Reader
6pub mod geotiff;
7/// GPX Reader
8pub mod gpx;
9/// Grib2 Reader
10pub mod grib2;
11/// GTFS Readers
12pub mod gtfs;
13/// JSON Readers
14pub mod json;
15/// LAS/LAZ Readers
16pub mod las;
17/// NAD Grid Reader
18pub mod nadgrid;
19/// NetCDF Reader
20pub mod netcdf;
21/// OSM (Open Street Map) PBF Reader
22pub mod osm;
23/// (S2)PMTiles Reader
24pub mod pmtiles;
25/// S2 Tiles Reader
26pub mod s2tiles;
27/// Shapefile Reader
28pub mod shapefile;
29/// Tile-based Readers
30pub mod tile;
31/// WKT Geometry Reader
32pub mod wkt;
33
34#[cfg(feature = "std")]
35use crate::parsers::FileReader;
36use crate::parsers::{BufferReader, FeatureReader, Reader};
37use alloc::{boxed::Box, collections::BTreeMap, string::String, vec, vec::Vec};
38use core::fmt::Debug;
39pub use csv::*;
40pub use gbfs::*;
41pub use geotiff::*;
42pub use gpx::*;
43pub use grib2::*;
44pub use gtfs::*;
45pub use image::*;
46pub use json::*;
47pub use las::*;
48pub use nadgrid::*;
49pub use netcdf::*;
50pub use osm::*;
51pub use pmtiles::*;
52use s2json::{MValue, Properties, VectorFeature};
53pub use s2tiles::*;
54use serde::{Deserialize, Serialize};
55pub use shapefile::*;
56#[cfg(feature = "std")]
57use std::path::Path;
58pub use tile::*;
59pub use wkt::*;
60
61/// The type of readers to choose from
62#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
63#[serde(rename_all = "lowercase")]
64pub enum ReaderType {
65    /// CSV data
66    CSV,
67    /// GeoTIFF data
68    GeoTIFF,
69    /// GPX data
70    GPX,
71    /// GRIB 2 data
72    GRIB2,
73    /// GTFS data
74    GTFS,
75    /// JSON data
76    JSON,
77    /// JSON-LD data
78    JSONLD,
79    /// JSON-SQ data
80    JSONSQ,
81    /// LAS data
82    LAS,
83    /// LAZ data
84    LAZ,
85    /// NAD Grid data
86    NADGrid,
87    /// NetCDF data
88    NetCDF,
89    /// OSM data
90    OSM,
91    /// Shapefile
92    Shapefile,
93    /// Tile data
94    Tile,
95    /// WKT
96    WKT,
97}
98impl From<&str> for ReaderType {
99    fn from(value: &str) -> Self {
100        match value {
101            "csv" => ReaderType::CSV,
102            "geotiff" | "tif" | "tiff" | "geotif" => ReaderType::GeoTIFF,
103            "gpx" => ReaderType::GPX,
104            "grib2" | "grib" => ReaderType::GRIB2,
105            "gtfs" => ReaderType::GTFS,
106            "json" | "geojson" | "s2json" => ReaderType::JSON,
107            "jsonld" | "geojsonld" | "s2jsonld" | "json-ld" | "geojson-ld" | "s2json-ld" => {
108                ReaderType::JSONLD
109            }
110            "jsonsq" | "geojsonseq" | "geojsonsq" | "s2jsonseq" | "s2jsonsq" | "json-seq"
111            | "json-sq" | "geojson-seq" | "geojson-sq" | "s2json-seq" | "s2json-sq" => {
112                ReaderType::JSONSQ
113            }
114            "las" => ReaderType::LAS,
115            "laz" => ReaderType::LAZ,
116            "nadgrid" | "nad" | "gsb" => ReaderType::NADGrid,
117            "netcdf" | "nc4" | "cdf" | "nc" => ReaderType::NetCDF,
118            "osm" | "pbf" => ReaderType::OSM,
119            "shapefile" | "shp" | "zip" => ReaderType::Shapefile,
120            "wkt" => ReaderType::WKT,
121            _ => ReaderType::Tile,
122        }
123    }
124}
125
126/// # GIS Reader
127///
128/// ## Description
129/// Parse all data types supported by this library
130///
131/// Implements the [`FeatureReader`] trait
132///
133/// It is recommended to use this reader for testing or ease of access, but the better
134/// alternative is to use the file type readers directly. Here is the list of readers:
135/// - [`CSVReader`]: Parse (Geo|S2)JSON from a file that is in the CSV format
136/// - [`GeoTIFFReader`]: This class reads a GeoTIFF file and returns a list of GeoTIFF images.
137/// - [`GPXReader`]: The GPX Reader is an XML-based GPS Exchange Format (GPX) reader.
138/// - [`GRIB2Reader`]: This class reads a GRIB2 file and returns a list of GRIB2 products.
139/// - [`GTFSScheduleReader`]: Schedule class that pulls in all of the GTFS schedule files and parses them into a single object
140/// - [`JSONReader`]: Parse (Geo|S2)JSON. Can handle millions of features.
141/// - [`NewLineDelimitedJSONReader`]: Parse (Geo|S2)JSON from a file that is in a newline-delimited format
142/// - [`SequenceJSONReader`]: Parse GeoJSON from a file that is in the `geojson-text-sequences` format.
143/// - [`LASReader`]: Reads LAS data. Supports up to the LAS 1.4 specification.
144/// - [`LAZReader`]: Reads LAS zipped data. Supports LAS 1.4 specification although missing some support.
145/// - [`NadGridReader`]: Loads/reads a binary NTv2 file (.gsb) implementing the {@link FeatureIterator} interface.
146/// - [`NetCDFReader`]: Read the NetCDF v3.x file format.
147/// - [`OSMLocalReader`]: OSM PBF Data. Direct use allows for the use of the [`OSMFileReader`] as well
148/// - [`ShapeFileReader`]: Reads data from a shapefile implementing the {@link FeatureIterator} interface
149/// - [`WKTGeometryReader`]: Parse a collection of WKT geometries from a string
150///
151/// ## Usage
152///
153/// ### Read from a file
154/// ```rust
155/// use gistools::{
156///     parsers::{FeatureReader},
157///     readers::{GISReader, ReaderType},
158/// };
159/// use s2json::{MValue, Properties, VectorFeature};
160/// use std::path::PathBuf;
161///
162/// let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
163/// path.push("tests/readers/csv/fixtures/basic.csv");
164///
165/// let reader = GISReader::from_path(path, None, None);
166/// assert_eq!(reader.get_type(), ReaderType::CSV);
167///
168/// let features: Vec<VectorFeature<(), Properties, MValue>> = reader.iter().collect();
169/// ```
170///
171/// ### Read from a buffer
172///
173/// It is recommended to use a Buffer Reader when the file is small because it is more efficient
174///
175/// ```rust
176/// use gistools::{
177///     parsers::{FeatureReader},
178///     readers::{GISReader, ReaderType},
179/// };
180/// use s2json::{MValue, Properties, VectorFeature};
181///
182/// // ignore the use of the filesystem, setup is just for the example
183/// use std::path::PathBuf;
184/// let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
185/// path.push("tests/readers/csv/fixtures/basic.csv");
186/// let bytes = std::fs::read(path.clone()).unwrap();
187///
188/// let reader = GISReader::from_buffer(bytes, ReaderType::CSV, None);
189/// let features: Vec<VectorFeature<(), Properties, MValue>> = reader.iter().collect();
190/// ```
191#[derive(Debug, Clone)]
192pub enum GISReader<T: Reader + Debug> {
193    /// CSV data
194    CSV(Box<CSVReader<T, Properties>>),
195    /// GeoTIFF data
196    GeoTIFF(Box<GeoTIFFReader<T>>),
197    /// GPX data
198    GPX(Box<GPXReader>),
199    /// GRIB 2 data
200    GRIB2(Box<GRIB2Reader>),
201    /// GTFS data
202    GTFS(Box<GTFSScheduleReader>),
203    /// JSON data
204    JSON(Box<JSONReader<T, (), Properties, MValue>>),
205    /// JSON-LD data
206    JSONLD(Box<NewLineDelimitedJSONReader<T, (), Properties, MValue>>),
207    /// JSON-SQ data
208    JSONSQ(Box<SequenceJSONReader<T, (), Properties, MValue>>),
209    /// LAS data
210    LAS(Box<LASReader<T>>),
211    /// LAZ data
212    LAZ(Box<LAZReader<T>>),
213    /// NAD Grid data
214    NADGrid(Box<NadGridReader<T>>),
215    /// NetCDF data
216    NetCDF(Box<NetCDFReader<T>>),
217    /// OSM data
218    OSM(Box<OSMLocalReader<T>>),
219    /// Shapefile
220    Shapefile(Box<ShapeFileReader<T, Properties>>),
221    // /// Tile data
222    // Tile(Box<TileReader<P, D>>),
223    /// WKT
224    WKT(Box<WKTGeometryReader>),
225}
226impl<T: Reader + Debug> GISReader<T> {
227    /// Get the type of the reader
228    pub fn get_type(&self) -> ReaderType {
229        match self {
230            GISReader::CSV(_) => ReaderType::CSV,
231            GISReader::GeoTIFF(_) => ReaderType::GeoTIFF,
232            GISReader::GPX(_) => ReaderType::GPX,
233            GISReader::GRIB2(_) => ReaderType::GRIB2,
234            GISReader::GTFS(_) => ReaderType::GTFS,
235            GISReader::JSON(_) => ReaderType::JSON,
236            GISReader::JSONLD(_) => ReaderType::JSONLD,
237            GISReader::JSONSQ(_) => ReaderType::JSONSQ,
238            GISReader::LAS(_) => ReaderType::LAS,
239            GISReader::LAZ(_) => ReaderType::LAZ,
240            GISReader::NADGrid(_) => ReaderType::NADGrid,
241            GISReader::NetCDF(_) => ReaderType::NetCDF,
242            GISReader::OSM(_) => ReaderType::OSM,
243            GISReader::Shapefile(_) => ReaderType::Shapefile,
244            // GISReader::Tile(_) => ReaderType::Tile,
245            GISReader::WKT(_) => ReaderType::WKT,
246        }
247    }
248}
249impl GISReader<BufferReader> {
250    /// Given a raw data and a file type, return the appropriate reader
251    ///
252    /// ## Parameters
253    ///
254    /// - `data`: The data to parse
255    /// - `file_type`: The file type to parse the data as
256    /// - `epsg_codes`: The EPSG codes to use. E.g. `{"4326": "...WKT STRING..."}`
257    ///
258    /// ## Returns
259    ///
260    /// The [`GISReader`] using a [`BufferReader`] for fast parsing
261    pub fn from_buffer(
262        data: Vec<u8>,
263        file_type: ReaderType,
264        epsg_codes: Option<BTreeMap<String, String>>,
265    ) -> GISReader<BufferReader> {
266        let buffer = BufferReader::new(data);
267        let epsg_codes = epsg_codes.unwrap_or_default();
268        match file_type {
269            ReaderType::CSV => GISReader::CSV(CSVReader::new(buffer, None).into()),
270            ReaderType::GeoTIFF => GISReader::GeoTIFF(
271                GeoTIFFReader::new(buffer, Some(GeoTIFFOptions { epsg_codes })).into(),
272            ),
273            ReaderType::GPX => {
274                let input_str = buffer.parse_string(None, None);
275                GISReader::GPX(GPXReader::new(&input_str).into())
276            }
277            ReaderType::GRIB2 => GISReader::GRIB2(GRIB2Reader::new(buffer.into(), vec![]).into()),
278            ReaderType::GTFS => {
279                GISReader::GTFS(GTFSScheduleReader::from_gzip(&buffer.slice(None, None)).into())
280            }
281            ReaderType::JSON => GISReader::JSON(JSONReader::new(buffer).into()),
282            ReaderType::JSONLD => {
283                GISReader::JSONLD(NewLineDelimitedJSONReader::new(buffer, None).into())
284            }
285            ReaderType::JSONSQ => GISReader::JSONSQ(SequenceJSONReader::new(buffer).into()),
286            ReaderType::LAS => GISReader::LAS(
287                LASReader::new(
288                    buffer,
289                    Some(LASReaderOptions { epsg_codes, dont_transform: false }),
290                )
291                .into(),
292            ),
293            ReaderType::LAZ => GISReader::LAZ(
294                LAZReader::new(
295                    buffer,
296                    Some(LASReaderOptions { epsg_codes, dont_transform: false }),
297                )
298                .into(),
299            ),
300            ReaderType::NADGrid => {
301                GISReader::NADGrid(NadGridReader::new("default".into(), buffer).into())
302            }
303            ReaderType::NetCDF => GISReader::NetCDF(NetCDFReader::new(buffer, None).into()),
304            ReaderType::OSM => {
305                let mut osm = OSMLocalReader::new(buffer, None);
306                osm.parse_blocks();
307                GISReader::OSM(osm.into())
308            }
309            ReaderType::Shapefile => GISReader::Shapefile(
310                shapefile_from_gzip(&buffer.slice(None, None), epsg_codes).into(),
311            ),
312            ReaderType::WKT => {
313                let input_str = buffer.parse_string(None, None);
314                GISReader::WKT(WKTGeometryReader::new(input_str).into())
315            }
316            _ => panic!("Unsupported file type: {file_type:?}"),
317        }
318    }
319}
320impl GISReader<FileReader> {
321    /// Given a file and a file type (or inferred if not provided), return a reader
322    ///
323    /// ## Parameters
324    ///
325    /// - `file`: The path to the file
326    /// - `file_type`: The file type if specified, otherwise it will be inferred. Useful for `zip` files
327    /// - `epsg_codes`: The EPSG codes to use. E.g. `{"4326": "...WKT STRING..."}`
328    ///
329    /// ## Returns
330    ///
331    /// The [`GISReader`] using a [`FileReader`]
332    #[cfg(feature = "std")]
333    pub fn from_path<P: AsRef<Path>>(
334        file: P,
335        file_type: Option<ReaderType>,
336        epsg_codes: Option<BTreeMap<String, String>>,
337    ) -> GISReader<FileReader> {
338        use crate::readers::file::shapefile_from_path;
339        use std::{ffi::OsStr, fs};
340
341        let path = file.as_ref().to_path_buf();
342        let path_ending = path.extension().and_then(OsStr::to_str).unwrap_or("");
343        let file_type: ReaderType = file_type.unwrap_or(path_ending.into());
344        let epsg_codes = epsg_codes.unwrap_or_default();
345        match file_type {
346            ReaderType::CSV => {
347                GISReader::CSV(CSVReader::new(FileReader::new(file).unwrap(), None).into())
348            }
349            ReaderType::GeoTIFF => GISReader::GeoTIFF(
350                GeoTIFFReader::new(
351                    FileReader::new(file).unwrap(),
352                    Some(GeoTIFFOptions { epsg_codes }),
353                )
354                .into(),
355            ),
356            ReaderType::GPX => {
357                GISReader::GPX(GPXReader::new(&fs::read_to_string(file).unwrap()).into())
358            }
359            ReaderType::GRIB2 => GISReader::GRIB2(
360                GRIB2Reader::new(FileReader::new(file).unwrap().into(), vec![]).into(),
361            ),
362            ReaderType::GTFS => {
363                GISReader::GTFS(GTFSScheduleReader::from_gzip(&fs::read(file).unwrap()).into())
364            }
365            ReaderType::JSON => {
366                GISReader::JSON(JSONReader::new(FileReader::new(file).unwrap()).into())
367            }
368            ReaderType::JSONLD => GISReader::JSONLD(
369                NewLineDelimitedJSONReader::new(FileReader::new(file).unwrap(), None).into(),
370            ),
371            ReaderType::JSONSQ => {
372                GISReader::JSONSQ(SequenceJSONReader::new(FileReader::new(file).unwrap()).into())
373            }
374            ReaderType::LAS => GISReader::LAS(
375                LASReader::new(
376                    FileReader::new(file).unwrap(),
377                    Some(LASReaderOptions { epsg_codes, dont_transform: false }),
378                )
379                .into(),
380            ),
381            ReaderType::LAZ => GISReader::LAZ(
382                LAZReader::new(
383                    FileReader::new(file).unwrap(),
384                    Some(LASReaderOptions { epsg_codes, dont_transform: false }),
385                )
386                .into(),
387            ),
388            ReaderType::NADGrid => GISReader::NADGrid(
389                NadGridReader::new("default".into(), FileReader::new(file).unwrap()).into(),
390            ),
391            ReaderType::NetCDF => {
392                GISReader::NetCDF(NetCDFReader::new(FileReader::new(file).unwrap(), None).into())
393            }
394            ReaderType::OSM => {
395                let mut osm = OSMLocalReader::new(FileReader::new(file).unwrap(), None);
396                osm.parse_blocks();
397                GISReader::OSM(osm.into())
398            }
399            ReaderType::Shapefile => {
400                // if file ends in zip, use shapefile_from_zip
401                if path_ending == "zip" {
402                    unimplemented!("Shapefile from zip not implemented yet")
403                } else {
404                    GISReader::Shapefile(shapefile_from_path(file, epsg_codes).into())
405                }
406            }
407            ReaderType::WKT => {
408                let input_str = fs::read_to_string(file).unwrap();
409                GISReader::WKT(WKTGeometryReader::new(input_str).into())
410            }
411            _ => panic!("Unsupported file type: {file_type:?}"),
412        }
413    }
414}
415
416/// The Global GIS Iterator tool
417#[derive(Debug)]
418pub enum GISIterator<'a, T: Reader + Debug> {
419    /// CSV Iterator
420    CSV(CSVIterator<'a, T, Properties>),
421    /// GeoTIFF Iterator
422    GeoTIFF(GeoTIFFIterator<'a, T>),
423    /// GPX Iterator
424    GPX(GPXIterator<'a>),
425    /// GRIB2 Iterator
426    GRIB2(GRIB2Iterator<'a>),
427    /// GTFS Iterator
428    GTFS(GTFSScheduleIterator),
429    /// JSON Iterator
430    JSON(JSONIterator<'a, T, (), Properties, MValue>),
431    /// JSON-LD Iterator
432    JSONLD(NewLineDelimitedJSONIterator<'a, T, (), Properties, MValue>),
433    /// JSON-SQ Iterator
434    JSONSQ(SequenceJSONIterator<'a, T, (), Properties, MValue>),
435    /// LAS Iterator
436    LAS(LASIterator<'a, T>),
437    /// LAZ Iterator
438    LAZ(LAZIterator<'a, T>),
439    /// NAD Grid Iterator
440    NADGrid(NadGridIterator<'a, T>),
441    /// NetCDF Iterator
442    NetCDF(CDFIterator<'a, T>),
443    /// OSM Iterator
444    OSM(OSMLocalReaderIter<'a, T>),
445    /// Shapefile Iterator
446    Shapefile(ShapefileIterator<'a, T, Properties>),
447    /// WKT Iterator
448    WKT(WKTIterator<'a>),
449}
450impl<'a, T: Reader + Debug> Iterator for GISIterator<'a, T> {
451    type Item = VectorFeature<(), Properties, MValue>;
452
453    fn next(&mut self) -> Option<Self::Item> {
454        match self {
455            GISIterator::CSV(iterator) => iterator.next().map(|f| f.to_m_vector_feature(|_| None)),
456            GISIterator::GeoTIFF(iterator) => {
457                iterator.next().map(|f| f.to_m_vector_feature(|_| None))
458            }
459            GISIterator::GPX(iterator) => iterator.next().map(|f| f.to_m_vector_feature(|_| None)),
460            GISIterator::GRIB2(iterator) => {
461                iterator.next().map(|f| f.to_m_vector_feature(|_| None))
462            }
463            GISIterator::GTFS(iterator) => iterator.next(),
464            GISIterator::JSON(iterator) => iterator.next(),
465            GISIterator::JSONLD(iterator) => iterator.next(),
466            GISIterator::JSONSQ(iterator) => iterator.next(),
467            GISIterator::LAS(iterator) => iterator.next().map(|f| f.to_m_vector_feature(|_| None)),
468            GISIterator::LAZ(iterator) => iterator.next().map(|f| f.to_m_vector_feature(|_| None)),
469            GISIterator::NADGrid(iterator) => iterator.next().map(|f| VectorFeature {
470                _type: f._type,
471                id: f.id,
472                face: f.face,
473                properties: f.properties,
474                geometry: f.geometry,
475                metadata: None,
476            }),
477            GISIterator::NetCDF(iterator) => iterator.next(),
478            GISIterator::OSM(iterator) => iterator.next().map(|f| VectorFeature {
479                _type: f._type,
480                id: f.id,
481                face: f.face,
482                properties: f.properties,
483                geometry: f.geometry,
484                metadata: None,
485            }),
486            GISIterator::Shapefile(iterator) => iterator.next(),
487            GISIterator::WKT(iterator) => iterator.next(),
488        }
489    }
490}
491impl<T: Reader + Debug> FeatureReader<(), Properties, MValue> for GISReader<T> {
492    type FeatureIterator<'a>
493        = GISIterator<'a, T>
494    where
495        T: 'a;
496
497    fn iter(&self) -> Self::FeatureIterator<'_> {
498        match self {
499            GISReader::CSV(reader) => GISIterator::CSV(reader.iter()),
500            GISReader::GeoTIFF(reader) => GISIterator::GeoTIFF(reader.iter()),
501            GISReader::GPX(reader) => GISIterator::GPX(reader.iter()),
502            GISReader::GRIB2(reader) => GISIterator::GRIB2(reader.iter()),
503            GISReader::GTFS(reader) => GISIterator::GTFS(reader.iter()),
504            GISReader::JSON(reader) => GISIterator::JSON(reader.iter()),
505            GISReader::JSONLD(reader) => GISIterator::JSONLD(reader.iter()),
506            GISReader::JSONSQ(reader) => GISIterator::JSONSQ(reader.iter()),
507            GISReader::LAS(reader) => GISIterator::LAS(reader.iter()),
508            GISReader::LAZ(reader) => GISIterator::LAZ(reader.iter()),
509            GISReader::NADGrid(reader) => GISIterator::NADGrid(reader.iter()),
510            GISReader::NetCDF(reader) => GISIterator::NetCDF(reader.iter()),
511            GISReader::OSM(reader) => GISIterator::OSM(reader.iter()),
512            GISReader::Shapefile(reader) => GISIterator::Shapefile(reader.iter()),
513            GISReader::WKT(reader) => GISIterator::WKT(reader.iter()),
514        }
515    }
516
517    #[cfg(feature = "std")]
518    fn par_iter(&self, pool_size: usize, thread_id: usize) -> Self::FeatureIterator<'_> {
519        match self {
520            GISReader::CSV(reader) => GISIterator::CSV(reader.par_iter(pool_size, thread_id)),
521            GISReader::GeoTIFF(reader) => {
522                GISIterator::GeoTIFF(reader.par_iter(pool_size, thread_id))
523            }
524            GISReader::GPX(reader) => GISIterator::GPX(reader.par_iter(pool_size, thread_id)),
525            GISReader::GRIB2(reader) => GISIterator::GRIB2(reader.par_iter(pool_size, thread_id)),
526            GISReader::GTFS(reader) => GISIterator::GTFS(reader.par_iter(pool_size, thread_id)),
527            GISReader::JSON(reader) => GISIterator::JSON(reader.par_iter(pool_size, thread_id)),
528            GISReader::JSONLD(reader) => GISIterator::JSONLD(reader.par_iter(pool_size, thread_id)),
529            GISReader::JSONSQ(reader) => GISIterator::JSONSQ(reader.par_iter(pool_size, thread_id)),
530            GISReader::LAS(reader) => GISIterator::LAS(reader.par_iter(pool_size, thread_id)),
531            GISReader::LAZ(reader) => GISIterator::LAZ(reader.par_iter(pool_size, thread_id)),
532            GISReader::NADGrid(reader) => {
533                GISIterator::NADGrid(reader.par_iter(pool_size, thread_id))
534            }
535            GISReader::NetCDF(reader) => GISIterator::NetCDF(reader.par_iter(pool_size, thread_id)),
536            GISReader::OSM(reader) => GISIterator::OSM(reader.par_iter(pool_size, thread_id)),
537            GISReader::Shapefile(reader) => {
538                GISIterator::Shapefile(reader.par_iter(pool_size, thread_id))
539            }
540            GISReader::WKT(reader) => GISIterator::WKT(reader.par_iter(pool_size, thread_id)),
541        }
542    }
543}
gistools/readers/mod.rs

gistools/readers/
mod.rs
