gistools/readers/
mod.rs

1/// CSV Reader
2pub mod csv;
3/// GBFS Readers
4pub mod gbfs;
5/// GeoTIFF Reader
6pub mod geotiff;
7/// GPX Reader
8pub mod gpx;
9/// Grib2 Reader
10pub mod grib2;
11/// GTFS Readers
12pub mod gtfs;
13/// JSON Readers
14pub mod json;
15/// LAS/LAZ Readers
16pub mod las;
17/// NAD Grid Reader
18pub mod nadgrid;
19/// NetCDF Reader
20pub mod netcdf;
21/// OSM (Open Street Map) PBF Reader
22pub mod osm;
23/// (S2)PMTiles Reader
24pub mod pmtiles;
25/// S2 Tiles Reader
26pub mod s2tiles;
27/// Shapefile Reader
28pub mod shapefile;
29/// Tile-based Readers
30#[cfg(feature = "std")]
31pub mod tile;
32/// WKT Geometry Reader
33pub mod wkt;
34
35#[cfg(feature = "std")]
36use crate::parsers::FileReader;
37use crate::parsers::{BufferReader, FeatureReader, Reader};
38use alloc::{boxed::Box, collections::BTreeMap, string::String, vec, vec::Vec};
39use core::fmt::Debug;
40pub use csv::*;
41pub use gbfs::*;
42pub use geotiff::*;
43pub use gpx::*;
44pub use grib2::*;
45pub use gtfs::*;
46pub use json::*;
47pub use las::*;
48pub use nadgrid::*;
49pub use netcdf::*;
50pub use osm::*;
51pub use pmtiles::*;
52use s2json::{MValue, Properties, VectorFeature};
53pub use s2tiles::*;
54use serde::{Deserialize, Serialize};
55pub use shapefile::*;
56#[cfg(feature = "std")]
57use std::path::Path;
58#[cfg(feature = "std")]
59pub use tile::*;
60pub use wkt::*;
61
62/// The type of readers to choose from
63#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
64#[serde(rename_all = "lowercase")]
65pub enum ReaderType {
66    /// CSV data
67    CSV,
68    /// GeoTIFF data
69    GeoTIFF,
70    /// GPX data
71    GPX,
72    /// GRIB 2 data
73    GRIB2,
74    /// GTFS data
75    GTFS,
76    /// JSON data
77    JSON,
78    /// JSON-LD data
79    JSONLD,
80    /// JSON-SQ data
81    JSONSQ,
82    /// LAS data
83    LAS,
84    /// LAZ data
85    LAZ,
86    /// NAD Grid data
87    NADGrid,
88    /// NetCDF data
89    NetCDF,
90    /// OSM data
91    OSM,
92    /// Shapefile
93    Shapefile,
94    /// Tile data
95    Tile,
96    /// WKT
97    WKT,
98}
99impl From<&str> for ReaderType {
100    fn from(value: &str) -> Self {
101        match value {
102            "csv" => ReaderType::CSV,
103            "geotiff" | "tif" | "tiff" | "geotif" => ReaderType::GeoTIFF,
104            "gpx" => ReaderType::GPX,
105            "grib2" | "grib" => ReaderType::GRIB2,
106            "gtfs" => ReaderType::GTFS,
107            "json" | "geojson" | "s2json" => ReaderType::JSON,
108            "jsonld" | "geojsonld" | "s2jsonld" | "json-ld" | "geojson-ld" | "s2json-ld" => {
109                ReaderType::JSONLD
110            }
111            "jsonsq" | "geojsonseq" | "geojsonsq" | "s2jsonseq" | "s2jsonsq" | "json-seq"
112            | "json-sq" | "geojson-seq" | "geojson-sq" | "s2json-seq" | "s2json-sq" => {
113                ReaderType::JSONSQ
114            }
115            "las" => ReaderType::LAS,
116            "laz" => ReaderType::LAZ,
117            "nadgrid" | "nad" | "gsb" => ReaderType::NADGrid,
118            "netcdf" | "nc4" | "cdf" | "nc" => ReaderType::NetCDF,
119            "osm" | "pbf" => ReaderType::OSM,
120            "shapefile" | "shp" | "zip" => ReaderType::Shapefile,
121            "wkt" => ReaderType::WKT,
122            _ => ReaderType::Tile,
123        }
124    }
125}
126
127/// # GIS Reader
128///
129/// ## Description
130/// Parse all data types supported by this library
131///
132/// Implements the [`FeatureReader`] trait
133///
134/// It is recommended to use this reader for testing or ease of access, but the better
135/// alternative is to use the file type readers directly. Here is the list of readers:
136/// - [`CSVReader`]: Parse (Geo|S2)JSON from a file that is in the CSV format
137/// - [`GeoTIFFReader`]: This class reads a GeoTIFF file and returns a list of GeoTIFF images.
138/// - [`GPXReader`]: The GPX Reader is an XML-based GPS Exchange Format (GPX) reader.
139/// - [`GRIB2Reader`]: This class reads a GRIB2 file and returns a list of GRIB2 products.
140/// - [`GTFSScheduleReader`]: Schedule class that pulls in all of the GTFS schedule files and parses them into a single object
141/// - [`JSONReader`]: Parse (Geo|S2)JSON. Can handle millions of features.
142/// - [`NewLineDelimitedJSONReader`]: Parse (Geo|S2)JSON from a file that is in a newline-delimited format
143/// - [`SequenceJSONReader`]: Parse GeoJSON from a file that is in the `geojson-text-sequences` format.
144/// - [`LASReader`]: Reads LAS data. Supports up to the LAS 1.4 specification.
145/// - [`LAZReader`]: Reads LAS zipped data. Supports LAS 1.4 specification although missing some support.
146/// - [`NadGridReader`]: Loads/reads a binary NTv2 file (.gsb) implementing the {@link FeatureIterator} interface.
147/// - [`NetCDFReader`]: Read the NetCDF v3.x file format.
148/// - [`OSMLocalReader`]: OSM PBF Data. Direct use allows for the use of the [`OSMFileReader`] as well
149/// - [`ShapeFileReader`]: Reads data from a shapefile implementing the {@link FeatureIterator} interface
150/// - [`WKTGeometryReader`]: Parse a collection of WKT geometries from a string
151///
152/// ## Usage
153///
154/// ### Read from a file
155/// ```rust
156/// use gistools::{
157///     parsers::{FeatureReader},
158///     readers::{GISReader, ReaderType},
159/// };
160/// use s2json::{MValue, Properties, VectorFeature};
161/// use std::path::PathBuf;
162///
163/// let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
164/// path.push("tests/readers/csv/fixtures/basic.csv");
165///
166/// let reader = GISReader::from_path(path, None, None);
167/// assert_eq!(reader.get_type(), ReaderType::CSV);
168///
169/// let features: Vec<VectorFeature<(), Properties, MValue>> = reader.iter().collect();
170/// ```
171///
172/// ### Read from a buffer
173///
174/// It is recommended to use a Buffer Reader when the file is small because it is more efficient
175///
176/// ```rust
177/// use gistools::{
178///     parsers::{FeatureReader},
179///     readers::{GISReader, ReaderType},
180/// };
181/// use s2json::{MValue, Properties, VectorFeature};
182///
183/// // ignore the use of the filesystem, setup is just for the example
184/// use std::path::PathBuf;
185/// let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
186/// path.push("tests/readers/csv/fixtures/basic.csv");
187/// let bytes = std::fs::read(path.clone()).unwrap();
188///
189/// let reader = GISReader::from_buffer(bytes, ReaderType::CSV, None);
190/// let features: Vec<VectorFeature<(), Properties, MValue>> = reader.iter().collect();
191/// ```
192#[derive(Debug, Clone)]
193pub enum GISReader<T: Reader + Debug> {
194    /// CSV data
195    CSV(Box<CSVReader<T, Properties>>),
196    /// GeoTIFF data
197    GeoTIFF(Box<GeoTIFFReader<T>>),
198    /// GPX data
199    GPX(Box<GPXReader>),
200    /// GRIB 2 data
201    GRIB2(Box<GRIB2Reader>),
202    /// GTFS data
203    GTFS(Box<GTFSScheduleReader>),
204    /// JSON data
205    JSON(Box<JSONReader<T, (), Properties, MValue>>),
206    /// JSON-LD data
207    JSONLD(Box<NewLineDelimitedJSONReader<T, (), Properties, MValue>>),
208    /// JSON-SQ data
209    JSONSQ(Box<SequenceJSONReader<T, (), Properties, MValue>>),
210    /// LAS data
211    LAS(Box<LASReader<T>>),
212    /// LAZ data
213    LAZ(Box<LAZReader<T>>),
214    /// NAD Grid data
215    NADGrid(Box<NadGridReader<T>>),
216    /// NetCDF data
217    NetCDF(Box<NetCDFReader<T>>),
218    /// OSM data
219    OSM(Box<OSMLocalReader<T>>),
220    /// Shapefile
221    Shapefile(Box<ShapeFileReader<T, Properties>>),
222    // /// Tile data
223    // Tile(Box<TileReader<P, D>>),
224    /// WKT
225    WKT(Box<WKTGeometryReader>),
226}
227impl<T: Reader + Debug> GISReader<T> {
228    /// Get the type of the reader
229    pub fn get_type(&self) -> ReaderType {
230        match self {
231            GISReader::CSV(_) => ReaderType::CSV,
232            GISReader::GeoTIFF(_) => ReaderType::GeoTIFF,
233            GISReader::GPX(_) => ReaderType::GPX,
234            GISReader::GRIB2(_) => ReaderType::GRIB2,
235            GISReader::GTFS(_) => ReaderType::GTFS,
236            GISReader::JSON(_) => ReaderType::JSON,
237            GISReader::JSONLD(_) => ReaderType::JSONLD,
238            GISReader::JSONSQ(_) => ReaderType::JSONSQ,
239            GISReader::LAS(_) => ReaderType::LAS,
240            GISReader::LAZ(_) => ReaderType::LAZ,
241            GISReader::NADGrid(_) => ReaderType::NADGrid,
242            GISReader::NetCDF(_) => ReaderType::NetCDF,
243            GISReader::OSM(_) => ReaderType::OSM,
244            GISReader::Shapefile(_) => ReaderType::Shapefile,
245            // GISReader::Tile(_) => ReaderType::Tile,
246            GISReader::WKT(_) => ReaderType::WKT,
247        }
248    }
249}
250impl GISReader<BufferReader> {
251    /// Given a raw data and a file type, return the appropriate reader
252    ///
253    /// ## Parameters
254    ///
255    /// - `data`: The data to parse
256    /// - `file_type`: The file type to parse the data as
257    /// - `epsg_codes`: The EPSG codes to use. E.g. `{"4326": "...WKT STRING..."}`
258    ///
259    /// ## Returns
260    ///
261    /// The [`GISReader`] using a [`BufferReader`] for fast parsing
262    pub fn from_buffer(
263        data: Vec<u8>,
264        file_type: ReaderType,
265        epsg_codes: Option<BTreeMap<String, String>>,
266    ) -> GISReader<BufferReader> {
267        let buffer = BufferReader::new(data);
268        let epsg_codes = epsg_codes.unwrap_or_default();
269        match file_type {
270            ReaderType::CSV => GISReader::CSV(CSVReader::new(buffer, None).into()),
271            ReaderType::GeoTIFF => GISReader::GeoTIFF(
272                GeoTIFFReader::new(buffer, Some(GeoTIFFOptions { epsg_codes })).into(),
273            ),
274            ReaderType::GPX => {
275                let input_str = buffer.parse_string(None, None);
276                GISReader::GPX(GPXReader::new(&input_str).into())
277            }
278            ReaderType::GRIB2 => GISReader::GRIB2(GRIB2Reader::new(buffer.into(), vec![]).into()),
279            ReaderType::GTFS => {
280                GISReader::GTFS(GTFSScheduleReader::from_gzip(&buffer.slice(None, None)).into())
281            }
282            ReaderType::JSON => GISReader::JSON(JSONReader::new(buffer).into()),
283            ReaderType::JSONLD => {
284                GISReader::JSONLD(NewLineDelimitedJSONReader::new(buffer, None).into())
285            }
286            ReaderType::JSONSQ => GISReader::JSONSQ(SequenceJSONReader::new(buffer).into()),
287            ReaderType::LAS => GISReader::LAS(
288                LASReader::new(
289                    buffer,
290                    Some(LASReaderOptions { epsg_codes, dont_transform: false }),
291                )
292                .into(),
293            ),
294            ReaderType::LAZ => GISReader::LAZ(
295                LAZReader::new(
296                    buffer,
297                    Some(LASReaderOptions { epsg_codes, dont_transform: false }),
298                )
299                .into(),
300            ),
301            ReaderType::NADGrid => {
302                GISReader::NADGrid(NadGridReader::new("default".into(), buffer).into())
303            }
304            ReaderType::NetCDF => GISReader::NetCDF(NetCDFReader::new(buffer, None).into()),
305            ReaderType::OSM => {
306                let mut osm = OSMLocalReader::new(buffer, None);
307                osm.parse_blocks();
308                GISReader::OSM(osm.into())
309            }
310            ReaderType::Shapefile => GISReader::Shapefile(
311                shapefile_from_gzip(&buffer.slice(None, None), epsg_codes).into(),
312            ),
313            ReaderType::WKT => {
314                let input_str = buffer.parse_string(None, None);
315                GISReader::WKT(WKTGeometryReader::new(input_str).into())
316            }
317            _ => panic!("Unsupported file type: {file_type:?}"),
318        }
319    }
320}
321#[cfg(feature = "std")]
322impl GISReader<FileReader> {
323    /// Given a file and a file type (or inferred if not provided), return a reader
324    ///
325    /// ## Parameters
326    ///
327    /// - `file`: The path to the file
328    /// - `file_type`: The file type if specified, otherwise it will be inferred. Useful for `zip` files
329    /// - `epsg_codes`: The EPSG codes to use. E.g. `{"4326": "...WKT STRING..."}`
330    ///
331    /// ## Returns
332    ///
333    /// The [`GISReader`] using a [`FileReader`]
334    #[cfg(feature = "std")]
335    pub fn from_path<P: AsRef<Path>>(
336        file: P,
337        file_type: Option<ReaderType>,
338        epsg_codes: Option<BTreeMap<String, String>>,
339    ) -> GISReader<FileReader> {
340        use crate::readers::file::shapefile_from_path;
341        use std::{ffi::OsStr, fs};
342
343        let path = file.as_ref().to_path_buf();
344        let path_ending = path.extension().and_then(OsStr::to_str).unwrap_or("");
345        let file_type: ReaderType = file_type.unwrap_or(path_ending.into());
346        let epsg_codes = epsg_codes.unwrap_or_default();
347        match file_type {
348            ReaderType::CSV => {
349                GISReader::CSV(CSVReader::new(FileReader::new(file).unwrap(), None).into())
350            }
351            ReaderType::GeoTIFF => GISReader::GeoTIFF(
352                GeoTIFFReader::new(
353                    FileReader::new(file).unwrap(),
354                    Some(GeoTIFFOptions { epsg_codes }),
355                )
356                .into(),
357            ),
358            ReaderType::GPX => {
359                GISReader::GPX(GPXReader::new(&fs::read_to_string(file).unwrap()).into())
360            }
361            ReaderType::GRIB2 => GISReader::GRIB2(
362                GRIB2Reader::new(FileReader::new(file).unwrap().into(), vec![]).into(),
363            ),
364            ReaderType::GTFS => {
365                GISReader::GTFS(GTFSScheduleReader::from_gzip(&fs::read(file).unwrap()).into())
366            }
367            ReaderType::JSON => {
368                GISReader::JSON(JSONReader::new(FileReader::new(file).unwrap()).into())
369            }
370            ReaderType::JSONLD => GISReader::JSONLD(
371                NewLineDelimitedJSONReader::new(FileReader::new(file).unwrap(), None).into(),
372            ),
373            ReaderType::JSONSQ => {
374                GISReader::JSONSQ(SequenceJSONReader::new(FileReader::new(file).unwrap()).into())
375            }
376            ReaderType::LAS => GISReader::LAS(
377                LASReader::new(
378                    FileReader::new(file).unwrap(),
379                    Some(LASReaderOptions { epsg_codes, dont_transform: false }),
380                )
381                .into(),
382            ),
383            ReaderType::LAZ => GISReader::LAZ(
384                LAZReader::new(
385                    FileReader::new(file).unwrap(),
386                    Some(LASReaderOptions { epsg_codes, dont_transform: false }),
387                )
388                .into(),
389            ),
390            ReaderType::NADGrid => GISReader::NADGrid(
391                NadGridReader::new("default".into(), FileReader::new(file).unwrap()).into(),
392            ),
393            ReaderType::NetCDF => {
394                GISReader::NetCDF(NetCDFReader::new(FileReader::new(file).unwrap(), None).into())
395            }
396            ReaderType::OSM => {
397                let mut osm = OSMLocalReader::new(FileReader::new(file).unwrap(), None);
398                osm.parse_blocks();
399                GISReader::OSM(osm.into())
400            }
401            ReaderType::Shapefile => {
402                // if file ends in zip, use shapefile_from_zip
403                if path_ending == "zip" {
404                    unimplemented!("Shapefile from zip not implemented yet")
405                } else {
406                    GISReader::Shapefile(shapefile_from_path(file, epsg_codes).into())
407                }
408            }
409            ReaderType::WKT => {
410                let input_str = fs::read_to_string(file).unwrap();
411                GISReader::WKT(WKTGeometryReader::new(input_str).into())
412            }
413            _ => panic!("Unsupported file type: {file_type:?}"),
414        }
415    }
416}
417
418/// The Global GIS Iterator tool
419#[derive(Debug)]
420pub enum GISIterator<'a, T: Reader + Debug> {
421    /// CSV Iterator
422    CSV(CSVIterator<'a, T, Properties>),
423    /// GeoTIFF Iterator
424    GeoTIFF(GeoTIFFIterator<'a, T>),
425    /// GPX Iterator
426    GPX(GPXIterator<'a>),
427    /// GRIB2 Iterator
428    GRIB2(GRIB2Iterator<'a>),
429    /// GTFS Iterator
430    GTFS(GTFSScheduleIterator),
431    /// JSON Iterator
432    JSON(JSONIterator<'a, T, (), Properties, MValue>),
433    /// JSON-LD Iterator
434    JSONLD(NewLineDelimitedJSONIterator<'a, T, (), Properties, MValue>),
435    /// JSON-SQ Iterator
436    JSONSQ(SequenceJSONIterator<'a, T, (), Properties, MValue>),
437    /// LAS Iterator
438    LAS(LASIterator<'a, T>),
439    /// LAZ Iterator
440    LAZ(LAZIterator<'a, T>),
441    /// NAD Grid Iterator
442    NADGrid(NadGridIterator<'a, T>),
443    /// NetCDF Iterator
444    NetCDF(CDFIterator<'a, T>),
445    /// OSM Iterator
446    OSM(OSMLocalReaderIter<'a, T>),
447    /// Shapefile Iterator
448    Shapefile(ShapefileIterator<'a, T, Properties>),
449    /// WKT Iterator
450    WKT(WKTIterator<'a>),
451}
452impl<'a, T: Reader + Debug> Iterator for GISIterator<'a, T> {
453    type Item = VectorFeature<(), Properties, MValue>;
454
455    fn next(&mut self) -> Option<Self::Item> {
456        match self {
457            GISIterator::CSV(iterator) => iterator.next().map(|f| f.to_m_vector_feature(|_| None)),
458            GISIterator::GeoTIFF(iterator) => {
459                iterator.next().map(|f| f.to_m_vector_feature(|_| None))
460            }
461            GISIterator::GPX(iterator) => iterator.next().map(|f| f.to_m_vector_feature(|_| None)),
462            GISIterator::GRIB2(iterator) => {
463                iterator.next().map(|f| f.to_m_vector_feature(|_| None))
464            }
465            GISIterator::GTFS(iterator) => iterator.next(),
466            GISIterator::JSON(iterator) => iterator.next(),
467            GISIterator::JSONLD(iterator) => iterator.next(),
468            GISIterator::JSONSQ(iterator) => iterator.next(),
469            GISIterator::LAS(iterator) => iterator.next().map(|f| f.to_m_vector_feature(|_| None)),
470            GISIterator::LAZ(iterator) => iterator.next().map(|f| f.to_m_vector_feature(|_| None)),
471            GISIterator::NADGrid(iterator) => iterator.next().map(|f| VectorFeature {
472                _type: f._type,
473                id: f.id,
474                face: f.face,
475                properties: f.properties,
476                geometry: f.geometry,
477                metadata: None,
478            }),
479            GISIterator::NetCDF(iterator) => iterator.next(),
480            GISIterator::OSM(iterator) => iterator.next().map(|f| VectorFeature {
481                _type: f._type,
482                id: f.id,
483                face: f.face,
484                properties: f.properties,
485                geometry: f.geometry,
486                metadata: None,
487            }),
488            GISIterator::Shapefile(iterator) => iterator.next(),
489            GISIterator::WKT(iterator) => iterator.next(),
490        }
491    }
492}
493impl<T: Reader + Debug> FeatureReader<(), Properties, MValue> for GISReader<T> {
494    type FeatureIterator<'a>
495        = GISIterator<'a, T>
496    where
497        T: 'a;
498
499    fn iter(&self) -> Self::FeatureIterator<'_> {
500        match self {
501            GISReader::CSV(reader) => GISIterator::CSV(reader.iter()),
502            GISReader::GeoTIFF(reader) => GISIterator::GeoTIFF(reader.iter()),
503            GISReader::GPX(reader) => GISIterator::GPX(reader.iter()),
504            GISReader::GRIB2(reader) => GISIterator::GRIB2(reader.iter()),
505            GISReader::GTFS(reader) => GISIterator::GTFS(reader.iter()),
506            GISReader::JSON(reader) => GISIterator::JSON(reader.iter()),
507            GISReader::JSONLD(reader) => GISIterator::JSONLD(reader.iter()),
508            GISReader::JSONSQ(reader) => GISIterator::JSONSQ(reader.iter()),
509            GISReader::LAS(reader) => GISIterator::LAS(reader.iter()),
510            GISReader::LAZ(reader) => GISIterator::LAZ(reader.iter()),
511            GISReader::NADGrid(reader) => GISIterator::NADGrid(reader.iter()),
512            GISReader::NetCDF(reader) => GISIterator::NetCDF(reader.iter()),
513            GISReader::OSM(reader) => GISIterator::OSM(reader.iter()),
514            GISReader::Shapefile(reader) => GISIterator::Shapefile(reader.iter()),
515            GISReader::WKT(reader) => GISIterator::WKT(reader.iter()),
516        }
517    }
518
519    fn par_iter(&self, pool_size: usize, thread_id: usize) -> Self::FeatureIterator<'_> {
520        match self {
521            GISReader::CSV(reader) => GISIterator::CSV(reader.par_iter(pool_size, thread_id)),
522            GISReader::GeoTIFF(reader) => {
523                GISIterator::GeoTIFF(reader.par_iter(pool_size, thread_id))
524            }
525            GISReader::GPX(reader) => GISIterator::GPX(reader.par_iter(pool_size, thread_id)),
526            GISReader::GRIB2(reader) => GISIterator::GRIB2(reader.par_iter(pool_size, thread_id)),
527            GISReader::GTFS(reader) => GISIterator::GTFS(reader.par_iter(pool_size, thread_id)),
528            GISReader::JSON(reader) => GISIterator::JSON(reader.par_iter(pool_size, thread_id)),
529            GISReader::JSONLD(reader) => GISIterator::JSONLD(reader.par_iter(pool_size, thread_id)),
530            GISReader::JSONSQ(reader) => GISIterator::JSONSQ(reader.par_iter(pool_size, thread_id)),
531            GISReader::LAS(reader) => GISIterator::LAS(reader.par_iter(pool_size, thread_id)),
532            GISReader::LAZ(reader) => GISIterator::LAZ(reader.par_iter(pool_size, thread_id)),
533            GISReader::NADGrid(reader) => {
534                GISIterator::NADGrid(reader.par_iter(pool_size, thread_id))
535            }
536            GISReader::NetCDF(reader) => GISIterator::NetCDF(reader.par_iter(pool_size, thread_id)),
537            GISReader::OSM(reader) => GISIterator::OSM(reader.par_iter(pool_size, thread_id)),
538            GISReader::Shapefile(reader) => {
539                GISIterator::Shapefile(reader.par_iter(pool_size, thread_id))
540            }
541            GISReader::WKT(reader) => GISIterator::WKT(reader.par_iter(pool_size, thread_id)),
542        }
543    }
544}
gistools/readers/mod.rs

gistools/readers/
mod.rs
