gistools/readers/grib2/
mod.rs

1/// Section content
2pub mod sections;
3
4use crate::{
5    parsers::{BufferReader, FeatureReader, Reader},
6    util::fetch_url,
7};
8use alloc::{
9    format,
10    string::{String, ToString},
11    vec,
12    vec::Vec,
13};
14use core::cell::RefCell;
15use s2json::{BBox3D, MValue, Properties, VectorFeature, VectorGeometry, VectorMultiPoint};
16pub use sections::*;
17
18/// An GRIB2 Shaped Vector Feature
19pub type GRIB2VectorFeature = VectorFeature<Vec<Grib2ProductDefinition>>;
20
21/// GFS sources available for download
22#[derive(Debug, Clone, PartialEq, Eq)]
23pub enum Grib2GFSSource {
24    /// AWS
25    Aws,
26    /// FTPPRD
27    Ftpprd,
28    /// NOMADS
29    Nomads,
30    /// Google
31    Google,
32    /// Azure
33    Azure,
34    /// User defined server
35    Other(String),
36}
37impl From<&str> for Grib2GFSSource {
38    fn from(value: &str) -> Self {
39        match value {
40            "aws" => Grib2GFSSource::Aws,
41            "ftpprd" => Grib2GFSSource::Ftpprd,
42            "nomads" => Grib2GFSSource::Nomads,
43            "google" => Grib2GFSSource::Google,
44            "azure" => Grib2GFSSource::Azure,
45            _ => Grib2GFSSource::Other(value.into()),
46        }
47    }
48}
49impl Grib2GFSSource {
50    /// Convert the source to a URL
51    pub fn to_url(&self) -> String {
52        match self {
53            Grib2GFSSource::Aws => "https://noaa-gfs-bdp-pds.s3.amazonaws.com/".into(),
54            Grib2GFSSource::Ftpprd => "https://ftpprd.ncep.noaa.gov/data/nccf/com/gfs/prod/".into(),
55            Grib2GFSSource::Nomads => {
56                "https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/".into()
57            }
58            Grib2GFSSource::Google => {
59                "https://storage.googleapis.com/global-forecast-system/".into()
60            }
61            Grib2GFSSource::Azure => "https://noaagfs.blob.core.windows.net/gfs/".into(),
62            Grib2GFSSource::Other(s) => s.into(),
63        }
64    }
65}
66
67/// GFS domains available for download
68#[derive(Debug, Clone, Copy, PartialEq, Eq)]
69pub enum Grib2GFSDomain {
70    /// Atmospheric
71    Atmos,
72    /// Ocean
73    Wave,
74}
75
76/// GFS ATMOS products available for download
77/// - `pgrb2.0p25` - common fields, 0.25 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2.0p25.f000.shtml)
78/// - `pgrb2.0p50` - common fields, 0.50 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2.0p50.f000.shtml)
79/// - `pgrb2.1p00` - common fields, 1.00 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2.1p00.f000.shtml)
80/// - `pgrb2b.0p25` - uncommon fields, 0.25 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2b.0p25.f000.shtml)
81/// - `pgrb2b.0p50` - uncommon fields, 0.50 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2b.0p50.f000.shtml)
82/// - `pgrb2b.1p00` - uncommon fields, 1.00 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2b.1p00.f000.shtml)
83/// - `pgrb2full.0p50` - combined grids of 0.50 resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t12z.pgrb2full.0p50.f000.shtml)
84/// - `sfluxgrb` - surface flux fields, T1534 Semi-Lagrangian grid [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.sfluxgrbf000.grib2.shtml)
85/// - `goesimpgrb2.0p25` - 0.50 degree resolution for GOES-IMP [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.goessimpgrb2.0p25.f000.shtml)
86#[derive(Debug, Clone, PartialEq, Eq)]
87pub enum Grib2AtmosGFSProduct {
88    /// `pgrb2.0p25` - common fields, 0.25 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2.0p25.f000.shtml)
89    Pgrb20p25,
90    /// - `pgrb2.0p50` - common fields, 0.50 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2.0p50.f000.shtml)
91    Pgrb20p50,
92    /// - `pgrb2.1p00` - common fields, 1.00 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2.1p00.f000.shtml)
93    Pgrb21p00,
94    /// - `pgrb2b.0p25` - uncommon fields, 0.25 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2b.0p25.f000.shtml)
95    Pgrb2b0p25,
96    /// - `pgrb2b.0p50` - uncommon fields, 0.50 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2b.0p50.f000.shtml)
97    Pgrb2b0p50,
98    /// - `pgrb2b.1p00` - uncommon fields, 1.00 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2b.1p00.f000.shtml)
99    Pgrb2b1p00,
100    /// - `pgrb2full.0p50` - combined grids of 0.50 resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t12z.pgrb2full.0p50.f000.shtml)
101    Pgrb2full0p50,
102    /// - `sfluxgrb` - surface flux fields, T1534 Semi-Lagrangian grid [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.sfluxgrbf000.grib2.shtml)
103    Sfluxgrb,
104    /// - `goesimpgrb2.0p25` - 0.50 degree resolution for GOES-IMP [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.goessimpgrb2.0p25.f000.shtml)
105    Goesimpgrb20p25,
106    /// - User defined product
107    Other(String),
108}
109impl From<&str> for Grib2AtmosGFSProduct {
110    fn from(value: &str) -> Self {
111        match value {
112            "pgrb2.0p25" => Self::Pgrb20p25,
113            "pgrb2.0p50" => Self::Pgrb20p50,
114            "pgrb2.1p00" => Self::Pgrb21p00,
115            "pgrb2b.0p25" => Self::Pgrb2b0p25,
116            "pgrb2b.0p50" => Self::Pgrb2b0p50,
117            "pgrb2b.1p00" => Self::Pgrb2b1p00,
118            "pgrb2full.0p50" => Self::Pgrb2full0p50,
119            "sfluxgrb" => Self::Sfluxgrb,
120            "goesimpgrb2.0p25" => Self::Goesimpgrb20p25,
121            _ => Self::Other(value.into()),
122        }
123    }
124}
125impl From<Grib2AtmosGFSProduct> for String {
126    fn from(value: Grib2AtmosGFSProduct) -> Self {
127        match value {
128            Grib2AtmosGFSProduct::Pgrb20p25 => "pgrb2.0p25".into(),
129            Grib2AtmosGFSProduct::Pgrb20p50 => "pgrb2.0p50".into(),
130            Grib2AtmosGFSProduct::Pgrb21p00 => "pgrb2.1p00".into(),
131            Grib2AtmosGFSProduct::Pgrb2b0p25 => "pgrb2b.0p25".into(),
132            Grib2AtmosGFSProduct::Pgrb2b0p50 => "pgrb2b.0p50".into(),
133            Grib2AtmosGFSProduct::Pgrb2b1p00 => "pgrb2b.1p00".into(),
134            Grib2AtmosGFSProduct::Pgrb2full0p50 => "pgrb2full.0p50".into(),
135            Grib2AtmosGFSProduct::Sfluxgrb => "sfluxgrb".into(),
136            Grib2AtmosGFSProduct::Goesimpgrb20p25 => "goesimpgrb2.0p25".into(),
137            Grib2AtmosGFSProduct::Other(v) => v,
138        }
139    }
140}
141
142/// GFS WAVE products available for download
143/// - `arctic.9km` - Arctic, 9km resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.arctic.9km.f003.grib2.shtml)
144/// - `atlocn.0p16` - Atlantic, 0.16 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.atlocn.0p16.f003.grib2.shtml)
145/// - `epacif.0p16` - Eastern Pacific, 0.16 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.epacif.0p16.f003.grib2.shtml)
146/// - `global.0p16` - Global, 0.16 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.global.0p16.f003.grib2.shtml)
147/// - `global.0p25` - Global, 0.25 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.global.0p25.f003.grib2.shtml)
148/// - `gsouth.0p25` - Gulf of South America, 0.25 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.gsouth.0p25.f003.grib2.shtml)
149/// - `wcoast.0p16` - West Coast, 0.16 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.wcoast.0p16.f003.grib2.shtml)
150#[derive(Debug, Clone, PartialEq, Eq)]
151pub enum Grib2WaveGFSProduct {
152    /// - `arctic.9km` - Arctic, 9km resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.arctic.9km.f003.grib2.shtml)
153    Arctic9km,
154    /// - `atlocn.0p16` - Atlantic, 0.16 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.atlocn.0p16.f003.grib2.shtml)
155    Atlocn0p16,
156    /// - `epacif.0p16` - Eastern Pacific, 0.16 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.epacif.0p16.f003.grib2.shtml)
157    Epacif0p16,
158    /// - `global.0p16` - Global, 0.16 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.global.0p16.f003.grib2.shtml)
159    Global0p16,
160    /// - `global.0p25` - Global, 0.25 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.global.0p25.f003.grib2.shtml)
161    Global0p25,
162    /// - `gsouth.0p25` - Gulf of South America, 0.25 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.gsouth.0p25.f003.grib2.shtml)
163    Gsouth0p25,
164    /// - `wcoast.0p16` - West Coast, 0.16 degree resolution [Study Variables here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.wcoast.0p16.f003.grib2.shtml)
165    Wcoast0p16,
166    /// User defined product
167    Other(String),
168}
169impl From<&str> for Grib2WaveGFSProduct {
170    fn from(value: &str) -> Self {
171        match value {
172            "arctic.9km" => Grib2WaveGFSProduct::Arctic9km,
173            "atlocn.0p16" => Grib2WaveGFSProduct::Atlocn0p16,
174            "epacif.0p16" => Grib2WaveGFSProduct::Epacif0p16,
175            "global.0p16" => Grib2WaveGFSProduct::Global0p16,
176            "global.0p25" => Grib2WaveGFSProduct::Global0p25,
177            "gsouth.0p25" => Grib2WaveGFSProduct::Gsouth0p25,
178            "wcoast.0p16" => Grib2WaveGFSProduct::Wcoast0p16,
179            _ => Grib2WaveGFSProduct::Other(value.into()),
180        }
181    }
182}
183impl From<Grib2WaveGFSProduct> for String {
184    fn from(value: Grib2WaveGFSProduct) -> Self {
185        match value {
186            Grib2WaveGFSProduct::Arctic9km => "arctic.9km".into(),
187            Grib2WaveGFSProduct::Atlocn0p16 => "atlocn.0p16".into(),
188            Grib2WaveGFSProduct::Epacif0p16 => "epacif.0p16".into(),
189            Grib2WaveGFSProduct::Global0p16 => "global.0p16".into(),
190            Grib2WaveGFSProduct::Global0p25 => "global.0p25".into(),
191            Grib2WaveGFSProduct::Gsouth0p25 => "gsouth.0p25".into(),
192            Grib2WaveGFSProduct::Wcoast0p16 => "wcoast.0p16".into(),
193            Grib2WaveGFSProduct::Other(value) => value,
194        }
195    }
196}
197
198/// GFS Hour
199#[derive(Debug, Clone, PartialEq, Eq)]
200pub enum Grib2GFSHour {
201    /// "00"
202    Hour0,
203    /// "06"
204    Hour6,
205    /// "12"
206    Hour12,
207    /// "18"
208    Hour18,
209}
210impl From<&str> for Grib2GFSHour {
211    fn from(value: &str) -> Self {
212        match value {
213            "00" => Grib2GFSHour::Hour0,
214            "06" => Grib2GFSHour::Hour6,
215            "12" => Grib2GFSHour::Hour12,
216            "18" => Grib2GFSHour::Hour18,
217            _ => panic!("Invalid hour"),
218        }
219    }
220}
221impl From<Grib2GFSHour> for String {
222    fn from(value: Grib2GFSHour) -> Self {
223        match value {
224            Grib2GFSHour::Hour0 => "00".into(),
225            Grib2GFSHour::Hour6 => "06".into(),
226            Grib2GFSHour::Hour12 => "12".into(),
227            Grib2GFSHour::Hour18 => "18".into(),
228        }
229    }
230}
231
232/// Description of a section in the GRIB2 file
233#[derive(Debug, Clone, PartialEq, Eq)]
234pub struct Grib2SectionLocations {
235    /// Start/offset of section
236    pub start: u64,
237    /// If missing, assume the end is the end of the file
238    pub end: Option<u64>,
239    /// The entire line detailing the section
240    pub line: String,
241    /// The name of the filter
242    pub name: String,
243}
244
245#[doc(hidden)]
246/// # Fetch ATMOS or WAVE GFS data.
247///
248/// ## ATMOS
249/// You can find some data to reference what's available [here](https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/).
250///
251/// An example of what variable data means can be found [here](https://www.nco.ncep.noaa.gov/pmb/products/gfs/gfs.t00z.pgrb2.0p50.f000.shtml).
252///
253/// ## WAVE
254/// You can find some data to reference what's available [here](https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/).
255///
256/// An example of what variable data means can be found [here](https://www.nco.ncep.noaa.gov/pmb/products/wave/gfswave.t12z.arctic.9km.f003.grib2.shtml).
257///
258/// ## Parameters
259///
260/// - `source`: The source of the data, `aws` | `ftpprd` | `nomads` | `google` | `azure` | or a user provided url
261/// - `product`: which product to fetch. Use [`Grib2AtmosGFSProduct`] or [`Grib2WaveGFSProduct`]
262/// - `domain`: The domain of the data, `atmos` or `wave`
263/// - `year`: The year to fetch given a 4 digit year
264/// - `month`: The month to fetch given a 2 digit month 01 is January and 12 is December
265/// - `day`: The day to fetch given a 2 digit day, e.g. '01' or '31'
266/// - `hour`: The forecast hour with 2 digits often in increments of 6 up to 18, e.g. '00' or '12'
267/// - `forecast`: The forecast hour with 3 digits often in increments of 3 up to 384, e.g. '000' or '003'
268/// - `filters`: The filters to apply by filtering lines in the .idx file
269///
270/// ## Returns
271///
272/// A [`GRIB2Reader`] of the specific sections
273///
274/// ## Example
275///
276/// ```rust
277/// use gistools::readers::{fetch_gfs_data, Grib2GFSSource, Grib2AtmosGFSProduct, Grib2GFSDomain};
278///
279/// async fn example() {
280///     let grib2_reader = fetch_gfs_data(
281///         Grib2GFSSource::Aws,
282///         Grib2AtmosGFSProduct::Pgrb2b1p00,
283///         Grib2GFSDomain::Atmos,
284///         "2024".into(),
285///         "12".into(),
286///         "14".into(),
287///         "12".into(),
288///         Some("003".into()),
289///         Some(vec!["TMP:2 m".into()]),
290///      )
291///     .await;
292///     assert_eq!(grib2_reader.idxs.len(), 1);
293/// }
294/// ```
295#[allow(clippy::too_many_arguments)]
296pub async fn fetch_gfs_data<P: Into<String>>(
297    source: Grib2GFSSource,
298    product: P,
299    domain: Grib2GFSDomain,
300    year: String,
301    month: String,
302    day: String,
303    hour: Grib2GFSHour,
304    forecast: Option<String>,
305    filters: Option<Vec<String>>,
306) -> GRIB2Reader {
307    // If year is not 4 chars, month not 2, day not 2, or forecast is not 3 chars, return error
308    let forecast = forecast.unwrap_or("000".into());
309    if year.len() != 4 || month.len() != 2 || day.len() != 2 || forecast.len() != 3 {
310        panic!("Year, month, day, and forecast must be 4, 2, 2, and 3 characters, respectively.",);
311    }
312    let link = get_gfs_link(source, product, domain, year, month, day, hour, forecast);
313    // pull .idx file FIRST
314    let idxs = parsed_idx_from_url(format!("{link}.idx"), filters.unwrap_or_default(), None).await;
315    let source_data = link_to_chunks(link, &idxs).await;
316
317    GRIB2Reader::new::<BufferReader>(source_data.into(), idxs)
318}
319
320/// Get the link to download GFS Atmos data relative to IDXs
321async fn link_to_chunks(link: String, idxs: &[Grib2SectionLocations]) -> Vec<BufferReader> {
322    let mut readers: Vec<BufferReader> = vec![];
323    for Grib2SectionLocations { start, end, .. } in idxs {
324        let end = end.map_or(String::new(), |e| e.to_string());
325        let chunk =
326            fetch_url::<()>(&link, &[("Range", &format!("bytes={start}-{end}"))], None, None)
327                .await
328                .unwrap();
329        readers.push(BufferReader::new(chunk));
330    }
331
332    readers
333}
334
335/// Get the link to download GFS Atmos data
336///
337/// ## Parameters
338///
339/// - `source`: The source of the data, `aws` | `ftpprd` | `nomads` | `google` | `azure` | or a user provided url
340/// - `product`: which product to fetch
341/// - `domain`: The domain of the data, either 'atmos' for atmospheric data or 'wave' for ocean wave data
342/// - `year`: The year to fetch given a 4 digit year
343/// - `month`: The month to fetch given a 2 digit month 01 is January and 12 is December
344/// - `day`: The day to fetch given a 2 digit day, e.g. '01' or '31'
345/// - `hour`: The forecast hour with 2 digits often in increments of 6 up to 18, e.g. '00' or '12'
346/// - `forecast`: The forecast hour with 3 digits often in increments of 3 up to 384, e.g. '000' or '003'
347///
348/// ## Returns
349///
350/// A [`String`] of the specific sections
351#[allow(clippy::too_many_arguments)]
352pub fn get_gfs_link<P: Into<String>>(
353    source: Grib2GFSSource,
354    product: P,
355    domain: Grib2GFSDomain,
356    year: String,
357    month: String,
358    day: String,
359    hour: Grib2GFSHour,
360    forecast: String,
361) -> String {
362    let mut link = source.to_url();
363    let domain_str = if domain == Grib2GFSDomain::Atmos { "atmos" } else { "wave/gridded" };
364    let start_name = if domain == Grib2GFSDomain::Atmos { "gfs" } else { "gfswave" };
365    let end_name = if domain == Grib2GFSDomain::Atmos { "" } else { ".grib2" };
366    let hour: String = hour.into();
367    let product: String = product.into();
368    link = format!(
369        "{link}gfs.{year}{month}{day}/{hour}/{domain_str}/{start_name}.t{hour}z.{product}.\
370         f{forecast}{end_name}",
371    );
372
373    link
374}
375
376/// Parse the .idx file for GRIB2 section details using a URL
377///
378/// ## Parameters
379/// - `url`: The URL of the .idx file
380/// - `filters`: The filters to apply
381/// - `offset_position`: The position of the offset in the ":" sequence
382///
383/// ## Returns
384/// An array of Grib2SectionLocations
385pub async fn parsed_idx_from_url(
386    url: String,
387    filters: Vec<String>,
388    offset_position: Option<usize>,
389) -> Vec<Grib2SectionLocations> {
390    let data = fetch_url::<()>(&url, &[], None, None).await.unwrap();
391    parse_idx(String::from_utf8_lossy(&data).into(), filters, offset_position)
392}
393
394/// Parse the .idx file for GRIB2 section details
395///
396/// ## Parameters
397/// - `data`: The contents of the .idx file
398/// - `filters`: The filters to apply
399/// - `offset_position`: The position of the offset in the ":" sequence
400///
401/// ## Returns
402/// An array of Grib2SectionLocations
403pub fn parse_idx(
404    data: String,
405    filters: Vec<String>,
406    offset_position: Option<usize>,
407) -> Vec<Grib2SectionLocations> {
408    let offset_position = offset_position.unwrap_or(1);
409    let mut res = vec![];
410    // split lines, parse information, and add to array
411    for line in data.lines() {
412        if line.is_empty() {
413            continue;
414        }
415        let offset = line
416            .split(':')
417            .nth(offset_position)
418            .and_then(|s| s.trim().parse::<u64>().ok())
419            .unwrap_or(0);
420        res.push(Grib2SectionLocations {
421            start: offset,
422            end: None,
423            line: line.into(),
424            name: line.into(),
425        });
426    }
427    // now add the "end"s
428    for i in 0..res.len() - 1 {
429        res[i].end = Some(res[i + 1].start);
430    }
431    // lastly add the filters
432    if !filters.is_empty() {
433        res = res
434            .iter()
435            .filter(|s_l| filters.iter().any(|f| s_l.line.contains(f)))
436            .cloned()
437            .collect();
438    }
439    // set names to filter names
440    for i in 0..res.len() {
441        res[i].name = filters[i].clone();
442    }
443
444    res
445}
446
447/// GRIB2 Reader inputs
448#[derive(Debug)]
449pub enum GRIB2ReaderInput<T: Reader> {
450    /// A single input reader (completely unparsed)
451    Reader(T),
452    /// A list of input readers, parsed into section chunks
453    SectionChunks(Vec<BufferReader>),
454}
455impl<T: Reader> From<T> for GRIB2ReaderInput<T> {
456    fn from(reader: T) -> Self {
457        GRIB2ReaderInput::Reader(reader)
458    }
459}
460impl<T: Reader> From<Vec<BufferReader>> for GRIB2ReaderInput<T> {
461    fn from(readers: Vec<BufferReader>) -> Self {
462        GRIB2ReaderInput::SectionChunks(readers)
463    }
464}
465
466/// # GRIB2 Reader
467///
468/// ## Description
469///
470/// This class reads a GRIB2 file and returns a list of GRIB2 products.
471///
472/// Implements the [`FeatureReader`] trait
473///
474/// ## Usage
475///
476/// The methods you have access to:
477/// - [`GRIB2Reader::new`]: Create a new GRIB2Reader
478/// - [`GRIB2Reader::from_idx`]: Create a GRIB2Reader with filtered .idx file data (see [`parse_idx`] and [`parsed_idx_from_url`])
479/// - [`GRIB2Reader::get_data`]: Get the Vector MultiPoint data
480/// - [`GRIB2Reader::get_feature`]: Get the VectorFeature data
481///
482/// Associated methods that are useful:
483/// - [`fetch_gfs_data`]: Fetch ATMOS or WAVE GFS data.
484/// - [`parsed_idx_from_url`]: Given an input URL pointing to an IDX file, parse the sections
485/// - [`parse_idx`]: Given an input string of an IDX file, parse the sections
486///
487/// ### The recommended way to parse grib files is to filter out what you want:
488/// ```rust
489/// use gistools::{parsers::{BufferReader, FeatureReader}, readers::{parse_idx, GRIB2Reader}};
490/// use std::{fs, path::PathBuf};
491///
492/// let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
493/// path.push("tests/readers/grib2/fixtures/ref_sec0.gdas.t12z.pgrb2.1p00.anl.75r.grib2.txt");
494///
495/// // parse the .idx file and apply a filter that we only need 3 sections
496/// let idx_data = fs::read_to_string(path).unwrap();
497/// let sections = parse_idx(
498///     idx_data,
499///     vec![":DZDT:0.01 mb:".into(), ":TMP:0.4 mb:".into(), ":ABSV:0.4 mb:anl:".into()],
500///     None,
501/// );
502///
503/// // grab the grib2 file itself building with the filtered IDX sections
504/// let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
505/// path.push("tests/readers/grib2/fixtures/ref_sec0.gdas.t12z.pgrb2.1p00.anl.75r.grib2");
506/// let bytes = std::fs::read(path.clone()).unwrap();
507/// let grib2_reader = GRIB2Reader::from_idx(&BufferReader::from(bytes), sections);
508///
509/// let features: Vec<_> = grib2_reader.iter().collect();
510/// assert_eq!(features.len(), 1);
511/// ```
512///
513/// ### Parsing the entire grib file:
514/// ```rust
515/// use gistools::{parsers::{BufferReader, FeatureReader}, readers::GRIB2Reader};
516/// use std::{fs, path::PathBuf};
517///
518/// let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
519/// path.push("tests/readers/grib2/fixtures/ref_simple_packing.grib2");
520///
521/// let bytes = fs::read(path.clone()).unwrap();
522/// let grib2_reader = GRIB2Reader::new(BufferReader::from(bytes).into(), vec![]);
523///
524/// let features: Vec<_> = grib2_reader.iter().collect();
525/// assert_eq!(features.len(), 1);
526/// ```
527///
528/// ## Links
529/// - <https://en.wikipedia.org/wiki/GRIB>
530/// - <https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/>
531#[derive(Debug, Clone)]
532pub struct GRIB2Reader {
533    /// The GRIB2 packets
534    pub packets: RefCell<Vec<Grib2Sections>>,
535    /// The list of section locations
536    pub idxs: Vec<Grib2SectionLocations>,
537}
538impl GRIB2Reader {
539    /// Create a GRIB2Reader
540    ///
541    /// ## Parameters
542    /// - `readers`: Reader(s) for entire GRIB file. If array, its grib chunks, otherwise it will be the entire file
543    /// - `idxs`: The list of section locations we will be parsing
544    ///
545    /// ## Returns
546    /// A [`GRIB2Reader`]
547    pub fn new<T: Reader>(readers: GRIB2ReaderInput<T>, idxs: Vec<Grib2SectionLocations>) -> Self {
548        let this = GRIB2Reader { packets: vec![].into(), idxs };
549        let grib_chunks = match readers {
550            GRIB2ReaderInput::Reader(reader) => split_grib_chunks(&reader),
551            GRIB2ReaderInput::SectionChunks(chunks) => chunks,
552        };
553        for grib_chunk in grib_chunks {
554            this.packets.borrow_mut().push(split_section_chunks(grib_chunk));
555        }
556
557        this
558    }
559
560    /// Create a GRIB2Reader from a .idx file
561    ///
562    /// ## Parameters
563    /// - `source`: Either the http path to the .idx file or the entire GRIB file
564    /// - `idxs`: The parsed .idx file with the locations of each section
565    ///
566    /// ## Returns
567    /// A GRIB2Reader of the specific sections
568    pub fn from_idx<T: Reader>(source: &T, idxs: Vec<Grib2SectionLocations>) -> GRIB2Reader {
569        let mut readers: Vec<BufferReader> = vec![];
570        for idx in &idxs {
571            readers.push(BufferReader::new(source.slice(Some(idx.start), idx.end)));
572        }
573        GRIB2Reader::new::<T>(readers.into(), idxs)
574    }
575
576    /// Get the Vector Point feature data
577    pub fn get_data(&self) -> Option<VectorMultiPoint> {
578        let geo_grid = self
579            .packets
580            .borrow_mut()
581            .get_mut(0)
582            .and_then(|p| Some(p.grid_definition.as_mut()?.values.build_grid()));
583        // setup geometry
584        if let Some(mut geometry) = geo_grid {
585            // add M-Values from each packet
586            for (i, packet) in self.packets.borrow().iter().enumerate() {
587                let name = self.idxs.get(i).map(|i| i.name.clone()).unwrap_or(i.to_string());
588                if let Some(data) = packet.data.as_ref().map(|d| d.data(packet)) {
589                    for (i, geo) in geometry.iter_mut().enumerate().take(data.len()) {
590                        if let Some(m_value) = data.get(i) {
591                            if geo.m.is_none() {
592                                geo.m = Some(MValue::new());
593                            }
594                            geo.m.as_mut().unwrap().insert((&name).into(), (*m_value).into());
595                        }
596                    }
597                }
598            }
599            Some(geometry)
600        } else {
601            None
602        }
603    }
604
605    /// Get the Vector Point feature
606    pub fn get_feature(&self) -> Option<GRIB2VectorFeature> {
607        if let Some(geometry) = self.get_data() {
608            // setup metadata
609            let product_metadata: Vec<Grib2ProductDefinition> = self
610                .packets
611                .borrow()
612                .iter()
613                .filter_map(|packet| Some(packet.product_definition.as_ref()?.values.clone()))
614                .collect();
615            // setup bbox
616            let bbox = BBox3D::from_linestring(&geometry);
617            Some(GRIB2VectorFeature::new_wm(
618                None,
619                Properties::default(),
620                VectorGeometry::new_multipoint(geometry, Some(bbox)),
621                Some(product_metadata),
622            ))
623        } else {
624            None
625        }
626    }
627}
628
629/// The GRIB2 Iterator tool
630#[derive(Debug)]
631pub struct GRIB2Iterator<'a> {
632    reader: &'a GRIB2Reader,
633    done: bool,
634}
635impl Iterator for GRIB2Iterator<'_> {
636    type Item = GRIB2VectorFeature;
637
638    fn next(&mut self) -> Option<Self::Item> {
639        if self.done {
640            return None;
641        }
642        self.done = true;
643        self.reader.get_feature()
644    }
645}
646/// A feature reader trait with a callback-based approach
647impl FeatureReader<Vec<Grib2ProductDefinition>, Properties, MValue> for GRIB2Reader {
648    type FeatureIterator<'a> = GRIB2Iterator<'a>;
649
650    fn iter(&self) -> Self::FeatureIterator<'_> {
651        GRIB2Iterator { reader: self, done: false }
652    }
653
654    fn par_iter(&self, _pool_size: usize, thread_id: usize) -> Self::FeatureIterator<'_> {
655        if thread_id == 0 { self.iter() } else { GRIB2Iterator { reader: self, done: true } }
656    }
657}
658
659/// Split the bytes of the GRIB file into individual GRIB chunks that represent sections
660///
661/// ## Parameters
662/// - `reader`: Reader for entire GRIB file
663///
664/// ## Returns
665/// Array of GRIB Chunk Buffers containing individual GRIB definitions in file
666fn split_grib_chunks<T: Reader>(reader: &T) -> Vec<BufferReader> {
667    if reader.len() == 0 {
668        return vec![];
669    }
670    let length = reader.uint64_be(Some(8));
671    let grib_data = BufferReader::new(reader.slice(Some(0), Some(length)));
672
673    let mut chunks: Vec<BufferReader> = vec![grib_data];
674    if length == reader.len() {
675        return chunks;
676    }
677    let rest = BufferReader::new(reader.slice(Some(length), None));
678    chunks.append(&mut split_grib_chunks(&rest));
679
680    chunks
681}
gistools/readers/grib2/mod.rs

gistools/readers/grib2/
mod.rs
