aemo_rs/
lib.rs

1// policy is to have an overall deny, and place allow attributes where needed 
2#![deny(clippy::all)]
3#![deny(warnings)]
4use serde::de::Error as SerdeDeError;
5use serde::{de, Deserialize, Serialize};
6use std::{collections, convert, error, fmt, io, num}; // need to expose trait but don't want to use name
7
8use chrono::TimeZone;
9use chrono_tz::Australia::Brisbane;
10use log::info;
11
12pub mod daily;
13pub mod dispatch_is;
14pub mod dispatch_scada;
15pub mod predispatch_is;
16pub mod predispatch_sensitivities;
17pub mod rooftop_actual;
18pub mod rooftop_forecast;
19pub mod yestbid;
20
21// this is useful to get the date part of nem settlementdate / lastchanged fields
22pub fn to_nem_date(ndt: &chrono::NaiveDateTime) -> chrono::Date<chrono_tz::Tz> {
23    Brisbane.from_local_datetime(ndt).unwrap().date()
24}
25
26#[derive(Debug)]
27pub enum Error {
28    /// This occurs when we are missing the footer record which lists the number of rows in the file
29    MissingFooterRecord,
30    MissingHeaderRecord,
31    /// This occurs when the desired file key can't be found in the RawAemoFile
32    MissingFile(FileKey),
33    /// This occurs when an entire row is empty after the first three columns
34    EmptyRow,
35    UnexpectedRowType(String),
36    TooShortRow(usize),
37    IncorrectLineCount { got: usize, expected: usize },
38    ThreadBroken,
39    ParseInt(num::ParseIntError),
40    Csv(csv::Error),
41}
42
43impl fmt::Display for Error {
44    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
45        match self {
46            Self::MissingHeaderRecord => write!(f, "aemo file is missing the first `c` record"),
47            Self::MissingFooterRecord => write!(f, "aemo file is missing the final `c` record"),
48            Self::MissingFile((name, sub_name, version)) => write!(
49                f,
50                "aemo file was missing {}.{}.v{} section in the file ",
51                name, sub_name, version
52            ),
53            Self::EmptyRow => write!(f, "aemo file row is empty"),
54            Self::UnexpectedRowType(t) => write!(f, "unexpeted row type of {}", t),
55            Self::TooShortRow(len) => {
56                write!(f, "aemo file data row of length {} is too short", len)
57            }
58            Self::IncorrectLineCount { got, expected } => write!(
59                f,
60                "aemo file was supposed to be {} lines long but was instead {} lines long",
61                expected, got
62            ),
63            Self::ThreadBroken => write!(f, "Broken Thread"),
64            Self::ParseInt(e) => write!(f, "parse int error: {}", e),
65            Self::Csv(e) => write!(f, "csv error: {}", e),
66        }
67    }
68}
69
70impl From<num::ParseIntError> for Error {
71    fn from(error: num::ParseIntError) -> Self {
72        Error::ParseInt(error)
73    }
74}
75
76impl From<csv::Error> for Error {
77    fn from(error: csv::Error) -> Self {
78        Error::Csv(error)
79    }
80}
81
82impl error::Error for Error {}
83
84type Result<T> = std::result::Result<T, Error>;
85
86#[derive(Deserialize, Serialize, Debug, Clone)]
87pub struct AemoHeader {
88    record_type: char,
89    data_source: String,
90    file_name: String,
91    participant_name: String,
92    privacy_level: String,
93    #[serde(deserialize_with = "au_date_deserialize")]
94    effective_date: chrono::NaiveDate,
95    #[serde(deserialize_with = "au_time_deserialize")]
96    effective_time: chrono::NaiveTime,
97    serial_number: u64,
98    file_name_2: String,
99    serial_number_2: u64,
100}
101
102#[derive(Deserialize, Serialize, Debug, Clone)]
103struct AemoFooter {
104    record_type: char,
105    end_of_report: String,
106    line_count_inclusive: usize,
107}
108
109#[derive(Debug, Clone)]
110pub struct RawAemoFile {
111    pub header: AemoHeader,
112    pub data: collections::HashMap<FileKey, Vec<csv::StringRecord>>,
113    //footer: AemoFooter, // don't reall
114}
115
116pub type FileKey = (String, String, i32);
117
118// potentially have RawAemoFile<T> where T: forms the key of the hashmap??
119
120impl RawAemoFile {
121    pub fn from_bufread(br: impl io::Read) -> Result<Self> {
122        let mut reader = csv::ReaderBuilder::new()
123            .has_headers(false)
124            .flexible(true)
125            .from_reader(br);
126        let mut records = reader.records();
127        let header: AemoHeader = records
128            .next()
129            .ok_or(Error::MissingHeaderRecord)??
130            .deserialize(None)?;
131
132        // placeholder
133        let mut footer: Result<AemoFooter> = Err(Error::MissingFooterRecord);
134        let mut data: collections::HashMap<FileKey, Vec<csv::StringRecord>> =
135            collections::HashMap::new();
136
137        for record in records {
138            let record = record?;
139            match record.get(0) {
140                Some("C") => {
141                    footer = record.deserialize(None).map_err(convert::Into::into);
142                }
143                Some("D") => {
144                    let row_len = record.len();
145                    if row_len < 5 {
146                        return Err(Error::TooShortRow(row_len));
147                    }
148                    let file: String = record[1].into();
149                    let sub_file: String = record[2].into();
150                    let sub_file_version: i32 = record[3].parse()?;
151
152                    // remove the unwanted fields from the stringrecord
153                    let rest_record =
154                        record
155                            .into_iter()
156                            .skip(4)
157                            .fold(csv::StringRecord::new(), |mut acc, x| {
158                                acc.push_field(x);
159                                acc
160                            });
161
162                    if let Some((k, mut v)) =
163                        data.remove_entry(&(file.clone(), sub_file.clone(), sub_file_version))
164                    {
165                        v.push(rest_record);
166                        data.insert(k, v);
167                    } else {
168                        data.insert(
169                            (file.clone(), sub_file.clone(), sub_file_version),
170                            vec![rest_record],
171                        );
172                    }
173
174                    // would be more ideal but can't use because rest_record is moved into the first closure
175                    // data.entry((sub_file, sub_file_version))
176                    //     .and_modify(|v| v.push(rest_record))
177                    //     .or_insert(vec![rest_record.clone()]);
178                }
179                Some("I") => continue, //"i" row, or unexpected row
180                Some(t) => return Err(Error::UnexpectedRowType(t.into())), //unexpected row, as correct files only have "C", "I" and "D"
181                None => return Err(Error::EmptyRow),
182            }
183        }
184        // set footer
185        let expected_line_count = footer?.line_count_inclusive;
186
187        let file = Self { header, data };
188
189        let data_rows = file.data.iter().fold(0, |acc, (_, v)| acc + 1 + v.len());
190
191        if data_rows + 2 == expected_line_count {
192            Ok(file)
193        } else {
194            Err(Error::IncorrectLineCount {
195                got: data_rows + 2,
196                expected: expected_line_count,
197            })
198        }
199    }
200}
201
202pub trait FileKeyable {
203    fn key() -> FileKey;
204}
205
206pub trait GetFromRawAemo {
207    type Output: FileKeyable + serde::de::DeserializeOwned;
208    fn from_map(
209        data: &mut collections::HashMap<FileKey, Vec<csv::StringRecord>>,
210    ) -> Result<Vec<Self::Output>> {
211        let key = &Self::Output::key();
212        info!("Extracting file {:?}", key);
213        data.remove_entry(key)
214            .ok_or_else(|| Error::MissingFile(Self::Output::key()))?
215            .1
216            .into_iter()
217            .map(|rec| rec.deserialize(None))
218            .collect::<std::result::Result<Vec<Self::Output>, csv::Error>>()
219            .map_err(convert::Into::into)
220    }
221}
222
223pub trait AemoFile: Sized + Send {
224    fn from_raw(raw: RawAemoFile) -> Result<Self>;
225}
226
227fn au_datetime_deserialize<'de, D>(d: D) -> std::result::Result<chrono::NaiveDateTime, D::Error>
228where
229    D: serde::Deserializer<'de>,
230{
231    let s = serde::Deserialize::deserialize(d)?;
232    chrono::NaiveDateTime::parse_from_str(s, "%Y/%m/%d %H:%M:%S").map_err(de::Error::custom)
233}
234
235fn opt_au_datetime_deserialize<'de, D>(
236    d: D,
237) -> std::result::Result<Option<chrono::NaiveDateTime>, D::Error>
238where
239    D: serde::Deserializer<'de>,
240{
241    let a_str: &'de str = serde::Deserialize::deserialize(d)?;
242    if a_str.len() == 0 {
243        Ok(None)
244    } else {
245        chrono::NaiveDateTime::parse_from_str(a_str, "%Y/%m/%d %H:%M:%S")
246            .map_err(D::Error::custom)
247            .map(Some)
248    }
249}
250
251fn au_date_deserialize<'de, D>(d: D) -> std::result::Result<chrono::NaiveDate, D::Error>
252where
253    D: serde::Deserializer<'de>,
254{
255    let s = serde::Deserialize::deserialize(d)?;
256    chrono::NaiveDate::parse_from_str(s, "%Y/%m/%d").map_err(de::Error::custom)
257}
258
259fn au_time_deserialize<'de, D>(d: D) -> std::result::Result<chrono::NaiveTime, D::Error>
260where
261    D: serde::Deserializer<'de>,
262{
263    let s = serde::Deserialize::deserialize(d)?;
264    chrono::NaiveTime::parse_from_str(s, "%H:%M:%S").map_err(de::Error::custom)
265}
aemo_rs/lib.rs

aemo_rs/
lib.rs
