hrdf_parser/
storage.rs

1use std::{path::Path, time::Instant};
2
3use chrono::{Days, NaiveDate};
4use rustc_hash::{FxHashMap, FxHashSet};
5use serde::{Deserialize, Serialize};
6
7use crate::{
8    JourneyError, JourneyId,
9    error::{HResult, HrdfError},
10    models::{
11        Attribute, BitField, Direction, ExchangeTimeAdministration, ExchangeTimeJourney,
12        ExchangeTimeLine, Holiday, InformationText, Journey, JourneyPlatform, Line, Model,
13        Platform, Stop, StopConnection, ThroughService, TimetableMetadataEntry, TransportCompany,
14        TransportType, Version,
15    },
16    parsing,
17    utils::{count_days_between_two_dates, timetable_end_date, timetable_start_date},
18};
19
20// ------------------------------------------------------------------------------------------------
21// --- DataStorage
22// ------------------------------------------------------------------------------------------------
23//
24
25#[derive(Debug, Serialize, Deserialize)]
26pub struct DataStorage {
27    // Time-relevant data.
28    bit_fields: ResourceStorage<BitField>,
29    holidays: ResourceStorage<Holiday>,
30    timetable_metadata: ResourceStorage<TimetableMetadataEntry>,
31
32    // Basic data.
33    attributes: ResourceStorage<Attribute>,
34    information_texts: ResourceStorage<InformationText>,
35    directions: ResourceStorage<Direction>,
36    lines: ResourceStorage<Line>,
37    transport_companies: ResourceStorage<TransportCompany>,
38    transport_types: ResourceStorage<TransportType>,
39
40    // Stop data
41    stops: ResourceStorage<Stop>,
42    stop_connections: ResourceStorage<StopConnection>,
43
44    // Timetable data
45    journeys: ResourceStorage<Journey>,
46    journey_platform: ResourceStorage<JourneyPlatform>,
47    platforms: ResourceStorage<Platform>,
48    through_service: ResourceStorage<ThroughService>,
49
50    // Exchange times
51    exchange_times_administration: ResourceStorage<ExchangeTimeAdministration>,
52    exchange_times_journey: ResourceStorage<ExchangeTimeJourney>,
53    exchange_times_line: ResourceStorage<ExchangeTimeLine>,
54
55    // Maps
56    bit_fields_by_day: FxHashMap<NaiveDate, FxHashSet<i32>>,
57    bit_fields_by_stop_id: FxHashMap<i32, FxHashSet<i32>>,
58    journeys_by_stop_id_and_bit_field_id: FxHashMap<(i32, i32), Vec<i32>>,
59    stop_connections_by_stop_id: FxHashMap<i32, FxHashSet<i32>>,
60    bit_field_id_for_through_service_by_journey_id_stop_id:
61        FxHashMap<(JourneyId, JourneyId, i32), i32>,
62    exchange_times_administration_map: FxHashMap<(Option<i32>, String, String), i32>,
63    exchange_times_journey_map: FxHashMap<(i32, JourneyId, JourneyId), FxHashSet<i32>>,
64
65    // Additional global data
66    default_exchange_time: (i16, i16), // (InterCity exchange time, Exchange time for all other journey types)
67}
68
69impl DataStorage {
70    pub fn new(version: Version, path: &Path) -> HResult<Self> {
71        // Time-relevant data
72        let complete = Instant::now();
73        let now = Instant::now();
74        let bit_fields = parsing::load_bit_fields(path)?;
75        log::info!("Time elapsed for bitfields parsing: {:?}", now.elapsed());
76        let now = Instant::now();
77        let holidays = parsing::load_holidays(path)?;
78        log::info!("Time elapsed for holidays parsing: {:?}", now.elapsed());
79
80        let now = Instant::now();
81        let timetable_metadata = parsing::load_timetable_metadata(path)?;
82        log::info!(
83            "Time elapsed for timetable_metadata parsing: {:?}",
84            now.elapsed()
85        );
86
87        // Basic data
88        let now = Instant::now();
89        let (attributes, attributes_pk_type_converter) = parsing::load_attributes(path)?;
90        log::info!("Time elapsed for attributes parsing: {:?}", now.elapsed());
91        let now = Instant::now();
92        let (directions, directions_pk_type_converter) = parsing::load_directions(path)?;
93        log::info!("Time elapsed for directions parsing: {:?}", now.elapsed());
94        let now = Instant::now();
95        let information_texts = parsing::load_information_texts(path)?;
96        log::info!(
97            "Time elapsed for information_texts parsing: {:?}",
98            now.elapsed()
99        );
100        let now = Instant::now();
101        let lines = parsing::load_lines(path)?;
102        log::info!("Time elapsed for line parsing: {:?}", now.elapsed());
103        let now = Instant::now();
104        let transport_companies = parsing::load_transport_companies(path)?;
105        log::info!(
106            "Time elapsed for transport_companies parsing: {:?}",
107            now.elapsed()
108        );
109        let now = Instant::now();
110        let (transport_types, transport_types_pk_type_converter) =
111            parsing::load_transport_types(path)?;
112        log::info!(
113            "Time elapsed for transport_types parsing: {:?}",
114            now.elapsed()
115        );
116
117        // Stop data
118        let now = Instant::now();
119        let stop_connections = parsing::load_stop_connections(path, &attributes_pk_type_converter)?;
120        log::info!(
121            "Time elapsed for stop_connections parsing: {:?}",
122            now.elapsed()
123        );
124        let now = Instant::now();
125        let (stops, default_exchange_time) = parsing::load_stops(version, path)?;
126        log::info!("Time elapsed for stops parsing: {:?}", now.elapsed());
127
128        // Timetable data
129        let now = Instant::now();
130        let (journeys, journeys_pk_type_converter) = parsing::load_journeys(
131            path,
132            &transport_types_pk_type_converter,
133            &attributes_pk_type_converter,
134            &directions_pk_type_converter,
135        )?;
136        log::info!("Time elapsed for journeys parsing: {:?}", now.elapsed());
137
138        let now = Instant::now();
139        let (journey_platform, platforms) =
140            parsing::load_platforms(version, path, &journeys_pk_type_converter)?;
141        log::info!("Time elapsed for platforms parsing: {:?}", now.elapsed());
142        let now = Instant::now();
143        let through_service = parsing::load_through_service(path, &journeys_pk_type_converter)?;
144        log::info!(
145            "Time elapsed for through_service parsing: {:?}",
146            now.elapsed()
147        );
148
149        // Exchange times
150        let now = Instant::now();
151        let exchange_times_administration = parsing::load_exchange_times_administration(path)?;
152        log::info!(
153            "Time elapsed for exchange_times_administration parsing: {:?}",
154            now.elapsed()
155        );
156        let now = Instant::now();
157        let exchange_times_journey =
158            parsing::load_exchange_times_journey(path, &journeys_pk_type_converter)?;
159        log::info!(
160            "Time elapsed for exchange_times_journey parsing: {:?}",
161            now.elapsed()
162        );
163        let now = Instant::now();
164        let exchange_times_line =
165            parsing::load_exchange_times_line(path, &transport_types_pk_type_converter)?;
166        log::info!(
167            "Time elapsed for exchange_times_line parsing: {:?}",
168            now.elapsed()
169        );
170
171        log::info!("Parsing of all HRDF files in {:?}", complete.elapsed());
172
173        log::info!("Building bit_fields_by_day...");
174        let bit_fields_by_day = create_bit_fields_by_day(&bit_fields, &timetable_metadata)?;
175        log::info!("Building bit_fields_by_stop_id...");
176        let bit_fields_by_stop_id = create_bit_fields_by_stop_id(&journeys)?;
177        log::info!("Building journeys by stop id and bit field_id...");
178        let journeys_by_stop_id_and_bit_field_id =
179            create_journeys_by_stop_id_and_bit_field_id(&journeys)?;
180        log::info!("Building stop connections by stop id...");
181        let bit_field_id_for_through_service_by_journey_id_stop_id =
182            create_bit_field_id_through_service_by_journey_id_stop_id(&through_service);
183        log::info!("Building stop connections by stop id...");
184        let stop_connections_by_stop_id = create_stop_connections_by_stop_id(&stop_connections);
185        log::info!("Building exchange times administration map...");
186        let exchange_times_administration_map =
187            create_exchange_times_administration_map(&exchange_times_administration);
188        log::info!("Building exchange times journey_map...");
189        let exchange_times_journey_map = create_exchange_times_journey_map(&exchange_times_journey);
190        log::info!("Building through service map...");
191
192        let data_storage = Self {
193            // Time-relevant data
194            bit_fields,
195            holidays,
196            timetable_metadata,
197            // Basic data
198            attributes,
199            information_texts,
200            directions,
201            lines,
202            transport_companies,
203            transport_types,
204            // Stop data
205            stop_connections,
206            stops,
207            // Timetable data
208            journeys,
209            journey_platform,
210            platforms,
211            through_service,
212            // Exchange times
213            exchange_times_administration,
214            exchange_times_journey,
215            exchange_times_line,
216            // Maps
217            bit_fields_by_day,
218            bit_fields_by_stop_id,
219            journeys_by_stop_id_and_bit_field_id,
220            stop_connections_by_stop_id,
221            bit_field_id_for_through_service_by_journey_id_stop_id,
222            exchange_times_administration_map,
223            exchange_times_journey_map,
224            // Additional global data
225            default_exchange_time,
226        };
227
228        Ok(data_storage)
229    }
230
231    // Getters/Setters
232
233    pub fn bit_fields(&self) -> &ResourceStorage<BitField> {
234        &self.bit_fields
235    }
236
237    pub fn journeys(&self) -> &ResourceStorage<Journey> {
238        &self.journeys
239    }
240
241    pub fn lines(&self) -> &ResourceStorage<Line> {
242        &self.lines
243    }
244
245    pub fn platforms(&self) -> &ResourceStorage<Platform> {
246        &self.platforms
247    }
248
249    pub fn stop_connections(&self) -> &ResourceStorage<StopConnection> {
250        &self.stop_connections
251    }
252
253    pub fn through_service(&self) -> &ResourceStorage<ThroughService> {
254        &self.through_service
255    }
256
257    pub fn stops(&self) -> &ResourceStorage<Stop> {
258        &self.stops
259    }
260
261    pub fn transport_types(&self) -> &ResourceStorage<TransportType> {
262        &self.transport_types
263    }
264
265    pub fn timetable_metadata(&self) -> &ResourceStorage<TimetableMetadataEntry> {
266        &self.timetable_metadata
267    }
268
269    pub fn exchange_times_administration(&self) -> &ResourceStorage<ExchangeTimeAdministration> {
270        &self.exchange_times_administration
271    }
272
273    pub fn exchange_times_journey(&self) -> &ResourceStorage<ExchangeTimeJourney> {
274        &self.exchange_times_journey
275    }
276
277    pub fn exchange_times_line(&self) -> &ResourceStorage<ExchangeTimeLine> {
278        &self.exchange_times_line
279    }
280
281    pub fn bit_fields_by_day(&self) -> &FxHashMap<NaiveDate, FxHashSet<i32>> {
282        &self.bit_fields_by_day
283    }
284
285    pub fn bit_fields_by_stop_id(&self) -> &FxHashMap<i32, FxHashSet<i32>> {
286        &self.bit_fields_by_stop_id
287    }
288
289    pub fn journeys_by_stop_id_and_bit_field_id(&self) -> &FxHashMap<(i32, i32), Vec<i32>> {
290        &self.journeys_by_stop_id_and_bit_field_id
291    }
292
293    pub fn stop_connections_by_stop_id(&self) -> &FxHashMap<i32, FxHashSet<i32>> {
294        &self.stop_connections_by_stop_id
295    }
296
297    pub fn bit_field_id_for_through_service_by_journey_id_stop_id(
298        &self,
299    ) -> &FxHashMap<(JourneyId, JourneyId, i32), i32> {
300        &self.bit_field_id_for_through_service_by_journey_id_stop_id
301    }
302
303    pub fn exchange_times_administration_map(
304        &self,
305    ) -> &FxHashMap<(Option<i32>, String, String), i32> {
306        &self.exchange_times_administration_map
307    }
308
309    pub fn exchange_times_journey_map(
310        &self,
311    ) -> &FxHashMap<(i32, JourneyId, JourneyId), FxHashSet<i32>> {
312        &self.exchange_times_journey_map
313    }
314
315    pub fn default_exchange_time(&self) -> (i16, i16) {
316        self.default_exchange_time
317    }
318}
319
320// ------------------------------------------------------------------------------------------------
321// --- ResourceStorage
322// ------------------------------------------------------------------------------------------------
323
324#[derive(Debug, Serialize, Deserialize)]
325pub struct ResourceStorage<M: Model<M>> {
326    data: FxHashMap<M::K, M>,
327}
328
329impl<M: Model<M>> ResourceStorage<M> {
330    pub fn new(data: FxHashMap<M::K, M>) -> Self {
331        Self { data }
332    }
333
334    pub fn data(&self) -> &FxHashMap<M::K, M> {
335        &self.data
336    }
337
338    pub fn find(&self, k: M::K) -> Option<&M> {
339        // TODO: there might be a problem when k is not in data so we can't unwrap here
340        self.data().get(&k)
341    }
342
343    pub fn entries(&self) -> Vec<&M> {
344        self.data.values().collect()
345    }
346
347    pub fn resolve_ids(&self, ids: &FxHashSet<M::K>) -> Option<Vec<&M>> {
348        ids.iter().map(|&id| self.find(id)).collect()
349    }
350}
351
352// ------------------------------------------------------------------------------------------------
353// --- Maps
354// ------------------------------------------------------------------------------------------------
355
356fn create_bit_fields_by_day(
357    bit_fields: &ResourceStorage<BitField>,
358    timetable_metadata: &ResourceStorage<TimetableMetadataEntry>,
359) -> HResult<FxHashMap<NaiveDate, FxHashSet<i32>>> {
360    let start_date = timetable_start_date(timetable_metadata)?;
361    let num_days =
362        count_days_between_two_dates(start_date, timetable_end_date(timetable_metadata)?);
363
364    let dates = (0..num_days)
365        .map(|i| {
366            let i = i.try_into().unwrap();
367            start_date
368                // unwrap: Converting i from usize to u64 will never fail.
369                .checked_add_days(Days::new(i))
370                .ok_or(HrdfError::FailedToAddDays(start_date, i))
371        })
372        .collect::<HResult<Vec<NaiveDate>>>()?;
373
374    let mut map = FxHashMap::default();
375    dates.iter().for_each(|date| {
376        map.entry(*date).or_insert(FxHashSet::default()).insert(0);
377    });
378
379    bit_fields
380        .data()
381        .keys()
382        .try_fold(map, |mut acc, bit_field_id| {
383            let bit_field = bit_fields
384                .find(*bit_field_id)
385                .ok_or(HrdfError::BitFieldIdNotFound(*bit_field_id))?;
386            let indexes: Vec<usize> = bit_field
387                .bits()
388                .iter()
389                // The first two bits must be ignored.
390                .skip(2)
391                .enumerate()
392                .filter(|&(ref i, &x)| *i < num_days && x == 1)
393                .map(|(i, _)| i)
394                .collect();
395
396            indexes.iter().for_each(|&i| {
397                acc.entry(dates[i]).or_default().insert(bit_field.id());
398            });
399
400            Ok(acc)
401        })
402}
403
404fn create_bit_fields_by_stop_id(
405    journeys: &ResourceStorage<Journey>,
406) -> HResult<FxHashMap<i32, FxHashSet<i32>>> {
407    journeys.entries().into_iter().try_fold(
408        FxHashMap::default(),
409        |mut acc: FxHashMap<i32, FxHashSet<i32>>, journey| {
410            journey.route().iter().try_for_each(|route_entry| {
411                acc.entry(route_entry.stop_id())
412                    .or_default()
413                    // If the journey has no bit_field_id, the default value is 0. A value of 0 means that the journey operates every day.
414                    .insert(journey.bit_field_id()?.unwrap_or(0));
415                Ok::<(), JourneyError>(())
416            })?;
417            Ok(acc)
418        },
419    )
420}
421
422fn create_journeys_by_stop_id_and_bit_field_id(
423    journeys: &ResourceStorage<Journey>,
424) -> HResult<FxHashMap<(i32, i32), Vec<i32>>> {
425    journeys.entries().into_iter().try_fold(
426        FxHashMap::default(),
427        |mut acc: FxHashMap<(i32, i32), Vec<i32>>, journey| {
428            journey.route().iter().try_for_each(|route_entry| {
429                // If the journey has no bit_field_id, the default value is 0. A value of 0 means that the journey operates every day.
430                acc.entry((route_entry.stop_id(), journey.bit_field_id()?.unwrap_or(0)))
431                    .or_default()
432                    .push(journey.id());
433                Ok::<(), JourneyError>(())
434            })?;
435            Ok(acc)
436        },
437    )
438}
439
440/// Given journey_stop_id, and journey_id_1, journey_id_2, we obtain the bit_field_id of the ThroughService
441fn create_bit_field_id_through_service_by_journey_id_stop_id(
442    through_services: &ResourceStorage<ThroughService>,
443) -> FxHashMap<(JourneyId, JourneyId, i32), i32> {
444    through_services
445        .entries()
446        .into_iter()
447        .fold(FxHashMap::default(), |mut acc, through_service| {
448            let journey_1_id = through_service.journey_1_id();
449            let journey_2_id = through_service.journey_2_id();
450            let journey_stop_id = through_service.journey_1_stop_id();
451            let bit_field_id = through_service.bit_field_id();
452
453            acc.insert(
454                (journey_1_id.clone(), journey_2_id.clone(), journey_stop_id),
455                bit_field_id,
456            );
457            acc
458        })
459}
460
461fn create_stop_connections_by_stop_id(
462    stop_connections: &ResourceStorage<StopConnection>,
463) -> FxHashMap<i32, FxHashSet<i32>> {
464    stop_connections
465        .entries()
466        .into_iter()
467        .fold(FxHashMap::default(), |mut acc, stop_connection| {
468            acc.entry(stop_connection.stop_id_1())
469                .or_default()
470                .insert(stop_connection.id());
471            acc
472        })
473}
474
475fn create_exchange_times_journey_map(
476    exchange_times_journey: &ResourceStorage<ExchangeTimeJourney>,
477) -> FxHashMap<(i32, JourneyId, JourneyId), FxHashSet<i32>> {
478    exchange_times_journey.entries().into_iter().fold(
479        FxHashMap::default(),
480        |mut acc, exchange_time| {
481            let key = (
482                exchange_time.stop_id(),
483                (
484                    exchange_time.journey_legacy_id_1(),
485                    exchange_time.administration_1().to_string(),
486                ),
487                (
488                    exchange_time.journey_legacy_id_2(),
489                    exchange_time.administration_2().to_string(),
490                ),
491            );
492
493            acc.entry(key).or_default().insert(exchange_time.id());
494            acc
495        },
496    )
497}
498
499fn create_exchange_times_administration_map(
500    exchange_times_administration: &ResourceStorage<ExchangeTimeAdministration>,
501) -> FxHashMap<(Option<i32>, String, String), i32> {
502    exchange_times_administration.entries().into_iter().fold(
503        FxHashMap::default(),
504        |mut acc, exchange_time| {
505            let key = (
506                exchange_time.stop_id(),
507                exchange_time.administration_1().into(),
508                exchange_time.administration_2().into(),
509            );
510
511            acc.insert(key, exchange_time.id());
512            acc
513        },
514    )
515}
hrdf_parser/storage.rs

hrdf_parser/
storage.rs
