Skip to main content

ngano/
model.rs

1//! Catalogue record types: datasets, countries, languages and aggregates.
2
3use serde::{Deserialize, Serialize};
4use std::collections::BTreeMap;
5
6/// One catalogue record: a published dataset, as described by its source.
7///
8/// Figures are reported as the source publishes them. [`Dataset::unverified_size`]
9/// marks self-reported totals of 20,000 hours or more, which are excluded from
10/// every hours aggregate in this crate.
11#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
12pub struct Dataset {
13    /// Stable ngano identifier, for example `waxal-corpus-paper`.
14    pub id: String,
15    /// Human readable dataset name.
16    pub name: String,
17    /// `ASR`, `TTS`, `ASR+TTS`, `Raw source` or `Other`.
18    pub task: String,
19    /// Language variety grouping, for example `Indigenous`.
20    pub variety: String,
21    /// Languages exactly as the source lists them.
22    #[serde(default)]
23    pub languages: Vec<String>,
24    /// Languages after normalisation against `data/languages.json`, in the
25    /// same order as [`Dataset::language_tags`].
26    #[serde(default)]
27    pub languages_clean: Vec<String>,
28    /// ISO 639-3 codes as the record was first catalogued. Prefer
29    /// [`Dataset::language_codes`], which is derived from the tags.
30    #[serde(default)]
31    pub iso: Vec<String>,
32    /// Canonical BCP 47 language tags, ordered, one per language the record
33    /// covers. The primary subtag is an ISO 639-3 code and a region subtag
34    /// marks a country-specific variety, so `eng-NG` is Nigerian English.
35    #[serde(default)]
36    pub language_tags: Vec<String>,
37    /// The bare ISO 639-3 codes behind [`Dataset::language_tags`],
38    /// deduplicated, so every regional variety collapses into its code.
39    #[serde(default)]
40    pub language_codes: Vec<String>,
41    /// An editorial note about how this record names its languages, present on
42    /// the handful of sources that describe their coverage in prose rather than
43    /// naming languages one by one.
44    #[serde(default)]
45    pub language_note: Option<String>,
46    /// Country names as the source lists them.
47    #[serde(default)]
48    pub countries: Vec<String>,
49    /// ISO 3166-1 alpha-2 codes for [`Dataset::countries`].
50    #[serde(default)]
51    pub country_codes: Vec<String>,
52    /// Regional groupings, for example `East Africa`.
53    #[serde(default)]
54    pub regions: Vec<String>,
55    /// Hours as published, verbatim, including any qualifying text.
56    #[serde(default)]
57    pub hours: Option<String>,
58    /// Numeric hours parsed from [`Dataset::hours`], when one could be parsed.
59    #[serde(default)]
60    pub hours_num: Option<f64>,
61    /// Speaker count as published.
62    #[serde(default)]
63    pub speakers: Option<String>,
64    /// Recording setup, for example `telephone` or `studio`.
65    #[serde(default)]
66    pub recording_type: Option<String>,
67    /// Audio quality band, for example `Standard (16 kHz)`.
68    pub quality: String,
69    /// `Transcribed`, `Unlabelled` or `Unstated`.
70    pub labelled: String,
71    /// Recording domain, for example `Broadcast news`.
72    pub domain: String,
73    /// Licence string as published.
74    pub licence: String,
75    /// Licence family, for example `Attribution (CC-BY)`.
76    pub licence_class: String,
77    /// `Yes`, `Yes, if purchased`, `No` or `Unstated`.
78    pub commercial: String,
79    /// `Open`, `Request`, `Paid`, `Scrape required` or `Unclear`.
80    pub access: String,
81    /// Where the data lives, for example `HuggingFace`.
82    pub host: String,
83    /// Canonical URL for the dataset.
84    #[serde(default)]
85    pub url: Option<String>,
86    /// Hugging Face repo id, when the dataset is hosted there.
87    #[serde(default)]
88    pub hf_repo: Option<String>,
89    /// Publication year as published.
90    #[serde(default)]
91    pub year: Option<String>,
92    /// Editorial notes about scope, caveats and provenance.
93    #[serde(default)]
94    pub notes: String,
95    /// True when the published size is self-reported and 20,000 hours or more.
96    /// Such figures are excluded from every total.
97    #[serde(default)]
98    pub unverified_size: bool,
99}
100
101impl Dataset {
102    /// Hours that may be counted in a total: `None` when the figure is missing
103    /// or flagged [`Dataset::unverified_size`].
104    ///
105    /// ```
106    /// use ngano::Catalogue;
107    ///
108    /// let cat = Catalogue::bundled()?;
109    /// let counted: f64 = cat.all().iter().filter_map(|d| d.countable_hours()).sum();
110    /// assert!(counted > 0.0);
111    /// # Ok::<(), ngano::NganoError>(())
112    /// ```
113    pub fn countable_hours(&self) -> Option<f64> {
114        if self.unverified_size {
115            None
116        } else {
117            self.hours_num
118        }
119    }
120
121    /// True when the dataset can be streamed, meaning it has a Hugging Face repo.
122    pub fn is_streamable(&self) -> bool {
123        self.hf_repo.as_deref().is_some_and(|r| !r.is_empty())
124    }
125
126    /// True when the licence permits commercial use outright or after purchase.
127    pub fn allows_commercial_use(&self) -> bool {
128        let c = self.commercial.to_ascii_lowercase();
129        c.starts_with("yes")
130    }
131}
132
133/// A country in the catalogue's coverage map.
134#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
135pub struct Country {
136    /// Canonical country name.
137    pub name: String,
138    /// ISO 3166-1 alpha-2 code.
139    pub iso2: String,
140    /// ISO 3166-1 alpha-3 code.
141    pub iso3: String,
142    /// Name used by the bundled GeoJSON polygons. `None` for island states,
143    /// which have no polygon and are drawn from `lat` and `lon` instead.
144    #[serde(default)]
145    pub map_name: Option<String>,
146    /// Centroid latitude.
147    pub lat: f64,
148    /// Centroid longitude.
149    pub lon: f64,
150    /// Regional grouping, for example `Southern Africa`.
151    pub region: String,
152    /// URL slug, normally the lowercased alpha-2 code.
153    pub slug: String,
154    /// Dataset count, when the record came from the HTTP API.
155    #[serde(default)]
156    pub datasets: Option<u32>,
157    /// Counted hours, when the record came from the HTTP API.
158    #[serde(default)]
159    pub hours: Option<f64>,
160}
161
162/// A language in the catalogue's coverage map, keyed on its BCP 47 tag.
163///
164/// The registry entry behind the tag is [`LanguageCode`](crate::LanguageCode);
165/// this record adds what the catalogue knows about the language's coverage.
166#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
167pub struct Language {
168    /// Canonical BCP 47 tag, for example `sna` or `eng-NG`.
169    pub tag: String,
170    /// The bare ISO 639-3 code behind the tag.
171    pub iso639_3: String,
172    /// ISO 3166-1 alpha-2 region subtag, when the tag names a country-specific
173    /// variety.
174    #[serde(default)]
175    pub region: Option<String>,
176    /// Canonical language name.
177    pub name: String,
178    /// ISO 639-3 scope: `I` individual, `M` macrolanguage, `S` special.
179    #[serde(default)]
180    pub scope: Option<String>,
181    /// ISO 639-3 type: `L` living, `E` extinct, and so on. Named `kind`
182    /// because `type` is a Rust keyword.
183    #[serde(default, rename = "type")]
184    pub kind: Option<String>,
185    /// Every catalogue spelling that resolves to this tag.
186    #[serde(default)]
187    pub aliases: Vec<String>,
188    /// URL slug, which is the tag lowercased.
189    pub slug: String,
190    /// Number of datasets covering it.
191    #[serde(default)]
192    pub datasets: Option<u32>,
193    /// Counted hours across its datasets.
194    #[serde(default)]
195    pub hours: Option<f64>,
196    /// Countries where the catalogue records it, by name.
197    #[serde(default)]
198    pub countries: Vec<String>,
199    /// ISO 3166-1 alpha-2 codes for [`Language::countries`].
200    #[serde(default)]
201    pub country_codes: Vec<String>,
202    /// Tasks covered by its datasets, for example `ASR`.
203    #[serde(default)]
204    pub tasks: Vec<String>,
205}
206
207/// Aggregates computed over a [`Catalogue`](crate::Catalogue).
208///
209/// Hours never include datasets flagged [`Dataset::unverified_size`].
210#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
211pub struct Stats {
212    /// Number of dataset records.
213    pub datasets: usize,
214    /// Sum of countable hours.
215    pub hours: f64,
216    /// Number of records with a self-reported, unverified size.
217    pub unverified_datasets: usize,
218    /// Number of records with a Hugging Face repo.
219    pub streamable_datasets: usize,
220    /// Distinct language tags named across the catalogue.
221    pub languages: usize,
222    /// Distinct ISO 639-3 codes, so every regional variety collapses into its
223    /// code.
224    pub language_codes: usize,
225    /// Distinct countries named across the catalogue.
226    pub countries: usize,
227    /// Record counts by task.
228    pub by_task: BTreeMap<String, usize>,
229    /// Record counts by access mode.
230    pub by_access: BTreeMap<String, usize>,
231    /// Record counts by licence family.
232    pub by_licence_class: BTreeMap<String, usize>,
233    /// Record counts by commercial-use flag.
234    pub by_commercial: BTreeMap<String, usize>,
235    /// Record counts by region.
236    pub by_region: BTreeMap<String, usize>,
237    /// Countable hours by region.
238    pub hours_by_region: BTreeMap<String, f64>,
239}