ngano/model.rs
1//! Catalogue record types: datasets, countries, languages and aggregates.
2
3use serde::{Deserialize, Serialize};
4use std::collections::BTreeMap;
5
6/// One catalogue record: a published dataset, as described by its source.
7///
8/// Figures are reported as the source publishes them. [`Dataset::unverified_size`]
9/// marks self-reported totals of 20,000 hours or more, which are excluded from
10/// every hours aggregate in this crate.
11#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
12pub struct Dataset {
13 /// Stable ngano identifier, for example `waxal-corpus-paper`.
14 pub id: String,
15 /// Human readable dataset name.
16 pub name: String,
17 /// `ASR`, `TTS`, `ASR+TTS`, `Raw source` or `Other`.
18 pub task: String,
19 /// Language variety grouping, for example `Indigenous`.
20 pub variety: String,
21 /// Languages exactly as the source lists them.
22 #[serde(default)]
23 pub languages: Vec<String>,
24 /// Languages after normalisation against `data/languages.json`, in the
25 /// same order as [`Dataset::language_tags`].
26 #[serde(default)]
27 pub languages_clean: Vec<String>,
28 /// ISO 639-3 codes as the record was first catalogued. Prefer
29 /// [`Dataset::language_codes`], which is derived from the tags.
30 #[serde(default)]
31 pub iso: Vec<String>,
32 /// Canonical BCP 47 language tags, ordered, one per language the record
33 /// covers. The primary subtag is an ISO 639-3 code and a region subtag
34 /// marks a country-specific variety, so `eng-NG` is Nigerian English.
35 #[serde(default)]
36 pub language_tags: Vec<String>,
37 /// The bare ISO 639-3 codes behind [`Dataset::language_tags`],
38 /// deduplicated, so every regional variety collapses into its code.
39 #[serde(default)]
40 pub language_codes: Vec<String>,
41 /// An editorial note about how this record names its languages, present on
42 /// the handful of sources that describe their coverage in prose rather than
43 /// naming languages one by one.
44 #[serde(default)]
45 pub language_note: Option<String>,
46 /// Country names as the source lists them.
47 #[serde(default)]
48 pub countries: Vec<String>,
49 /// ISO 3166-1 alpha-2 codes for [`Dataset::countries`].
50 #[serde(default)]
51 pub country_codes: Vec<String>,
52 /// Regional groupings, for example `East Africa`.
53 #[serde(default)]
54 pub regions: Vec<String>,
55 /// Hours as published, verbatim, including any qualifying text.
56 #[serde(default)]
57 pub hours: Option<String>,
58 /// Numeric hours parsed from [`Dataset::hours`], when one could be parsed.
59 #[serde(default)]
60 pub hours_num: Option<f64>,
61 /// Speaker count as published.
62 #[serde(default)]
63 pub speakers: Option<String>,
64 /// Recording setup, for example `telephone` or `studio`.
65 #[serde(default)]
66 pub recording_type: Option<String>,
67 /// Audio quality band, for example `Standard (16 kHz)`.
68 pub quality: String,
69 /// `Transcribed`, `Unlabelled` or `Unstated`.
70 pub labelled: String,
71 /// Recording domain, for example `Broadcast news`.
72 pub domain: String,
73 /// Licence string as published.
74 pub licence: String,
75 /// Licence family, for example `Attribution (CC-BY)`.
76 pub licence_class: String,
77 /// `Yes`, `Yes, if purchased`, `No` or `Unstated`.
78 pub commercial: String,
79 /// `Open`, `Request`, `Paid`, `Scrape required` or `Unclear`.
80 pub access: String,
81 /// Where the data lives, for example `HuggingFace`.
82 pub host: String,
83 /// Canonical URL for the dataset.
84 #[serde(default)]
85 pub url: Option<String>,
86 /// Hugging Face repo id, when the dataset is hosted there.
87 #[serde(default)]
88 pub hf_repo: Option<String>,
89 /// Publication year as published.
90 #[serde(default)]
91 pub year: Option<String>,
92 /// Editorial notes about scope, caveats and provenance.
93 #[serde(default)]
94 pub notes: String,
95 /// True when the published size is self-reported and 20,000 hours or more.
96 /// Such figures are excluded from every total.
97 #[serde(default)]
98 pub unverified_size: bool,
99}
100
101impl Dataset {
102 /// Hours that may be counted in a total: `None` when the figure is missing
103 /// or flagged [`Dataset::unverified_size`].
104 ///
105 /// ```
106 /// use ngano::Catalogue;
107 ///
108 /// let cat = Catalogue::bundled()?;
109 /// let counted: f64 = cat.all().iter().filter_map(|d| d.countable_hours()).sum();
110 /// assert!(counted > 0.0);
111 /// # Ok::<(), ngano::NganoError>(())
112 /// ```
113 pub fn countable_hours(&self) -> Option<f64> {
114 if self.unverified_size {
115 None
116 } else {
117 self.hours_num
118 }
119 }
120
121 /// True when the dataset can be streamed, meaning it has a Hugging Face repo.
122 pub fn is_streamable(&self) -> bool {
123 self.hf_repo.as_deref().is_some_and(|r| !r.is_empty())
124 }
125
126 /// True when the licence permits commercial use outright or after purchase.
127 pub fn allows_commercial_use(&self) -> bool {
128 let c = self.commercial.to_ascii_lowercase();
129 c.starts_with("yes")
130 }
131}
132
133/// A country in the catalogue's coverage map.
134#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
135pub struct Country {
136 /// Canonical country name.
137 pub name: String,
138 /// ISO 3166-1 alpha-2 code.
139 pub iso2: String,
140 /// ISO 3166-1 alpha-3 code.
141 pub iso3: String,
142 /// Name used by the bundled GeoJSON polygons. `None` for island states,
143 /// which have no polygon and are drawn from `lat` and `lon` instead.
144 #[serde(default)]
145 pub map_name: Option<String>,
146 /// Centroid latitude.
147 pub lat: f64,
148 /// Centroid longitude.
149 pub lon: f64,
150 /// Regional grouping, for example `Southern Africa`.
151 pub region: String,
152 /// URL slug, normally the lowercased alpha-2 code.
153 pub slug: String,
154 /// Dataset count, when the record came from the HTTP API.
155 #[serde(default)]
156 pub datasets: Option<u32>,
157 /// Counted hours, when the record came from the HTTP API.
158 #[serde(default)]
159 pub hours: Option<f64>,
160}
161
162/// A language in the catalogue's coverage map, keyed on its BCP 47 tag.
163///
164/// The registry entry behind the tag is [`LanguageCode`](crate::LanguageCode);
165/// this record adds what the catalogue knows about the language's coverage.
166#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
167pub struct Language {
168 /// Canonical BCP 47 tag, for example `sna` or `eng-NG`.
169 pub tag: String,
170 /// The bare ISO 639-3 code behind the tag.
171 pub iso639_3: String,
172 /// ISO 3166-1 alpha-2 region subtag, when the tag names a country-specific
173 /// variety.
174 #[serde(default)]
175 pub region: Option<String>,
176 /// Canonical language name.
177 pub name: String,
178 /// ISO 639-3 scope: `I` individual, `M` macrolanguage, `S` special.
179 #[serde(default)]
180 pub scope: Option<String>,
181 /// ISO 639-3 type: `L` living, `E` extinct, and so on. Named `kind`
182 /// because `type` is a Rust keyword.
183 #[serde(default, rename = "type")]
184 pub kind: Option<String>,
185 /// Every catalogue spelling that resolves to this tag.
186 #[serde(default)]
187 pub aliases: Vec<String>,
188 /// URL slug, which is the tag lowercased.
189 pub slug: String,
190 /// Number of datasets covering it.
191 #[serde(default)]
192 pub datasets: Option<u32>,
193 /// Counted hours across its datasets.
194 #[serde(default)]
195 pub hours: Option<f64>,
196 /// Countries where the catalogue records it, by name.
197 #[serde(default)]
198 pub countries: Vec<String>,
199 /// ISO 3166-1 alpha-2 codes for [`Language::countries`].
200 #[serde(default)]
201 pub country_codes: Vec<String>,
202 /// Tasks covered by its datasets, for example `ASR`.
203 #[serde(default)]
204 pub tasks: Vec<String>,
205}
206
207/// Aggregates computed over a [`Catalogue`](crate::Catalogue).
208///
209/// Hours never include datasets flagged [`Dataset::unverified_size`].
210#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
211pub struct Stats {
212 /// Number of dataset records.
213 pub datasets: usize,
214 /// Sum of countable hours.
215 pub hours: f64,
216 /// Number of records with a self-reported, unverified size.
217 pub unverified_datasets: usize,
218 /// Number of records with a Hugging Face repo.
219 pub streamable_datasets: usize,
220 /// Distinct language tags named across the catalogue.
221 pub languages: usize,
222 /// Distinct ISO 639-3 codes, so every regional variety collapses into its
223 /// code.
224 pub language_codes: usize,
225 /// Distinct countries named across the catalogue.
226 pub countries: usize,
227 /// Record counts by task.
228 pub by_task: BTreeMap<String, usize>,
229 /// Record counts by access mode.
230 pub by_access: BTreeMap<String, usize>,
231 /// Record counts by licence family.
232 pub by_licence_class: BTreeMap<String, usize>,
233 /// Record counts by commercial-use flag.
234 pub by_commercial: BTreeMap<String, usize>,
235 /// Record counts by region.
236 pub by_region: BTreeMap<String, usize>,
237 /// Countable hours by region.
238 pub hours_by_region: BTreeMap<String, f64>,
239}