1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
//! Dataset selection.
use crate::language::LanguageRegistry;
use crate::model::Dataset;
/// A declarative dataset filter.
///
/// Values given to the same method are combined with OR, and different methods
/// are combined with AND. Every string comparison is case-insensitive.
///
/// ```
/// use ngano::{Catalogue, Filter};
///
/// let cat = Catalogue::bundled()?;
/// let f = Filter::new().language("sna").commercial(true).task("ASR");
/// for d in cat.datasets(&f) {
/// assert!(d.allows_commercial_use());
/// }
/// # Ok::<(), ngano::NganoError>(())
/// ```
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Filter {
/// Free-text query applied to name, id, languages, countries and notes.
pub q: Option<String>,
/// Languages to keep, each written as a bare ISO 639-3 code, a BCP 47 tag
/// or a name. Every value is resolved to canonical tags and matched against
/// [`Dataset::language_tags`].
pub languages: Vec<String>,
/// Widen a bare ISO 639-3 code to every regional variety of it, so `eng`
/// also matches `eng-NG` and `eng-ZA`.
pub include_varieties: bool,
/// Countries, given as ISO 3166-1 alpha-2 codes or as names.
pub countries: Vec<String>,
/// Regional groupings.
pub regions: Vec<String>,
/// Tasks, for example `ASR`.
pub tasks: Vec<String>,
/// Language varieties.
pub varieties: Vec<String>,
/// Licence families.
pub licence_classes: Vec<String>,
/// Access modes.
pub access: Vec<String>,
/// Labelling states.
pub labelled: Vec<String>,
/// Quality bands.
pub quality: Vec<String>,
/// Recording domains.
pub domains: Vec<String>,
/// Hosts, for example `HuggingFace`.
pub hosts: Vec<String>,
/// `Some(true)` keeps datasets usable commercially (`Yes` or
/// `Yes, if purchased`), `Some(false)` keeps only those that are not.
pub commercial: Option<bool>,
/// `Some(true)` keeps only datasets with a Hugging Face repo.
pub hf_only: Option<bool>,
/// `Some(true)` keeps only datasets with a numeric hours figure.
pub has_hours: Option<bool>,
/// Lower bound on countable hours, inclusive.
pub min_hours: Option<f64>,
/// Upper bound on countable hours, inclusive.
pub max_hours: Option<f64>,
/// When true, drop records whose size is self-reported and unverified.
pub exclude_unverified: bool,
}
macro_rules! push_setter {
($(#[$m:meta])* $name:ident, $field:ident) => {
$(#[$m])*
#[must_use]
pub fn $name(mut self, value: impl Into<String>) -> Self {
self.$field.push(value.into());
self
}
};
}
impl Filter {
/// An empty filter, which matches every record.
#[must_use]
pub fn new() -> Self {
Self::default()
}
/// Free-text query over name, id, languages, countries, domain and notes.
#[must_use]
pub fn q(mut self, value: impl Into<String>) -> Self {
self.q = Some(value.into());
self
}
push_setter!(
/// Add a language to match, written as a bare ISO 639-3 code (`sna`),
/// a BCP 47 tag (`eng-NG`) or a name (`Shona`, `isiZulu`). Case does
/// not matter, and an unresolvable value simply matches nothing.
language, languages);
push_setter!(
/// Add a language by its ISO 639-3 code. This is the code-shaped
/// spelling of [`Filter::language`] and resolves identically.
iso, languages);
push_setter!(
/// Add a country, as an ISO 3166-1 alpha-2 code or a name.
country, countries);
push_setter!(
/// Add a regional grouping to match.
region, regions);
push_setter!(
/// Add a task to match, for example `ASR`.
task, tasks);
push_setter!(
/// Add a language variety to match.
variety, varieties);
push_setter!(
/// Add a licence family to match.
licence_class, licence_classes);
push_setter!(
/// Add an access mode to match, for example `Open`.
access_mode, access);
push_setter!(
/// Add a labelling state to match, for example `Transcribed`.
labelled, labelled);
push_setter!(
/// Add a quality band to match.
quality, quality);
push_setter!(
/// Add a recording domain to match.
domain, domains);
push_setter!(
/// Add a host to match, for example `HuggingFace`.
host, hosts);
/// Keep datasets by commercial usability. `true` keeps `Yes` and
/// `Yes, if purchased`, `false` keeps everything else.
#[must_use]
pub fn commercial(mut self, value: bool) -> Self {
self.commercial = Some(value);
self
}
/// Widen every bare ISO 639-3 code given to [`Filter::language`] to its
/// regional varieties, so `eng` also keeps `eng-NG` and `eng-ZA`. Off by
/// default: a bare code means the language at large and must never silently
/// pick up a country-specific variety.
///
/// ```
/// use ngano::{Catalogue, Filter};
///
/// let cat = Catalogue::bundled()?;
/// let plain = cat.datasets(&Filter::new().language("eng"));
/// let wide = cat.datasets(&Filter::new().language("eng").include_varieties(true));
/// assert!(wide.len() > plain.len());
/// # Ok::<(), ngano::NganoError>(())
/// ```
#[must_use]
pub fn include_varieties(mut self, value: bool) -> Self {
self.include_varieties = value;
self
}
/// Keep only datasets hosted on Hugging Face, which are the streamable ones.
#[must_use]
pub fn hf_only(mut self, value: bool) -> Self {
self.hf_only = Some(value);
self
}
/// Keep only datasets that publish a numeric hours figure.
#[must_use]
pub fn has_hours(mut self, value: bool) -> Self {
self.has_hours = Some(value);
self
}
/// Lower bound on countable hours, inclusive.
#[must_use]
pub fn min_hours(mut self, value: f64) -> Self {
self.min_hours = Some(value);
self
}
/// Upper bound on countable hours, inclusive.
#[must_use]
pub fn max_hours(mut self, value: f64) -> Self {
self.max_hours = Some(value);
self
}
/// Drop records whose published size is self-reported and unverified.
#[must_use]
pub fn exclude_unverified(mut self, value: bool) -> Self {
self.exclude_unverified = value;
self
}
/// The canonical BCP 47 tags this filter keeps, deduplicated and in the
/// order the languages were added. A value that resolves to nothing
/// contributes nothing, so a filter written only of unknown languages keeps
/// no records at all.
///
/// ```
/// use ngano::Filter;
///
/// let f = Filter::new().language("isiZulu").iso("sna");
/// assert_eq!(f.resolved_languages(), vec!["zul".to_string(), "sna".to_string()]);
/// ```
pub fn resolved_languages(&self) -> Vec<String> {
LanguageRegistry::shared().resolve_all(&self.languages, self.include_varieties)
}
/// Languages given to this filter that the registry does not recognise.
/// Useful for telling a caller that a spelling matched nothing, rather than
/// silently returning an empty result.
pub fn unresolved_languages(&self) -> Vec<String> {
let registry = LanguageRegistry::shared();
self.languages
.iter()
.filter(|v| registry.resolve(v, self.include_varieties).is_empty())
.cloned()
.collect()
}
/// True when this filter would keep every record.
pub fn is_empty(&self) -> bool {
*self == Filter::default()
}
/// Test one record against the filter.
///
/// ```
/// use ngano::{Catalogue, Filter};
///
/// let cat = Catalogue::bundled()?;
/// let d = cat.all().first().expect("catalogue is not empty");
/// assert!(Filter::new().matches(d));
/// # Ok::<(), ngano::NganoError>(())
/// ```
pub fn matches(&self, d: &Dataset) -> bool {
if self.exclude_unverified && d.unverified_size {
return false;
}
if let Some(want) = self.commercial {
if d.allows_commercial_use() != want {
return false;
}
}
if self.hf_only == Some(true) && !d.is_streamable() {
return false;
}
if self.hf_only == Some(false) && d.is_streamable() {
return false;
}
if let Some(want) = self.has_hours {
if d.hours_num.is_some() != want {
return false;
}
}
if self.min_hours.is_some() || self.max_hours.is_some() {
let Some(h) = d.countable_hours() else {
return false;
};
if self.min_hours.is_some_and(|m| h < m) || self.max_hours.is_some_and(|m| h > m) {
return false;
}
}
if !any_match(&self.tasks, std::slice::from_ref(&d.task))
|| !any_match(&self.varieties, std::slice::from_ref(&d.variety))
|| !any_match(
&self.licence_classes,
std::slice::from_ref(&d.licence_class),
)
|| !any_match(&self.access, std::slice::from_ref(&d.access))
|| !any_match(&self.labelled, std::slice::from_ref(&d.labelled))
|| !any_match(&self.quality, std::slice::from_ref(&d.quality))
|| !any_match(&self.domains, std::slice::from_ref(&d.domain))
|| !any_match(&self.hosts, std::slice::from_ref(&d.host))
|| !any_match(&self.regions, &d.regions)
{
return false;
}
if !self.languages.is_empty() {
let wanted = self.resolved_languages();
if !d
.language_tags
.iter()
.any(|tag| wanted.iter().any(|w| w.eq_ignore_ascii_case(tag)))
{
return false;
}
}
if !any_match_either(&self.countries, &d.country_codes, &d.countries) {
return false;
}
if let Some(q) = &self.q {
if !matches_query(d, q) {
return false;
}
}
true
}
}
/// True when `wanted` is empty, or when any wanted value equals any candidate,
/// ignoring case.
fn any_match(wanted: &[String], candidates: &[String]) -> bool {
if wanted.is_empty() {
return true;
}
wanted.iter().any(|w| {
candidates
.iter()
.any(|c| c.eq_ignore_ascii_case(w.trim()) || c.trim().eq_ignore_ascii_case(w.trim()))
})
}
/// True when `wanted` is empty, or when it matches either candidate list.
fn any_match_either(wanted: &[String], first: &[String], second: &[String]) -> bool {
if wanted.is_empty() {
return true;
}
any_match(wanted, first) || any_match(wanted, second)
}
/// Case-insensitive substring search across the fields a human would search.
fn matches_query(d: &Dataset, q: &str) -> bool {
let needle = q.trim().to_lowercase();
if needle.is_empty() {
return true;
}
let mut haystack = String::with_capacity(256);
haystack.push_str(&d.name);
haystack.push('\n');
haystack.push_str(&d.id);
haystack.push('\n');
haystack.push_str(&d.domain);
haystack.push('\n');
haystack.push_str(&d.notes);
haystack.push('\n');
if let Some(r) = &d.hf_repo {
haystack.push_str(r);
haystack.push('\n');
}
for list in [
&d.languages,
&d.languages_clean,
&d.language_tags,
&d.countries,
&d.regions,
] {
for v in list {
haystack.push_str(v);
haystack.push('\n');
}
}
haystack.to_lowercase().contains(&needle)
}