1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
//! `ngano` is an open catalogue and streaming loader for African-language
//! speech datasets.
//!
//! It ships a snapshot of the catalogue inside the binary, so browsing works
//! with no network at all, and it streams rows from any catalogued dataset that
//! is hosted on Hugging Face, mapping heterogeneous columns onto one canonical
//! schema shared with the Python and JavaScript SDKs.
//!
//! # Browse the catalogue, offline
//!
//! ```
//! use ngano::{Catalogue, Filter};
//!
//! let cat = Catalogue::bundled()?;
//! let hits = cat.datasets(&Filter::new().language("sna").commercial(true).task("ASR"));
//! for d in &hits {
//! println!("{} ({})", d.name, d.licence);
//! }
//!
//! let stats = cat.stats();
//! println!("{} datasets, {:.0} counted hours", stats.datasets, stats.hours);
//! # Ok::<(), ngano::NganoError>(())
//! ```
//!
//! # Languages
//!
//! A language is a BCP 47 tag whose primary subtag is an ISO 639-3 code, so
//! Shona is `sna`. A region subtag marks a country-specific variety, so
//! Nigerian English is `eng-NG` and Mozambican Portuguese is `por-MZ`.
//! [`Filter::language`] accepts a bare code, a tag or a name, in any case, and
//! [`Catalogue::resolve_language`] shows what a given spelling resolves to. A
//! bare code never picks up a regional variety unless you ask for it with
//! [`Filter::include_varieties`].
//!
//! ```
//! use ngano::{Catalogue, Filter};
//!
//! let cat = Catalogue::bundled()?;
//! assert_eq!(cat.resolve_language("isiZulu"), vec!["zul".to_string()]);
//! let nigerian_english = cat.datasets(&Filter::new().language("eng-NG"));
//! assert!(!nigerian_english.is_empty());
//! # Ok::<(), ngano::NganoError>(())
//! ```
//!
//! # Stream rows
//!
//! ```no_run
//! use futures::TryStreamExt;
//! use ngano::{Filter, Interleave, Loader};
//!
//! # async fn run() -> Result<(), ngano::NganoError> {
//! let mut stream = Loader::new()
//! .filter(Filter::new().language("sna").country("ZW").commercial(true))
//! .split("train")
//! .hf_token(std::env::var("HF_TOKEN").ok())
//! .interleave(Interleave::RoundRobin)
//! .limit(Some(1000))
//! .stream()
//! .await?;
//!
//! while let Some(row) = stream.try_next().await? {
//! println!("{:?}", row.transcript.as_deref());
//! if let Some(audio) = &row.audio {
//! let bytes = audio.read().await?; // nothing was fetched before this
//! println!("{} bytes", bytes.len());
//! }
//! }
//! # Ok(()) }
//! ```
//!
//! # One repo, by name
//!
//! [`Loader::repo`] streams a Hugging Face repo whether or not the catalogue
//! lists it, which is the Rust spelling of `load_dataset("google/fleurs",
//! config="sw_ke")` in the Python SDK.
//!
//! ```no_run
//! # async fn run() -> Result<(), ngano::NganoError> {
//! let stream = ngano::Loader::new()
//! .repo("google/fleurs")
//! .config("sw_ke")
//! .limit(Some(100))
//! .stream()
//! .await?;
//! # let _ = stream;
//! # Ok(()) }
//! ```
//!
//! # Hours figures
//!
//! Some sources self-report totals of 20,000 hours or more. Those records carry
//! [`Dataset::unverified_size`] and are excluded from every hours aggregate this
//! crate computes.
//!
//! # Runtime, features and MSRV
//!
//! The async API expects a Tokio runtime, which is what `reqwest` uses for its
//! own timers. Default features give `rustls` TLS; `native-tls` swaps it,
//! `blocking` adds synchronous wrappers, and `cli` builds the `ngano` binary.
//! The minimum supported Rust version is 1.82.
pub use ;
pub use ;
pub use ;
pub use Filter;
pub use ;
pub use ;
pub use ;
pub use ;
pub use ;
/// The version of this crate.
pub const VERSION: &str = env!;
/// The README examples, compiled and run as doctests.
///
/// The README shows the `blocking` API, so it is checked under
/// `cargo test --features blocking`.
;