use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::Arc;
use serde::Deserialize;
use crate::error::{NganoError, Result};
use crate::filter::Filter;
use crate::http::HttpClient;
use crate::language::{LanguageCode, LanguageRegistry};
use crate::model::{Country, Dataset, Language, Stats};
pub const DEFAULT_API_BASE: &str = "https://ngano.dev/api/v1";
const BUNDLED_CATALOGUE: &str = include_str!(concat!(env!("OUT_DIR"), "/catalogue.json"));
const BUNDLED_COUNTRIES: &str = include_str!(concat!(env!("OUT_DIR"), "/countries.json"));
const BUNDLED_LANGUAGES: &str = include_str!(concat!(env!("OUT_DIR"), "/languages.json"));
#[derive(Debug, Deserialize)]
struct Page {
data: Vec<Dataset>,
#[serde(default)]
meta: PageMeta,
}
#[derive(Debug, Default, Deserialize)]
struct PageMeta {
#[serde(default)]
total_pages: Option<u32>,
}
#[derive(Debug, Clone)]
pub struct Catalogue {
inner: Arc<Inner>,
}
#[derive(Debug)]
pub(crate) struct Inner {
datasets: Vec<Dataset>,
countries: Vec<Country>,
languages: Vec<Language>,
by_id: HashMap<String, usize>,
by_repo: HashMap<String, usize>,
country_by_name: HashMap<String, String>,
language_by_tag: HashMap<String, usize>,
}
impl Catalogue {
pub fn bundled() -> Result<Self> {
let datasets: Vec<Dataset> = serde_json::from_str(BUNDLED_CATALOGUE)?;
let countries: Vec<Country> = serde_json::from_str(BUNDLED_COUNTRIES)?;
let languages: Vec<Language> = serde_json::from_str(BUNDLED_LANGUAGES)?;
Ok(Self::from_parts(datasets, countries, languages))
}
pub fn from_parts(
datasets: Vec<Dataset>,
countries: Vec<Country>,
languages: Vec<Language>,
) -> Self {
let by_id = datasets
.iter()
.enumerate()
.map(|(i, d)| (d.id.to_lowercase(), i))
.collect();
let by_repo = datasets
.iter()
.enumerate()
.filter_map(|(i, d)| Some((d.hf_repo.as_ref()?.to_lowercase(), i)))
.collect();
let mut country_by_name = HashMap::with_capacity(countries.len() * 3);
for c in &countries {
country_by_name.insert(c.name.to_lowercase(), c.iso2.clone());
if let Some(map_name) = &c.map_name {
country_by_name.insert(map_name.to_lowercase(), c.iso2.clone());
}
country_by_name.insert(c.iso3.to_lowercase(), c.iso2.clone());
}
let language_by_tag = languages
.iter()
.enumerate()
.map(|(i, l)| (l.tag.to_lowercase(), i))
.collect();
Self {
inner: Arc::new(Inner {
datasets,
countries,
languages,
by_id,
by_repo,
country_by_name,
language_by_tag,
}),
}
}
pub async fn from_api() -> Result<Self> {
Self::from_api_base(DEFAULT_API_BASE).await
}
pub async fn from_api_base(base: &str) -> Result<Self> {
let base = base.trim_end_matches('/');
let http = HttpClient::new("ngano api")?;
let mut datasets = Vec::new();
let mut page = 1u32;
loop {
let url = format!("{base}/datasets?page={page}&per_page=200");
let body: Page = http.get_json(&url, None).await?;
let empty = body.data.is_empty();
datasets.extend(body.data);
let total_pages = body.meta.total_pages.unwrap_or(page);
if empty || page >= total_pages {
break;
}
page += 1;
}
let countries: Vec<Country> = http.get_json(&format!("{base}/countries"), None).await?;
let languages: Vec<Language> = http.get_json(&format!("{base}/languages"), None).await?;
Ok(Self::from_parts(datasets, countries, languages))
}
#[cfg(feature = "blocking")]
#[cfg_attr(docsrs, doc(cfg(feature = "blocking")))]
pub fn from_api_blocking() -> Result<Self> {
crate::blocking::block_on(Self::from_api())
}
#[cfg(feature = "blocking")]
#[cfg_attr(docsrs, doc(cfg(feature = "blocking")))]
pub fn from_api_base_blocking(base: &str) -> Result<Self> {
crate::blocking::block_on(Self::from_api_base(base))
}
pub fn all(&self) -> &[Dataset] {
&self.inner.datasets
}
pub fn len(&self) -> usize {
self.inner.datasets.len()
}
pub fn is_empty(&self) -> bool {
self.inner.datasets.is_empty()
}
pub fn countries(&self) -> &[Country] {
&self.inner.countries
}
pub fn languages(&self) -> &[Language] {
&self.inner.languages
}
pub fn language_codes(&self) -> &[LanguageCode] {
self.registry().codes()
}
pub fn registry(&self) -> &'static LanguageRegistry {
LanguageRegistry::shared()
}
pub fn resolve_language(&self, value: &str) -> Vec<String> {
self.registry().resolve(value, false)
}
pub fn language(&self, value: &str) -> Option<&Language> {
self.resolve_language(value).into_iter().find_map(|tag| {
self.inner
.language_by_tag
.get(&tag.to_lowercase())
.map(|i| &self.inner.languages[*i])
})
}
pub fn datasets(&self, filter: &Filter) -> Vec<&Dataset> {
self.inner
.datasets
.iter()
.filter(|d| filter.matches(d))
.collect()
}
pub fn get(&self, id: &str) -> Option<&Dataset> {
self.inner
.by_id
.get(&id.to_lowercase())
.map(|i| &self.inner.datasets[*i])
}
pub fn by_hf_repo(&self, repo: &str) -> Option<&Dataset> {
self.inner
.by_repo
.get(&repo.trim().to_lowercase())
.map(|i| &self.inner.datasets[*i])
}
pub fn require(&self, id: &str) -> Result<&Dataset> {
self.get(id)
.ok_or_else(|| NganoError::NotFound(format!("dataset {id}")))
}
pub fn search(&self, query: &str) -> Vec<&Dataset> {
self.datasets(&Filter::new().q(query))
}
pub fn stats(&self) -> Stats {
self.stats_for(self.all())
}
pub fn stats_for<'a, I>(&self, datasets: I) -> Stats
where
I: IntoIterator<Item = &'a Dataset>,
{
let mut s = Stats {
datasets: 0,
hours: 0.0,
unverified_datasets: 0,
streamable_datasets: 0,
languages: 0,
language_codes: 0,
countries: 0,
by_task: BTreeMap::new(),
by_access: BTreeMap::new(),
by_licence_class: BTreeMap::new(),
by_commercial: BTreeMap::new(),
by_region: BTreeMap::new(),
hours_by_region: BTreeMap::new(),
};
let mut languages = BTreeSet::new();
let mut language_codes = BTreeSet::new();
let mut countries = BTreeSet::new();
for d in datasets {
s.datasets += 1;
if d.unverified_size {
s.unverified_datasets += 1;
}
if d.is_streamable() {
s.streamable_datasets += 1;
}
let hours = d.countable_hours().unwrap_or(0.0);
s.hours += hours;
for tag in &d.language_tags {
languages.insert(tag.clone());
}
for code in &d.language_codes {
language_codes.insert(code.clone());
}
for c in &d.country_codes {
countries.insert(c.clone());
}
*s.by_task.entry(d.task.clone()).or_default() += 1;
*s.by_access.entry(d.access.clone()).or_default() += 1;
*s.by_licence_class
.entry(d.licence_class.clone())
.or_default() += 1;
*s.by_commercial.entry(d.commercial.clone()).or_default() += 1;
for r in &d.regions {
*s.by_region.entry(r.clone()).or_default() += 1;
*s.hours_by_region.entry(r.clone()).or_default() += hours;
}
}
s.languages = languages.len();
s.language_codes = language_codes.len();
s.countries = countries.len();
s
}
pub fn country_code(&self, value: &str) -> Option<String> {
let trimmed = value.trim();
if trimmed.len() == 2 {
let upper = trimmed.to_uppercase();
if self.inner.countries.iter().any(|c| c.iso2 == upper) {
return Some(upper);
}
}
self.inner
.country_by_name
.get(&trimmed.to_lowercase())
.cloned()
}
pub(crate) fn country_index(&self) -> Arc<Inner> {
Arc::clone(&self.inner)
}
}
impl Inner {
pub(crate) fn resolve_country(&self, value: &str) -> Option<String> {
let trimmed = value.trim();
if trimmed.is_empty() {
return None;
}
if trimmed.len() == 2 {
let upper = trimmed.to_uppercase();
if self.countries.iter().any(|c| c.iso2 == upper) {
return Some(upper);
}
}
self.country_by_name.get(&trimmed.to_lowercase()).cloned()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn bundled_catalogue_parses() {
let cat = Catalogue::bundled().expect("bundled catalogue parses");
assert!(
cat.len() >= 600,
"expected the full catalogue, got {}",
cat.len()
);
assert_eq!(cat.countries().len(), 58);
assert!(cat.languages().len() >= 300);
assert!(cat
.languages()
.iter()
.all(|l| l.slug == l.tag.to_lowercase()));
}
#[test]
fn languages_resolve_through_the_registry() {
let cat = Catalogue::bundled().unwrap();
assert_eq!(cat.resolve_language("isiZulu"), vec!["zul".to_string()]);
assert_eq!(cat.resolve_language("eng-ng"), vec!["eng-NG".to_string()]);
assert!(cat.resolve_language("").is_empty());
assert_eq!(cat.language("sna").map(|l| l.name.as_str()), Some("Shona"));
assert!(cat.language_codes().len() >= 300);
}
#[test]
fn unverified_hours_are_excluded_from_totals() {
let cat = Catalogue::bundled().unwrap();
let naive: f64 = cat.all().iter().filter_map(|d| d.hours_num).sum();
let counted = cat.stats().hours;
assert!(counted < naive, "unverified hours must be excluded");
for d in cat.all().iter().filter(|d| d.unverified_size) {
assert_eq!(d.countable_hours(), None);
}
}
}