use std::collections::BTreeMap;
use std::sync::Arc;
use bytes::Bytes;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::catalogue::Inner as CatalogueInner;
use crate::error::{NganoError, Result};
use crate::http::HttpClient;
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct AudioRef {
pub url: Option<String>,
pub path: Option<String>,
pub sampling_rate: Option<u32>,
#[serde(skip)]
bytes: Option<Bytes>,
#[serde(skip)]
fetcher: Option<HttpClient>,
#[serde(skip)]
repo: Option<String>,
}
impl PartialEq for AudioRef {
fn eq(&self, other: &Self) -> bool {
self.url == other.url
&& self.path == other.path
&& self.sampling_rate == other.sampling_rate
&& self.bytes == other.bytes
}
}
impl AudioRef {
pub fn from_url(url: impl Into<String>) -> Self {
Self {
url: Some(url.into()),
..Self::default()
}
}
pub fn from_bytes(bytes: impl Into<Bytes>) -> Self {
Self {
bytes: Some(bytes.into()),
..Self::default()
}
}
#[must_use]
pub fn with_sampling_rate(mut self, hz: u32) -> Self {
self.sampling_rate = Some(hz);
self
}
pub fn bytes(&self) -> Option<&Bytes> {
self.bytes.as_ref()
}
pub fn needs_fetch(&self) -> bool {
self.bytes.is_none() && self.url.is_some()
}
pub async fn read(&self) -> Result<Bytes> {
if let Some(b) = &self.bytes {
return Ok(b.clone());
}
let Some(url) = &self.url else {
return Err(NganoError::AudioUnavailable(
self.path.clone().unwrap_or_else(|| "<unknown>".to_string()),
));
};
let client = match &self.fetcher {
Some(c) => c.clone(),
None => HttpClient::new("huggingface")?,
};
client.get_bytes(url, self.repo.as_deref()).await
}
#[cfg(feature = "blocking")]
#[cfg_attr(docsrs, doc(cfg(feature = "blocking")))]
pub fn read_blocking(&self) -> Result<Bytes> {
crate::blocking::block_on(self.read())
}
pub(crate) fn with_fetcher(mut self, client: HttpClient, repo: &str) -> Self {
self.fetcher = Some(client);
self.repo = Some(repo.to_string());
self
}
pub(crate) fn from_value(value: &Value) -> Option<Self> {
match value {
Value::Null => None,
Value::String(s) if s.is_empty() => None,
Value::String(s) => Some(if is_url(s) {
Self::from_url(s.clone())
} else {
Self {
path: Some(s.clone()),
..Self::default()
}
}),
Value::Array(items) => items.iter().find_map(Self::from_value),
Value::Object(map) => {
let mut out = Self::default();
for key in ["src", "url", "audio_url"] {
if let Some(s) = map.get(key).and_then(Value::as_str) {
out.url = Some(s.to_string());
break;
}
}
if let Some(s) = map.get("path").and_then(Value::as_str) {
if out.url.is_none() && is_url(s) {
out.url = Some(s.to_string());
} else {
out.path = Some(s.to_string());
}
}
if let Some(b) = map.get("bytes") {
if let Some(s) = b.as_str() {
out.bytes = Some(Bytes::from(s.as_bytes().to_vec()));
} else if let Some(arr) = b.as_array() {
let raw: Vec<u8> = arr
.iter()
.filter_map(|v| v.as_u64())
.map(|v| v as u8)
.collect();
if !raw.is_empty() {
out.bytes = Some(Bytes::from(raw));
}
}
}
out.sampling_rate = map
.get("sampling_rate")
.or_else(|| map.get("sample_rate"))
.and_then(as_u32);
if out.url.is_none() && out.path.is_none() && out.bytes.is_none() {
None
} else {
Some(out)
}
}
_ => None,
}
}
}
fn is_url(s: &str) -> bool {
s.starts_with("http://") || s.starts_with("https://")
}
pub(crate) fn as_u32(v: &Value) -> Option<u32> {
match v {
Value::Number(n) => n
.as_u64()
.map(|x| x as u32)
.or_else(|| n.as_f64().map(|x| x.round() as u32)),
Value::String(s) => s.trim().parse::<f64>().ok().map(|x| x.round() as u32),
_ => None,
}
}
pub(crate) fn as_f64(v: &Value) -> Option<f64> {
match v {
Value::Number(n) => n.as_f64(),
Value::String(s) => s.trim().parse::<f64>().ok(),
_ => None,
}
}
pub(crate) fn as_text(v: &Value) -> Option<String> {
match v {
Value::String(s) if s.trim().is_empty() => None,
Value::String(s) => Some(s.clone()),
Value::Number(n) => Some(n.to_string()),
Value::Bool(b) => Some(b.to_string()),
_ => None,
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Row {
pub audio: Option<AudioRef>,
pub transcript: Option<String>,
pub language: Option<String>,
pub language_iso: Option<String>,
pub language_tag: Option<String>,
pub country: Option<String>,
pub speaker_id: Option<String>,
pub gender: Option<String>,
pub age: Option<String>,
pub duration_s: Option<f64>,
pub sampling_rate: Option<u32>,
pub domain: Option<String>,
pub split: String,
pub dataset_id: Option<String>,
pub hf_repo: String,
pub licence: String,
pub source_url: String,
pub extra: BTreeMap<String, Value>,
}
#[derive(Debug, Clone)]
pub(crate) struct RowContext {
pub dataset_id: Option<String>,
pub hf_repo: String,
pub licence: String,
pub source_url: String,
pub split: String,
pub default_language: Option<String>,
pub default_language_iso: Option<String>,
pub default_language_tag: Option<String>,
pub default_country: Option<String>,
pub default_domain: Option<String>,
pub catalogue: Option<Arc<CatalogueInner>>,
pub fetcher: Option<HttpClient>,
}
impl RowContext {
pub(crate) fn blank_row(&self) -> Row {
Row {
audio: None,
transcript: None,
language: None,
language_iso: None,
language_tag: None,
country: None,
speaker_id: None,
gender: None,
age: None,
duration_s: None,
sampling_rate: None,
domain: None,
split: self.split.clone(),
dataset_id: self.dataset_id.clone(),
hf_repo: self.hf_repo.clone(),
licence: self.licence.clone(),
source_url: self.source_url.clone(),
extra: BTreeMap::new(),
}
}
pub(crate) fn fill_defaults(&self, row: &mut Row) {
if row.language.is_none() {
row.language.clone_from(&self.default_language);
}
self.settle_language(row);
if row.country.is_none() {
row.country.clone_from(&self.default_country);
}
if row.domain.is_none() {
row.domain.clone_from(&self.default_domain);
}
if let (Some(audio), None) = (&row.audio, row.sampling_rate) {
row.sampling_rate = audio.sampling_rate;
}
}
fn settle_language(&self, row: &mut Row) {
let registry = crate::language::LanguageRegistry::shared();
let resolve_one = |value: &str| -> Option<String> {
match registry.resolve(value, false).as_slice() {
[one] => Some(one.clone()),
_ => None,
}
};
let tag = match row.language_tag.take() {
Some(stated) => resolve_one(&stated).or(Some(stated)),
None => row
.language_iso
.as_deref()
.and_then(resolve_one)
.or_else(|| row.language.as_deref().and_then(resolve_one))
.or_else(|| self.default_language_tag.clone()),
};
row.language_tag.clone_from(&tag);
if let Some(code) = tag
.as_deref()
.and_then(|t| registry.get(t))
.map(|c| c.iso639_3.clone())
{
row.language_iso = Some(code);
} else if row.language_iso.is_none() {
row.language_iso.clone_from(&self.default_language_iso);
}
}
pub(crate) fn normalise_country(&self, raw: &str) -> String {
self.catalogue
.as_ref()
.and_then(|c| c.resolve_country(raw))
.unwrap_or_else(|| raw.trim().to_string())
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn audio_understands_the_datasets_server_shape() {
let a = AudioRef::from_value(&json!({"src": "https://x/y.wav", "type": "audio/wav"}))
.expect("handle");
assert_eq!(a.url.as_deref(), Some("https://x/y.wav"));
assert!(a.needs_fetch());
}
#[test]
fn audio_understands_path_bytes_and_rate() {
let a = AudioRef::from_value(&json!({
"path": "clips/one.mp3",
"bytes": [1, 2, 3],
"sampling_rate": 48000
}))
.expect("handle");
assert_eq!(a.path.as_deref(), Some("clips/one.mp3"));
assert_eq!(a.sampling_rate, Some(48_000));
assert_eq!(a.bytes().map(|b| b.len()), Some(3));
assert!(!a.needs_fetch());
}
#[test]
fn audio_unwraps_lists_and_plain_strings() {
let a = AudioRef::from_value(&json!(["clips/one.wav"])).expect("handle");
assert_eq!(a.path.as_deref(), Some("clips/one.wav"));
let b = AudioRef::from_value(&json!("https://x/y.wav")).expect("handle");
assert_eq!(b.url.as_deref(), Some("https://x/y.wav"));
assert!(AudioRef::from_value(&json!(null)).is_none());
assert!(AudioRef::from_value(&json!("")).is_none());
}
}