use std::time::Duration;
use serde::Deserialize;
use serde_json::{Map, Value};
use crate::error::{NganoError, Result};
use crate::http::HttpClient;
pub const DEFAULT_HF_BASE: &str = "https://datasets-server.huggingface.co";
pub const MAX_PAGE_ROWS: usize = 100;
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
pub(crate) struct SplitRef {
pub config: String,
pub split: String,
}
#[derive(Debug, Deserialize)]
struct SplitsResponse {
#[serde(default)]
splits: Vec<SplitRef>,
}
#[derive(Debug, Deserialize)]
struct FeatureDef {
name: String,
}
#[derive(Debug, Deserialize)]
struct RowEnvelope {
#[serde(default)]
row: Map<String, Value>,
}
#[derive(Debug, Deserialize)]
struct RowsResponse {
#[serde(default)]
features: Vec<FeatureDef>,
#[serde(default)]
rows: Vec<RowEnvelope>,
#[serde(default)]
num_rows_total: Option<u64>,
}
#[derive(Debug)]
pub(crate) struct RowsPage {
pub columns: Vec<String>,
pub rows: Vec<Map<String, Value>>,
pub total: Option<u64>,
}
#[derive(Debug, Deserialize)]
struct InfoResponse {
#[serde(default)]
dataset_info: Value,
}
#[derive(Debug, Clone)]
pub(crate) struct HfClient {
http: HttpClient,
base: String,
}
impl HfClient {
pub(crate) fn new(
base: &str,
token: Option<String>,
max_retries: u32,
backoff: Duration,
) -> Result<Self> {
Ok(Self {
http: HttpClient::new("datasets-server")?
.with_token(token)
.with_retries(max_retries, backoff),
base: base.trim_end_matches('/').to_string(),
})
}
pub(crate) fn http(&self) -> &HttpClient {
&self.http
}
pub(crate) async fn splits(&self, repo: &str) -> Result<Vec<SplitRef>> {
let url = format!("{}/splits?dataset={}", self.base, encode(repo));
let body: SplitsResponse = self.http.get_json(&url, Some(repo)).await?;
if body.splits.is_empty() {
return Err(NganoError::InvalidResponse {
url,
detail: format!("dataset {repo} reports no splits"),
});
}
Ok(body.splits)
}
pub(crate) async fn rows(
&self,
repo: &str,
split: &SplitRef,
offset: usize,
length: usize,
) -> Result<RowsPage> {
let length = length.clamp(1, MAX_PAGE_ROWS);
let url = format!(
"{}/rows?dataset={}&config={}&split={}&offset={}&length={}",
self.base,
encode(repo),
encode(&split.config),
encode(&split.split),
offset,
length
);
let body: RowsResponse = self.http.get_json(&url, Some(repo)).await?;
let mut columns: Vec<String> = body.features.into_iter().map(|f| f.name).collect();
if columns.is_empty() {
if let Some(first) = body.rows.first() {
columns = first.row.keys().cloned().collect();
}
}
Ok(RowsPage {
columns,
rows: body.rows.into_iter().map(|r| r.row).collect(),
total: body.num_rows_total,
})
}
pub(crate) async fn columns(&self, repo: &str, config: &str) -> Result<Vec<String>> {
let url = format!(
"{}/info?dataset={}&config={}",
self.base,
encode(repo),
encode(config)
);
let body: InfoResponse = self.http.get_json(&url, Some(repo)).await?;
let features = body
.dataset_info
.get("features")
.and_then(Value::as_object)
.map(|m| m.keys().cloned().collect())
.unwrap_or_default();
Ok(features)
}
}
fn encode(value: &str) -> String {
let mut out = String::with_capacity(value.len() + 8);
for byte in value.as_bytes() {
match byte {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
out.push(*byte as char)
}
other => out.push_str(&format!("%{other:02X}")),
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn query_values_are_percent_encoded() {
assert_eq!(encode("google/fleurs"), "google%2Ffleurs");
assert_eq!(encode("sn_zw"), "sn_zw");
assert_eq!(encode("a b"), "a%20b");
}
}