1use crate::error::{Error, Result};
4
5pub const VALID_TASK_CATEGORIES: &[&str] = &[
8 "text-classification",
10 "token-classification",
11 "table-question-answering",
12 "question-answering",
13 "zero-shot-classification",
14 "translation",
15 "summarization",
16 "feature-extraction",
17 "text-generation",
18 "text2text-generation",
19 "fill-mask",
20 "sentence-similarity",
21 "text-to-speech",
22 "text-to-audio",
23 "automatic-speech-recognition",
24 "audio-to-audio",
25 "audio-classification",
26 "voice-activity-detection",
27 "image-classification",
29 "object-detection",
30 "image-segmentation",
31 "text-to-image",
32 "image-to-text",
33 "image-to-image",
34 "image-to-video",
35 "unconditional-image-generation",
36 "video-classification",
37 "reinforcement-learning",
38 "robotics",
39 "tabular-classification",
40 "tabular-regression",
41 "visual-question-answering",
43 "document-question-answering",
44 "zero-shot-image-classification",
45 "graph-ml",
46 "mask-generation",
47 "zero-shot-object-detection",
48 "text-to-3d",
49 "image-to-3d",
50 "image-feature-extraction",
51 "other",
53];
54
55pub const VALID_SIZE_CATEGORIES: &[&str] = &[
57 "n<1K",
58 "1K<n<10K",
59 "10K<n<100K",
60 "100K<n<1M",
61 "1M<n<10M",
62 "10M<n<100M",
63 "100M<n<1B",
64 "1B<n<10B",
65 "10B<n<100B",
66 "100B<n<1T",
67 "n>1T",
68];
69
70pub const VALID_LICENSES: &[&str] = &[
72 "apache-2.0",
73 "mit",
74 "gpl-3.0",
75 "gpl-2.0",
76 "bsd-3-clause",
77 "bsd-2-clause",
78 "cc-by-4.0",
79 "cc-by-sa-4.0",
80 "cc-by-nc-4.0",
81 "cc-by-nc-sa-4.0",
82 "cc0-1.0",
83 "unlicense",
84 "openrail",
85 "openrail++",
86 "bigscience-openrail-m",
87 "creativeml-openrail-m",
88 "llama2",
89 "llama3",
90 "llama3.1",
91 "gemma",
92 "other",
93];
94
95#[derive(Debug, Clone)]
97pub struct ValidationError {
98 pub field: String,
100 pub value: String,
102 pub suggestions: Vec<String>,
104}
105
106impl std::fmt::Display for ValidationError {
107 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
108 write!(f, "Invalid '{}': '{}' is not valid", self.field, self.value)?;
109 if !self.suggestions.is_empty() {
110 write!(f, ". Did you mean: {}?", self.suggestions.join(", "))?;
111 }
112 Ok(())
113 }
114}
115
116#[derive(Debug, Default)]
137pub struct DatasetCardValidator;
138
139impl DatasetCardValidator {
140 pub fn validate_readme(content: &str) -> Vec<ValidationError> {
146 let mut errors = Vec::new();
147
148 let Some(yaml_content) = Self::extract_frontmatter(content) else {
150 return errors;
151 };
152
153 if yaml_content.trim().is_empty() {
154 return errors;
155 }
156
157 let yaml = match serde_saphyr::from_str_with_options::<serde_json::Value>(
159 &yaml_content,
160 serde_saphyr::options! { strict_booleans: true, merge_keys: serde_saphyr::MergeKeyPolicy::AsOrdinary, emit_comments: false, reject_non_finite_typeless_float: false },
161 ) {
162 Ok(yaml) => yaml,
163 Err(e) => {
164 errors.push(ValidationError {
165 field: "front matter".to_string(),
166 value: e.to_string(),
167 suggestions: Vec::new(),
168 });
169 return errors;
170 }
171 };
172
173 Self::check_field(&yaml, "task_categories", VALID_TASK_CATEGORIES, &mut errors);
174 Self::check_field(&yaml, "size_categories", VALID_SIZE_CATEGORIES, &mut errors);
175
176 errors
177 }
178
179 fn check_field(
181 yaml: &serde_json::Value,
182 field: &str,
183 valid: &[&str],
184 errors: &mut Vec<ValidationError>,
185 ) {
186 let Some(arr) = yaml.get(field).and_then(serde_json::Value::as_array) else {
187 return;
188 };
189 for value in arr.iter().filter_map(serde_json::Value::as_str) {
190 if !valid.contains(&value) {
191 errors.push(ValidationError {
192 field: field.to_string(),
193 value: value.to_string(),
194 suggestions: Self::suggest_similar(value, valid),
195 });
196 }
197 }
198 }
199
200 pub fn validate_readme_strict(content: &str) -> Result<()> {
204 let errors = Self::validate_readme(content);
205 if errors.is_empty() {
206 Ok(())
207 } else {
208 let msg = errors
209 .iter()
210 .map(|e| e.to_string())
211 .collect::<Vec<_>>()
212 .join("; ");
213 Err(Error::invalid_config(msg))
214 }
215 }
216
217 fn extract_frontmatter(content: &str) -> Option<String> {
219 let content = content.trim_start();
220 if !content.starts_with("---") {
221 return None;
222 }
223
224 let rest = &content[3..];
225 let end_idx = rest.find("\n---")?;
226 Some(rest[..end_idx].to_string())
227 }
228
229 pub(crate) fn suggest_similar(value: &str, valid: &[&str]) -> Vec<String> {
231 let value_lower = value.to_lowercase();
232 valid
233 .iter()
234 .filter(|v| {
235 let v_lower = v.to_lowercase();
236 v_lower.contains(&value_lower)
237 || value_lower.contains(&v_lower)
238 || Self::levenshtein(&value_lower, &v_lower) <= 3
239 })
240 .take(3)
241 .map(|s| (*s).to_string())
242 .collect()
243 }
244
245 pub(crate) fn levenshtein(a: &str, b: &str) -> usize {
247 let a_chars: Vec<char> = a.chars().collect();
248 let b_chars: Vec<char> = b.chars().collect();
249 let m = a_chars.len();
250 let n = b_chars.len();
251
252 if m == 0 {
253 return n;
254 }
255 if n == 0 {
256 return m;
257 }
258
259 let mut dp = vec![vec![0; n + 1]; m + 1];
260
261 for (i, row) in dp.iter_mut().enumerate().take(m + 1) {
262 row[0] = i;
263 }
264 for (j, cell) in dp[0].iter_mut().enumerate().take(n + 1) {
265 *cell = j;
266 }
267
268 for i in 1..=m {
269 for j in 1..=n {
270 let cost = usize::from(a_chars[i - 1] != b_chars[j - 1]);
271 dp[i][j] = (dp[i - 1][j] + 1)
272 .min(dp[i][j - 1] + 1)
273 .min(dp[i - 1][j - 1] + cost);
274 }
275 }
276
277 dp[m][n]
278 }
279
280 #[must_use]
282 pub fn is_valid_task_category(category: &str) -> bool {
283 VALID_TASK_CATEGORIES.contains(&category)
284 }
285
286 #[must_use]
288 pub fn is_valid_license(license: &str) -> bool {
289 let lower = license.to_lowercase();
290 VALID_LICENSES.contains(&lower.as_str())
291 }
292
293 #[must_use]
295 pub fn is_valid_size_category(size: &str) -> bool {
296 VALID_SIZE_CATEGORIES.contains(&size)
297 }
298
299 #[must_use]
301 pub fn suggest_task_category(invalid: &str) -> Option<&'static str> {
302 match invalid {
303 "text2text-generation" => Some("text2text-generation"), "code-generation" | "code" => Some("text-generation"),
305 "qa" | "QA" => Some("question-answering"),
306 "ner" | "NER" => Some("token-classification"),
307 "sentiment" => Some("text-classification"),
308 _ => VALID_TASK_CATEGORIES
309 .iter()
310 .find(|c| c.starts_with(invalid) || invalid.starts_with(*c))
311 .copied(),
312 }
313 }
314}