document_svg/document/
arff.rs

1//! Bounded Attribute-Relation File Format (ARFF) dataset preview.
2//!
3//! ARFF is the text dataset format used by Weka and related tools.  The
4//! converter keeps the relation header and renders dense or sparse instances
5//! as a paginated inert table.  It never evaluates dates, nominal values, or
6//! relation-valued attributes.
7
8use std::path::Path;
9
10use crate::convert::{ConvertOptions, PageConsumer, read_limited_file};
11use crate::error::{Error, Result};
12use crate::table::{TableAlign, TableData, convert_table_pages};
13
14const MAX_ARFF_BYTES: u64 = 64 * 1024 * 1024;
15const MAX_ARFF_LINES: usize = 1_000_000;
16const MAX_ARFF_LINE_BYTES: usize = 1024 * 1024;
17const MAX_ARFF_ATTRIBUTES: usize = 256;
18const MAX_ARFF_RECORDS: usize = 50_000;
19const MAX_ARFF_CELLS: usize = 1_000_000;
20const MAX_ARFF_VALUE_BYTES: usize = 64 * 1024;
21const MAX_ARFF_NAME_BYTES: usize = 4 * 1024;
22
23/// Returns true for an ARFF header prefix without requiring a file extension.
24pub(crate) fn looks_like_prefix(prefix: &[u8]) -> bool {
25    let Ok(text) = std::str::from_utf8(prefix) else {
26        return false;
27    };
28    let mut relation = false;
29    let mut attribute = false;
30    for line in text.lines() {
31        let line = strip_comment(line).trim();
32        let keyword = line
33            .split_ascii_whitespace()
34            .next()
35            .unwrap_or_default()
36            .to_ascii_lowercase();
37        relation |= keyword == "@relation";
38        attribute |= keyword == "@attribute";
39    }
40    relation && attribute
41}
42
43pub(crate) fn convert(
44    path: &Path,
45    options: &ConvertOptions,
46    sink: &mut dyn PageConsumer,
47) -> Result<Vec<String>> {
48    let bytes = read_limited_file(
49        path,
50        options.max_input_bytes.min(MAX_ARFF_BYTES),
51        "ARFF input",
52    )?;
53    let text = String::from_utf8(bytes)
54        .map_err(|error| Error::InvalidInput(format!("ARFF input must be UTF-8/ASCII: {error}")))?;
55    let (mut table, relation, warnings) = parse_arff(&text)?;
56    let mut page_sink = ArffPageSink {
57        inner: sink,
58        relation: &relation,
59        warnings: &warnings,
60    };
61    convert_table_pages(&mut table, "arff", options, &mut page_sink)?;
62    Ok(warnings)
63}
64
65struct ArffPageSink<'a> {
66    inner: &'a mut dyn PageConsumer,
67    relation: &'a str,
68    warnings: &'a [String],
69}
70
71impl PageConsumer for ArffPageSink<'_> {
72    fn consume(&mut self, mut page: crate::ir::Page) -> Result<()> {
73        page.source_format = "arff".into();
74        page.title = if self.relation.is_empty() {
75            "ARFF dataset".into()
76        } else {
77            format!("ARFF — {}", self.relation)
78        };
79        if !self.relation.is_empty() {
80            page.description = format!("ARFF relation '{}' dataset table", self.relation);
81        }
82        for warning in self.warnings {
83            page.warn(warning.clone());
84        }
85        self.inner.consume(page)
86    }
87}
88
89#[derive(Clone, Debug)]
90struct Attribute {
91    name: String,
92    kind: String,
93}
94
95fn parse_arff(text: &str) -> Result<(TableData, String, Vec<String>)> {
96    if text.len() as u64 > MAX_ARFF_BYTES {
97        return Err(Error::LimitExceeded(format!(
98            "ARFF input exceeds {MAX_ARFF_BYTES} bytes"
99        )));
100    }
101    let mut relation = String::new();
102    let mut attributes = Vec::<Attribute>::new();
103    let mut in_data = false;
104    let mut saw_data = false;
105    let mut rows = Vec::<Vec<String>>::new();
106    let mut warnings = Vec::new();
107    let mut total_cells = 0usize;
108    let lines = text.lines().collect::<Vec<_>>();
109    if lines.len() > MAX_ARFF_LINES {
110        return Err(Error::LimitExceeded(format!(
111            "ARFF input exceeds {MAX_ARFF_LINES} lines"
112        )));
113    }
114
115    for (line_number, original) in lines.iter().enumerate() {
116        let line = original.trim_end_matches('\r');
117        if line.len() > MAX_ARFF_LINE_BYTES {
118            return Err(Error::LimitExceeded(format!(
119                "ARFF line {} exceeds {MAX_ARFF_LINE_BYTES} bytes",
120                line_number + 1
121            )));
122        }
123        let content = strip_comment(line).trim();
124        let content = if line_number == 0 {
125            content.strip_prefix('\u{feff}').unwrap_or(content).trim()
126        } else {
127            content
128        };
129        if content.is_empty() {
130            continue;
131        }
132        if !in_data {
133            let Some((keyword, rest)) = directive(content) else {
134                return Err(Error::InvalidInput(format!(
135                    "ARFF line {} must contain a header directive",
136                    line_number + 1
137                )));
138            };
139            match keyword.as_str() {
140                "@relation" => {
141                    if !relation.is_empty() {
142                        return Err(Error::InvalidInput(
143                            "ARFF declares @relation more than once".into(),
144                        ));
145                    }
146                    relation = parse_name(rest, "relation", line_number + 1)?;
147                }
148                "@attribute" => {
149                    if relation.is_empty() {
150                        return Err(Error::InvalidInput(format!(
151                            "ARFF @attribute appears before @relation at line {}",
152                            line_number + 1
153                        )));
154                    }
155                    if attributes.len() >= MAX_ARFF_ATTRIBUTES {
156                        return Err(Error::LimitExceeded(format!(
157                            "ARFF exceeds {MAX_ARFF_ATTRIBUTES} attributes"
158                        )));
159                    }
160                    let (name, kind) = parse_attribute(rest, line_number + 1)?;
161                    if attributes.iter().any(|attribute| attribute.name == name) {
162                        return Err(Error::InvalidInput(format!(
163                            "ARFF declares duplicate attribute '{name}'"
164                        )));
165                    }
166                    if kind.eq_ignore_ascii_case("relational") {
167                        return Err(Error::Unsupported(
168                            "ARFF relational attributes are unsupported; flat attributes are required".into(),
169                        ));
170                    }
171                    attributes.push(Attribute { name, kind });
172                }
173                "@data" => {
174                    if relation.is_empty() || attributes.is_empty() {
175                        return Err(Error::InvalidInput(
176                            "ARFF @data requires a relation and at least one attribute".into(),
177                        ));
178                    }
179                    if saw_data {
180                        return Err(Error::InvalidInput(
181                            "ARFF declares @data more than once".into(),
182                        ));
183                    }
184                    saw_data = true;
185                    in_data = true;
186                }
187                "@end" => {
188                    return Err(Error::Unsupported(
189                        "ARFF relational attribute sections are unsupported".into(),
190                    ));
191                }
192                _ => warnings.push(format!("ARFF header directive '{}' was ignored", keyword)),
193            }
194            continue;
195        }
196
197        if rows.len() >= MAX_ARFF_RECORDS {
198            return Err(Error::LimitExceeded(format!(
199                "ARFF exceeds {MAX_ARFF_RECORDS} data records"
200            )));
201        }
202        let row = if content.starts_with('{') {
203            parse_sparse_record(content, attributes.len(), line_number + 1)?
204        } else {
205            parse_dense_record(content, attributes.len(), line_number + 1)?
206        };
207        total_cells = total_cells
208            .checked_add(row.len())
209            .ok_or_else(|| Error::LimitExceeded("ARFF cell count overflowed".into()))?;
210        if total_cells > MAX_ARFF_CELLS {
211            return Err(Error::LimitExceeded(format!(
212                "ARFF exceeds {MAX_ARFF_CELLS} cells"
213            )));
214        }
215        rows.push(row);
216    }
217
218    if relation.is_empty() {
219        return Err(Error::InvalidInput("ARFF is missing @relation".into()));
220    }
221    if attributes.is_empty() {
222        return Err(Error::InvalidInput(
223            "ARFF is missing @attribute declarations".into(),
224        ));
225    }
226    if !saw_data {
227        return Err(Error::InvalidInput("ARFF is missing @data".into()));
228    }
229    let headers = attributes
230        .iter()
231        .map(|attribute| attribute.name.clone())
232        .collect::<Vec<_>>();
233    let mut table = TableData {
234        headers,
235        rows,
236        alignments: attributes
237            .iter()
238            .map(|attribute| {
239                let kind = attribute.kind.to_ascii_lowercase();
240                if matches!(kind.as_str(), "numeric" | "real" | "integer")
241                    || kind.starts_with("date")
242                {
243                    TableAlign::Right
244                } else {
245                    TableAlign::Left
246                }
247            })
248            .collect(),
249        raw_source: String::new(),
250    };
251    // Keep the table renderer's own cell truncation warning, while making the
252    // type visible in the header without changing the underlying column name.
253    for (header, attribute) in table.headers.iter_mut().zip(attributes.iter()) {
254        if !attribute.kind.is_empty() {
255            *header = format!("{} ({})", header, attribute.kind);
256        }
257    }
258    Ok((table, relation, warnings))
259}
260
261fn directive(line: &str) -> Option<(String, &str)> {
262    let at = line.find('@')?;
263    if at != 0 {
264        return None;
265    }
266    let end = line
267        .char_indices()
268        .find(|(_, character)| character.is_ascii_whitespace())
269        .map(|(index, _)| index)
270        .unwrap_or(line.len());
271    Some((line[..end].to_ascii_lowercase(), line[end..].trim()))
272}
273
274fn parse_attribute(rest: &str, line_number: usize) -> Result<(String, String)> {
275    let (name, remainder) = parse_token(rest, line_number, "attribute name")?;
276    let kind = remainder.trim();
277    if kind.is_empty() {
278        return Err(Error::InvalidInput(format!(
279            "ARFF attribute '{name}' has no type at line {line_number}"
280        )));
281    }
282    let kind = if kind.starts_with('{') {
283        if !kind.ends_with('}') {
284            return Err(Error::InvalidInput(format!(
285                "ARFF nominal type is not closed at line {line_number}"
286            )));
287        }
288        // Keep nominal values as a compact type label; data values remain inert text.
289        format!("nominal {}", kind)
290    } else {
291        kind.to_string()
292    };
293    Ok((name, kind))
294}
295
296fn parse_name(rest: &str, label: &str, line_number: usize) -> Result<String> {
297    let (name, remainder) = parse_token(rest, line_number, label)?;
298    if !remainder.trim().is_empty() {
299        return Err(Error::InvalidInput(format!(
300            "ARFF {label} has trailing content at line {line_number}"
301        )));
302    }
303    Ok(name)
304}
305
306fn parse_token<'a>(text: &'a str, line_number: usize, label: &str) -> Result<(String, &'a str)> {
307    let text = text.trim_start();
308    if text.is_empty() {
309        return Err(Error::InvalidInput(format!(
310            "ARFF {label} is missing at line {line_number}"
311        )));
312    }
313    let first = text.as_bytes()[0];
314    if first == b'\'' || first == b'"' {
315        let quote = first as char;
316        let mut escaped = false;
317        for (offset, character) in text[1..].char_indices() {
318            if escaped {
319                escaped = false;
320                continue;
321            }
322            if character == '\\' {
323                escaped = true;
324                continue;
325            }
326            if character == quote {
327                let end = 1 + offset + character.len_utf8();
328                let name = decode_quoted(&text[1..1 + offset], line_number)?;
329                if name.len() > MAX_ARFF_NAME_BYTES {
330                    return Err(Error::LimitExceeded(format!(
331                        "ARFF {label} exceeds {MAX_ARFF_NAME_BYTES} bytes at line {line_number}"
332                    )));
333                }
334                return Ok((name, &text[end..]));
335            }
336        }
337        return Err(Error::InvalidInput(format!(
338            "ARFF {label} quote is not closed at line {line_number}"
339        )));
340    }
341    let end = text
342        .char_indices()
343        .find(|(_, character)| character.is_ascii_whitespace())
344        .map(|(index, _)| index)
345        .unwrap_or(text.len());
346    if end > MAX_ARFF_NAME_BYTES {
347        return Err(Error::LimitExceeded(format!(
348            "ARFF {label} exceeds {MAX_ARFF_NAME_BYTES} bytes at line {line_number}"
349        )));
350    }
351    Ok((text[..end].to_string(), &text[end..]))
352}
353
354fn parse_dense_record(line: &str, columns: usize, line_number: usize) -> Result<Vec<String>> {
355    let values = split_fields(line, ',')?;
356    if values.len() != columns {
357        return Err(Error::InvalidInput(format!(
358            "ARFF dense record at line {line_number} has {} values; expected {columns}",
359            values.len()
360        )));
361    }
362    values
363        .into_iter()
364        .map(|value| normalize_value(&value, line_number))
365        .collect()
366}
367
368fn parse_sparse_record(line: &str, columns: usize, line_number: usize) -> Result<Vec<String>> {
369    if !line.ends_with('}') {
370        return Err(Error::InvalidInput(format!(
371            "ARFF sparse record at line {line_number} is not closed"
372        )));
373    }
374    let inner = line[1..line.len() - 1].trim();
375    let mut values = vec!["0".to_string(); columns];
376    if inner.is_empty() {
377        return Ok(values);
378    }
379    let mut seen = vec![false; columns];
380    for entry in split_fields(inner, ',')? {
381        let entry = entry.trim();
382        let split = entry
383            .char_indices()
384            .find(|(_, character)| character.is_ascii_whitespace())
385            .map(|(index, _)| index)
386            .ok_or_else(|| {
387                Error::InvalidInput(format!(
388                    "ARFF sparse entry at line {line_number} is missing a value"
389                ))
390            })?;
391        let index = entry[..split].parse::<usize>().map_err(|_| {
392            Error::InvalidInput(format!(
393                "ARFF sparse entry has invalid index at line {line_number}"
394            ))
395        })?;
396        if index >= columns {
397            return Err(Error::InvalidInput(format!(
398                "ARFF sparse index {index} is outside {columns} columns at line {line_number}"
399            )));
400        }
401        if seen[index] {
402            return Err(Error::InvalidInput(format!(
403                "ARFF sparse index {index} is repeated at line {line_number}"
404            )));
405        }
406        seen[index] = true;
407        values[index] = normalize_value(entry[split..].trim(), line_number)?;
408    }
409    Ok(values)
410}
411
412fn split_fields(text: &str, delimiter: char) -> Result<Vec<String>> {
413    let mut fields = Vec::new();
414    let mut start = 0usize;
415    let mut quote = None::<char>;
416    let mut escaped = false;
417    for (index, character) in text.char_indices() {
418        if escaped {
419            escaped = false;
420            continue;
421        }
422        if quote.is_some() && character == '\\' {
423            escaped = true;
424            continue;
425        }
426        if let Some(active) = quote {
427            if character == active {
428                quote = None;
429            }
430        } else if character == '\'' || character == '"' {
431            quote = Some(character);
432        } else if character == delimiter {
433            fields.push(text[start..index].trim().to_string());
434            start = index + character.len_utf8();
435        }
436    }
437    if quote.is_some() {
438        return Err(Error::InvalidInput(
439            "ARFF data value quote is not closed".into(),
440        ));
441    }
442    fields.push(text[start..].trim().to_string());
443    Ok(fields)
444}
445
446fn normalize_value(value: &str, line_number: usize) -> Result<String> {
447    let value = value.trim();
448    let decoded = if value.len() >= 2
449        && ((value.starts_with('\'') && value.ends_with('\''))
450            || (value.starts_with('"') && value.ends_with('"')))
451    {
452        decode_quoted(&value[1..value.len() - 1], line_number)?
453    } else {
454        value.to_string()
455    };
456    if decoded.len() > MAX_ARFF_VALUE_BYTES {
457        return Err(Error::LimitExceeded(format!(
458            "ARFF value at line {line_number} exceeds {MAX_ARFF_VALUE_BYTES} bytes"
459        )));
460    }
461    Ok(decoded)
462}
463
464fn decode_quoted(value: &str, line_number: usize) -> Result<String> {
465    let mut output = String::with_capacity(value.len());
466    let mut escaped = false;
467    for character in value.chars() {
468        if escaped {
469            output.push(character);
470            escaped = false;
471        } else if character == '\\' {
472            escaped = true;
473        } else {
474            output.push(character);
475        }
476        if output.len() > MAX_ARFF_VALUE_BYTES {
477            return Err(Error::LimitExceeded(format!(
478                "ARFF value at line {line_number} exceeds {MAX_ARFF_VALUE_BYTES} bytes"
479            )));
480        }
481    }
482    if escaped {
483        return Err(Error::InvalidInput(format!(
484            "ARFF value has a trailing escape at line {line_number}"
485        )));
486    }
487    Ok(output)
488}
489
490fn strip_comment(line: &str) -> &str {
491    let mut quote = None::<char>;
492    let mut escaped = false;
493    for (index, character) in line.char_indices() {
494        if escaped {
495            escaped = false;
496            continue;
497        }
498        if quote.is_some() && character == '\\' {
499            escaped = true;
500            continue;
501        }
502        if let Some(active) = quote {
503            if character == active {
504                quote = None;
505            }
506        } else if character == '\'' || character == '"' {
507            quote = Some(character);
508        } else if character == '%' {
509            return &line[..index];
510        }
511    }
512    line
513}
514
515#[cfg(test)]
516mod tests {
517    use super::{looks_like_prefix, parse_arff};
518
519    #[test]
520    fn parses_dense_nominal_and_quoted_values() {
521        let source = r#"% comment
522@relation weather
523@attribute outlook {sunny, overcast, rainy}
524@attribute temperature numeric
525@attribute note string
526@data
527sunny,25,'windy, warm'
528?,18,"clear"
529"#;
530        let (table, relation, warnings) = parse_arff(source).unwrap();
531        assert_eq!(relation, "weather");
532        assert!(warnings.is_empty());
533        assert_eq!(table.rows[0][2], "windy, warm");
534        assert_eq!(table.rows[1][0], "?");
535        assert!(table.headers[1].contains("numeric"));
536    }
537
538    #[test]
539    fn expands_sparse_records_with_zero_defaults() {
540        let source = "@relation sparse\n@attribute a numeric\n@attribute b string\n@attribute c {x,y}\n@data\n{1 'hello', 2 y}\n";
541        let (table, _, _) = parse_arff(source).unwrap();
542        assert_eq!(table.rows[0], vec!["0", "hello", "y"]);
543    }
544
545    #[test]
546    fn requires_arff_header_for_content_sniffing() {
547        assert!(looks_like_prefix(b"@relation r\n@attribute x numeric\n"));
548        assert!(!looks_like_prefix(b"@relation r\n@data\n1\n"));
549    }
550
551    #[test]
552    fn accepts_a_utf8_bom_before_the_relation_directive() {
553        let source = "\u{feff}@relation r\n@attribute x numeric\n@data\n1\n";
554        let (_, relation, _) = parse_arff(source).unwrap();
555        assert_eq!(relation, "r");
556    }
557
558    #[test]
559    fn rejects_relational_attributes_instead_of_flattening_them() {
560        let source =
561            "@relation r\n@attribute bag relational\n@attribute x numeric\n@end bag\n@data\n\n";
562        let error = parse_arff(source).unwrap_err();
563        assert!(error.to_string().contains("relational attributes"));
564    }
565}
document_svg/document/arff.rs

document_svg/document/
arff.rs
