oxirs_ttl/toolkit/
format_detector.rs

1//! Automatic RDF format detection
2//!
3//! This module provides utilities for detecting RDF serialization formats from:
4//! - File extensions
5//! - Content analysis
6//! - MIME types
7//!
8//! Supported formats:
9//! - Turtle (.ttl)
10//! - N-Triples (.nt)
11//! - N-Quads (.nq)
12//! - TriG (.trig)
13
14use crate::toolkit::FastScanner;
15use std::path::Path;
16
17/// RDF serialization formats
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
19pub enum RdfFormat {
20    /// Turtle (Terse RDF Triple Language)
21    Turtle,
22    /// N-Triples (line-oriented triple format)
23    NTriples,
24    /// N-Quads (line-oriented quad format)
25    NQuads,
26    /// TriG (Turtle with named graphs)
27    TriG,
28}
29
30impl RdfFormat {
31    /// Get the standard file extension for this format
32    pub fn extension(&self) -> &'static str {
33        match self {
34            RdfFormat::Turtle => "ttl",
35            RdfFormat::NTriples => "nt",
36            RdfFormat::NQuads => "nq",
37            RdfFormat::TriG => "trig",
38        }
39    }
40
41    /// Get the MIME type for this format
42    pub fn mime_type(&self) -> &'static str {
43        match self {
44            RdfFormat::Turtle => "text/turtle",
45            RdfFormat::NTriples => "application/n-triples",
46            RdfFormat::NQuads => "application/n-quads",
47            RdfFormat::TriG => "application/trig",
48        }
49    }
50
51    /// Get a human-readable name for this format
52    pub fn name(&self) -> &'static str {
53        match self {
54            RdfFormat::Turtle => "Turtle",
55            RdfFormat::NTriples => "N-Triples",
56            RdfFormat::NQuads => "N-Quads",
57            RdfFormat::TriG => "TriG",
58        }
59    }
60}
61
62/// Format detection result with confidence score
63#[derive(Debug, Clone)]
64pub struct DetectionResult {
65    /// Detected format
66    pub format: RdfFormat,
67    /// Confidence score (0.0 to 1.0)
68    pub confidence: f64,
69    /// Detection method used
70    pub method: DetectionMethod,
71}
72
73/// Method used for format detection
74#[derive(Debug, Clone, Copy, PartialEq, Eq)]
75pub enum DetectionMethod {
76    /// Detected from file extension
77    FileExtension,
78    /// Detected from MIME type
79    MimeType,
80    /// Detected by analyzing content
81    ContentAnalysis,
82    /// Multiple methods agreed
83    Combined,
84}
85
86/// Format detector with configurable strategies
87#[derive(Debug, Clone)]
88pub struct FormatDetector {
89    /// Number of bytes to analyze for content detection
90    sample_size: usize,
91    /// Minimum confidence threshold (0.0 to 1.0)
92    min_confidence: f64,
93}
94
95impl Default for FormatDetector {
96    fn default() -> Self {
97        Self::new()
98    }
99}
100
101impl FormatDetector {
102    /// Create a new format detector with default settings
103    pub fn new() -> Self {
104        Self {
105            sample_size: 4096, // Analyze first 4KB
106            min_confidence: 0.6,
107        }
108    }
109
110    /// Set the sample size for content analysis
111    pub fn with_sample_size(mut self, size: usize) -> Self {
112        self.sample_size = size;
113        self
114    }
115
116    /// Set the minimum confidence threshold
117    pub fn with_min_confidence(mut self, confidence: f64) -> Self {
118        self.min_confidence = confidence.clamp(0.0, 1.0);
119        self
120    }
121
122    /// Detect format from file path
123    pub fn detect_from_path(&self, path: &Path) -> Option<DetectionResult> {
124        path.extension()
125            .and_then(|ext| ext.to_str())
126            .and_then(|ext| self.detect_from_extension(ext))
127    }
128
129    /// Detect format from file extension
130    pub fn detect_from_extension(&self, extension: &str) -> Option<DetectionResult> {
131        let ext_lower = extension.to_lowercase();
132        let format = match ext_lower.as_str() {
133            "ttl" | "turtle" => RdfFormat::Turtle,
134            "nt" | "ntriples" => RdfFormat::NTriples,
135            "nq" | "nquads" => RdfFormat::NQuads,
136            "trig" => RdfFormat::TriG,
137            _ => return None,
138        };
139
140        Some(DetectionResult {
141            format,
142            confidence: 0.9, // High confidence from extension
143            method: DetectionMethod::FileExtension,
144        })
145    }
146
147    /// Detect format from MIME type
148    pub fn detect_from_mime_type(&self, mime_type: &str) -> Option<DetectionResult> {
149        let mime_lower = mime_type.to_lowercase();
150        let format = if mime_lower.contains("turtle") {
151            RdfFormat::Turtle
152        } else if mime_lower.contains("n-triples") || mime_lower.contains("ntriples") {
153            RdfFormat::NTriples
154        } else if mime_lower.contains("n-quads") || mime_lower.contains("nquads") {
155            RdfFormat::NQuads
156        } else if mime_lower.contains("trig") {
157            RdfFormat::TriG
158        } else {
159            return None;
160        };
161
162        Some(DetectionResult {
163            format,
164            confidence: 0.95, // Very high confidence from MIME type
165            method: DetectionMethod::MimeType,
166        })
167    }
168
169    /// Detect format by analyzing content
170    pub fn detect_from_content(&self, content: &[u8]) -> Option<DetectionResult> {
171        let sample = &content[..content.len().min(self.sample_size)];
172        let scanner = FastScanner::new(sample);
173
174        let mut scores = FormatScores::default();
175
176        // Analyze content for format-specific patterns
177        self.analyze_directives(&scanner, &mut scores);
178        self.analyze_syntax(&scanner, &mut scores);
179        self.analyze_structure(&scanner, &mut scores);
180
181        // Determine format from scores
182        scores.determine_format(self.min_confidence)
183    }
184
185    /// Detect format using all available information
186    pub fn detect(
187        &self,
188        path: Option<&Path>,
189        mime_type: Option<&str>,
190        content: Option<&[u8]>,
191    ) -> Option<DetectionResult> {
192        let mut results = Vec::new();
193
194        // Try file extension
195        if let Some(path) = path {
196            if let Some(result) = self.detect_from_path(path) {
197                results.push(result);
198            }
199        }
200
201        // Try MIME type
202        if let Some(mime) = mime_type {
203            if let Some(result) = self.detect_from_mime_type(mime) {
204                results.push(result);
205            }
206        }
207
208        // Try content analysis
209        if let Some(content) = content {
210            if let Some(result) = self.detect_from_content(content) {
211                results.push(result);
212            }
213        }
214
215        // Combine results
216        self.combine_results(results)
217    }
218
219    /// Analyze directives (@prefix, @base, PREFIX, BASE)
220    fn analyze_directives(&self, scanner: &FastScanner, scores: &mut FormatScores) {
221        let mut pos = 0;
222        let mut has_prefix_directive = false;
223
224        while pos < scanner.len() {
225            pos = scanner.skip_whitespace_and_comments(pos);
226            if pos >= scanner.len() {
227                break;
228            }
229
230            // Check for @prefix or @base (Turtle/TriG)
231            // Give Turtle slightly higher base score since it's more common
232            if scanner.byte_at(pos) == Some(b'@') {
233                scores.turtle += 0.8;
234                scores.trig += 0.5; // Lower base score for TriG
235                has_prefix_directive = true;
236                break; // Found strong indicator
237            }
238
239            // Check for PREFIX or BASE (also Turtle/TriG)
240            let slice = scanner.slice(pos, pos + 6);
241            if slice.starts_with(b"PREFIX") || slice.starts_with(b"BASE") {
242                scores.turtle += 0.7;
243                scores.trig += 0.4; // Lower base score for TriG
244                has_prefix_directive = true;
245                break;
246            }
247
248            // Move to next line
249            if let Some(newline) = scanner.find_line_end(pos) {
250                pos = newline + 1;
251            } else {
252                break;
253            }
254        }
255
256        // Prefix directives rule OUT N-Triples and N-Quads
257        if has_prefix_directive {
258            scores.ntriples = 0.0;
259            scores.nquads = 0.0;
260        }
261    }
262
263    /// Analyze syntax features (abbreviated syntax, named graphs)
264    fn analyze_syntax(&self, scanner: &FastScanner, scores: &mut FormatScores) {
265        let content = scanner.slice(0, scanner.len());
266        let mut has_curly_braces = false;
267
268        // Look for Turtle/TriG abbreviated syntax
269        for byte in content {
270            match byte {
271                b';' => {
272                    scores.turtle += 0.1;
273                    scores.trig += 0.1;
274                }
275                b',' => {
276                    scores.turtle += 0.05;
277                    scores.trig += 0.05;
278                }
279                b'[' | b']' => {
280                    scores.turtle += 0.08;
281                    scores.trig += 0.08;
282                }
283                b'{' | b'}' => {
284                    // Named graphs indicate TriG - strong indicator!
285                    has_curly_braces = true;
286                    scores.trig += 0.5;
287                }
288                _ => {}
289            }
290        }
291
292        // If curly braces found, it's likely TriG not Turtle
293        if has_curly_braces {
294            scores.turtle *= 0.5; // Reduce Turtle confidence
295        }
296    }
297
298    /// Analyze line-based structure (N-Triples/N-Quads)
299    fn analyze_structure(&self, scanner: &FastScanner, scores: &mut FormatScores) {
300        let mut pos = 0;
301        let mut line_count = 0;
302        let mut triple_lines = 0;
303        let mut quad_lines = 0;
304        let mut has_prefixed_names = false;
305        let mut has_full_iris = false;
306
307        while pos < scanner.len() && line_count < 10 {
308            pos = scanner.skip_whitespace_and_comments(pos);
309            if pos >= scanner.len() {
310                break;
311            }
312
313            let line_start = pos;
314            let line_end = scanner.find_line_end(pos).unwrap_or(scanner.len());
315
316            // Count elements on this line
317            let line_content = scanner.slice(line_start, line_end);
318            let element_count = self.count_line_elements(line_content);
319
320            // Check for prefixed names (e.g., ex:subject)
321            // N-Triples/N-Quads only use full IRIs or blank nodes
322            if self.has_prefixed_name(line_content) {
323                has_prefixed_names = true;
324            }
325
326            // Check if line starts with < (full IRI) or _: (blank node)
327            let trimmed_start = scanner.skip_whitespace(line_start);
328            if trimmed_start < scanner.len() {
329                match scanner.byte_at(trimmed_start) {
330                    Some(b'<') | Some(b'_') => has_full_iris = true,
331                    _ => {}
332                }
333            }
334
335            match element_count {
336                3 => triple_lines += 1,
337                4 => quad_lines += 1,
338                _ => {}
339            }
340
341            line_count += 1;
342            pos = line_end + 1;
343        }
344
345        // Prefixed names rule OUT N-Triples and N-Quads
346        if has_prefixed_names {
347            scores.turtle += 0.4;
348            scores.trig += 0.4;
349            scores.ntriples = 0.0;
350            scores.nquads = 0.0;
351            return;
352        }
353
354        // Score based on line patterns (only if no prefixed names)
355        if line_count > 0 && has_full_iris {
356            let triple_ratio = triple_lines as f64 / line_count as f64;
357            let quad_ratio = quad_lines as f64 / line_count as f64;
358
359            if triple_ratio > 0.7 {
360                scores.ntriples += 0.6;
361            }
362
363            if quad_ratio > 0.7 {
364                scores.nquads += 0.7;
365            }
366        }
367    }
368
369    /// Check if a line contains a prefixed name (e.g., ex:subject)
370    fn has_prefixed_name(&self, line: &[u8]) -> bool {
371        let mut i = 0;
372        while i < line.len() {
373            // Skip whitespace
374            while i < line.len() && (line[i] == b' ' || line[i] == b'\t') {
375                i += 1;
376            }
377
378            if i >= line.len() {
379                break;
380            }
381
382            // Skip angle bracket IRIs
383            if line[i] == b'<' {
384                while i < line.len() && line[i] != b'>' {
385                    i += 1;
386                }
387                i += 1;
388                continue;
389            }
390
391            // Skip strings
392            if line[i] == b'"' {
393                i += 1;
394                while i < line.len() && line[i] != b'"' {
395                    if line[i] == b'\\' {
396                        i += 2;
397                    } else {
398                        i += 1;
399                    }
400                }
401                i += 1;
402                continue;
403            }
404
405            // Check for prefixed name pattern: [a-zA-Z]+:
406            if line[i].is_ascii_alphabetic() {
407                while i < line.len() && (line[i].is_ascii_alphanumeric() || line[i] == b'_') {
408                    i += 1;
409                }
410
411                // Found a colon after alphanumeric chars?
412                if i < line.len() && line[i] == b':' {
413                    // Make sure it's not a trailing colon at end
414                    if i + 1 < line.len() && line[i + 1] != b' ' {
415                        return true;
416                    }
417                }
418
419                continue;
420            }
421
422            i += 1;
423        }
424
425        false
426    }
427
428    /// Count elements on a line (space-separated)
429    ///
430    /// Counts RDF elements on a line, excluding the trailing period.
431    /// N-Triples has 3 elements (subject predicate object)
432    /// N-Quads has 4 elements (subject predicate object graph)
433    fn count_line_elements(&self, line: &[u8]) -> usize {
434        let mut count = 0;
435        let mut in_string = false;
436        let mut in_angle_bracket = false;
437        let mut prev_space = true;
438        let mut elements = Vec::new();
439        let mut current_element_start = 0;
440
441        for (i, &byte) in line.iter().enumerate() {
442            match byte {
443                b'"' => in_string = !in_string,
444                b'<' if !in_string => in_angle_bracket = true,
445                b'>' if !in_string => in_angle_bracket = false,
446                b' ' | b'\t' if !in_string && !in_angle_bracket => {
447                    if !prev_space {
448                        elements.push(&line[current_element_start..i]);
449                    }
450                    prev_space = true;
451                    current_element_start = i + 1;
452                }
453                _ => {
454                    if prev_space && !in_string {
455                        prev_space = false;
456                    }
457                }
458            }
459        }
460
461        // Add final element if any
462        if !prev_space && current_element_start < line.len() {
463            elements.push(&line[current_element_start..]);
464        }
465
466        // Filter out trailing period and count
467        for elem in elements {
468            // Skip if it's just a period or period with whitespace
469            if elem.is_empty() || elem == b"." {
470                continue;
471            }
472            count += 1;
473        }
474
475        count
476    }
477
478    /// Combine multiple detection results
479    fn combine_results(&self, results: Vec<DetectionResult>) -> Option<DetectionResult> {
480        if results.is_empty() {
481            return None;
482        }
483
484        if results.len() == 1 {
485            return Some(results[0].clone());
486        }
487
488        // Weight results by confidence and method
489        let mut format_scores: std::collections::HashMap<RdfFormat, f64> =
490            std::collections::HashMap::new();
491
492        for result in &results {
493            let weight = match result.method {
494                DetectionMethod::MimeType => 1.5,
495                DetectionMethod::FileExtension => 1.2,
496                DetectionMethod::ContentAnalysis => 1.0,
497                DetectionMethod::Combined => 1.3,
498            };
499
500            *format_scores.entry(result.format).or_insert(0.0) += result.confidence * weight;
501        }
502
503        // Find format with highest score
504        format_scores
505            .into_iter()
506            .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
507            .map(|(format, score)| DetectionResult {
508                format,
509                confidence: (score / results.len() as f64).min(1.0),
510                method: DetectionMethod::Combined,
511            })
512    }
513}
514
515/// Scores for each format during content analysis
516#[derive(Debug, Default)]
517struct FormatScores {
518    turtle: f64,
519    ntriples: f64,
520    nquads: f64,
521    trig: f64,
522}
523
524impl FormatScores {
525    fn determine_format(&self, min_confidence: f64) -> Option<DetectionResult> {
526        let formats = [
527            (RdfFormat::Turtle, self.turtle),
528            (RdfFormat::NTriples, self.ntriples),
529            (RdfFormat::NQuads, self.nquads),
530            (RdfFormat::TriG, self.trig),
531        ];
532
533        formats
534            .iter()
535            .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
536            .filter(|(_, score)| *score >= min_confidence)
537            .map(|(format, score)| DetectionResult {
538                format: *format,
539                confidence: *score,
540                method: DetectionMethod::ContentAnalysis,
541            })
542    }
543}
544
545#[cfg(test)]
546mod tests {
547    use super::*;
548
549    #[test]
550    fn test_detect_turtle_from_extension() {
551        let detector = FormatDetector::new();
552
553        let result = detector
554            .detect_from_extension("ttl")
555            .expect("detection should succeed");
556        assert_eq!(result.format, RdfFormat::Turtle);
557        assert!(result.confidence > 0.8);
558    }
559
560    #[test]
561    fn test_detect_ntriples_from_extension() {
562        let detector = FormatDetector::new();
563
564        let result = detector
565            .detect_from_extension("nt")
566            .expect("detection should succeed");
567        assert_eq!(result.format, RdfFormat::NTriples);
568    }
569
570    #[test]
571    fn test_detect_from_path() {
572        let detector = FormatDetector::new();
573        let path = Path::new("test.ttl");
574
575        let result = detector
576            .detect_from_path(path)
577            .expect("detection should succeed");
578        assert_eq!(result.format, RdfFormat::Turtle);
579    }
580
581    #[test]
582    fn test_detect_turtle_from_content() {
583        let detector = FormatDetector::new();
584        let content = b"@prefix ex: <http://example.org/> .\nex:subject ex:predicate ex:object .";
585
586        let result = detector
587            .detect_from_content(content)
588            .expect("detection should succeed");
589        assert_eq!(result.format, RdfFormat::Turtle);
590    }
591
592    #[test]
593    fn test_detect_trig_from_content() {
594        let detector = FormatDetector::new();
595        let content = b"@prefix ex: <http://example.org/> .\nex:graph { ex:s ex:p ex:o . }";
596
597        let result = detector
598            .detect_from_content(content)
599            .expect("detection should succeed");
600        assert_eq!(result.format, RdfFormat::TriG);
601    }
602
603    #[test]
604    fn test_detect_ntriples_from_content() {
605        let detector = FormatDetector::new();
606        let content = b"<http://example.org/s> <http://example.org/p> <http://example.org/o> .\n<http://example.org/s2> <http://example.org/p2> <http://example.org/o2> .";
607
608        let result = detector
609            .detect_from_content(content)
610            .expect("detection should succeed");
611        assert_eq!(result.format, RdfFormat::NTriples);
612    }
613
614    #[test]
615    fn test_detect_from_mime_type() {
616        let detector = FormatDetector::new();
617
618        let result = detector
619            .detect_from_mime_type("text/turtle")
620            .expect("detection should succeed");
621        assert_eq!(result.format, RdfFormat::Turtle);
622
623        let result = detector
624            .detect_from_mime_type("application/n-triples")
625            .expect("detection should succeed");
626        assert_eq!(result.format, RdfFormat::NTriples);
627    }
628
629    #[test]
630    fn test_combined_detection() {
631        let detector = FormatDetector::new();
632        let path = Path::new("test.ttl");
633        let content = b"@prefix ex: <http://example.org/> .";
634
635        let result = detector
636            .detect(Some(path), None, Some(content))
637            .expect("detection should succeed");
638        assert_eq!(result.format, RdfFormat::Turtle);
639        assert!(result.confidence > 0.8);
640    }
641
642    #[test]
643    fn test_format_properties() {
644        assert_eq!(RdfFormat::Turtle.extension(), "ttl");
645        assert_eq!(RdfFormat::Turtle.mime_type(), "text/turtle");
646        assert_eq!(RdfFormat::Turtle.name(), "Turtle");
647    }
648}
oxirs_ttl/toolkit/format_detector.rs

oxirs_ttl/toolkit/
format_detector.rs
