Skip to main content

chardet/
lib.rs

1//! Universal character encoding detector — Rust port of chardet 7.
2#![forbid(unsafe_code)]
3
4pub mod codecs;
5pub mod enums;
6pub mod models;
7pub mod output_names;
8pub mod pipeline;
9mod registry_data;
10pub mod registry;
11pub mod result;
12pub mod utils;
13
14pub use pipeline::orchestrator::{run_pipeline, PipelineOptions};
15pub use result::DetectionResult;
16
17/// Public options for [`detect`] and [`detect_all`].
18#[derive(Clone)]
19pub struct DetectOptions {
20    pub encoding_era: u8,
21    pub max_bytes: usize,
22    pub prefer_superset: bool,
23    /// Return chardet 5.x/6.x compatible encoding names (default `true`).
24    pub compat_names: bool,
25    /// If given, only these encodings are considered (names or aliases).
26    pub include_encodings: Option<Vec<String>>,
27    /// If given, these encodings are removed from the candidate set.
28    pub exclude_encodings: Option<Vec<String>>,
29    /// Encoding returned when no candidate survives (default `cp1252`).
30    pub no_match_encoding: String,
31    /// Encoding returned for empty input (default `utf-8`).
32    pub empty_input_encoding: String,
33    /// When `true`, `detect_all` keeps every candidate regardless of confidence.
34    pub ignore_threshold: bool,
35}
36
37impl Default for DetectOptions {
38    fn default() -> Self {
39        DetectOptions {
40            encoding_era: enums::Era::ALL,
41            max_bytes: utils::DEFAULT_MAX_BYTES,
42            prefer_superset: false,
43            compat_names: true,
44            include_encodings: None,
45            exclude_encodings: None,
46            no_match_encoding: "cp1252".to_string(),
47            empty_input_encoding: "utf-8".to_string(),
48            ignore_threshold: false,
49        }
50    }
51}
52
53fn canonical_filter(list: &Option<Vec<String>>) -> Result<Option<Vec<String>>, String> {
54    match list {
55        None => Ok(None),
56        Some(items) => {
57            let mut out = Vec::with_capacity(items.len());
58            for name in items {
59                out.push(registry::validate_encoding(name)?.to_string());
60            }
61            if out.is_empty() {
62                return Err("encoding filter must not be empty".to_string());
63            }
64            Ok(Some(out))
65        }
66    }
67}
68
69fn build_run_options(opts: &DetectOptions) -> Result<PipelineOptions, String> {
70    let include = canonical_filter(&opts.include_encodings)?;
71    let exclude = canonical_filter(&opts.exclude_encodings)?;
72    let no_match = registry::validate_encoding(&opts.no_match_encoding)?.to_string();
73    let empty = registry::validate_encoding(&opts.empty_input_encoding)?.to_string();
74    Ok(PipelineOptions {
75        encoding_era: opts.encoding_era,
76        max_bytes: opts.max_bytes,
77        include_encodings: include,
78        exclude_encodings: exclude,
79        no_match_encoding: no_match,
80        empty_input_encoding: empty,
81        full_ranking: false,
82        input_truncated: false,
83    })
84}
85
86fn apply_output_names(mut result: DetectionResult, data: &[u8], opts: &DetectOptions) -> DetectionResult {
87    let window = &data[..data.len().min(opts.max_bytes)];
88    if opts.prefer_superset {
89        output_names::apply_preferred_superset(&mut result, Some(window));
90    }
91    if opts.compat_names {
92        output_names::apply_compat_names(&mut result);
93    }
94    result
95}
96
97/// Detect the encoding of `data`, applying the public output-name transforms.
98///
99/// Mirrors `chardet.detect`: prefer-superset remapping (when enabled) followed
100/// by chardet 5.x/6.x compatible naming (when enabled).
101pub fn detect(data: &[u8]) -> DetectionResult {
102    detect_with(data, &DetectOptions::default()).unwrap_or_else(|_| result::none_result())
103}
104
105/// Like [`detect`], with explicit options.
106pub fn detect_with(data: &[u8], opts: &DetectOptions) -> Result<DetectionResult, String> {
107    let run_opts = build_run_options(opts)?;
108    let results = run_pipeline(data, &run_opts);
109    let result = results
110        .into_iter()
111        .next()
112        .unwrap_or_else(result::none_result);
113    Ok(apply_output_names(result, data, opts))
114}
115
116/// Return all candidate results sorted by descending confidence.
117///
118/// Mirrors `chardet.detect_all`: results at or below `MINIMUM_THRESHOLD` are
119/// filtered out unless `ignore_threshold` is set (or nothing clears it).
120pub fn detect_all(data: &[u8], opts: &DetectOptions) -> Result<Vec<DetectionResult>, String> {
121    let mut run_opts = build_run_options(opts)?;
122    run_opts.full_ranking = true;
123    let results = run_pipeline(data, &run_opts);
124    let mut dicts: Vec<DetectionResult> = if opts.ignore_threshold {
125        results
126    } else {
127        let filtered: Vec<DetectionResult> = results
128            .iter()
129            .filter(|r| r.confidence > utils::MINIMUM_THRESHOLD)
130            .cloned()
131            .collect();
132        if filtered.is_empty() {
133            results
134        } else {
135            filtered
136        }
137    };
138    for d in &mut dicts {
139        let out = apply_output_names(d.clone(), data, opts);
140        *d = out;
141    }
142    dicts.sort_by(|a, b| b.confidence.partial_cmp(&a.confidence).unwrap());
143    Ok(dicts)
144}