use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
pub static CANONICAL_DICTIONARY_JSON: &str = include_str!("../schema/type_dictionary.json");
#[path = "type_interpreter_magic.rs"]
mod magic;
pub use magic::*;
#[derive(Clone)]
pub struct TypeDefinition {
pub mime: String,
pub extensions: Vec<String>,
pub universe: String,
pub is_binary: bool,
pub category: String,
pub description: String,
pub magic_bytes: Vec<MagicSignature>,
pub text_patterns: Vec<String>,
pub validator: Option<Arc<dyn Fn(&[u8]) -> bool + Send + Sync>>,
pub priority: i32,
pub metadata: HashMap<String, String>,
}
impl Default for TypeDefinition {
fn default() -> Self {
Self {
mime: String::new(),
extensions: Vec::new(),
universe: "U0".to_string(),
is_binary: false,
category: "text".to_string(),
description: String::new(),
magic_bytes: Vec::new(),
text_patterns: Vec::new(),
validator: None,
priority: 0,
metadata: HashMap::new(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TypeJudgment {
pub mime: String,
pub universe: String,
pub is_binary: bool,
pub confidence: f32,
pub matched_rule: String,
pub extension: Option<String>,
pub category: String,
pub metadata: HashMap<String, String>,
}
#[derive(Debug, Deserialize)]
struct RawDictionary {
#[serde(default)]
types: std::collections::BTreeMap<String, RawTypeEntry>,
}
#[derive(Debug, Deserialize)]
struct RawTypeEntry {
#[serde(default)]
extensions: Vec<String>,
#[serde(default = "default_universe")]
universe: String,
#[serde(default)]
is_binary: bool,
#[serde(default = "default_category")]
category: String,
#[serde(default)]
description: String,
#[serde(default)]
magic_bytes: Vec<MagicSignature>,
#[serde(default)]
text_patterns: Vec<String>,
}
fn default_universe() -> String {
"U0".to_string()
}
fn default_category() -> String {
"text".to_string()
}
pub struct TypeInterpreter {
types: HashMap<String, TypeDefinition>,
ext_map: HashMap<String, String>,
magic_signatures: Vec<(MagicSignature, String)>,
compiled_patterns: Vec<(regex::Regex, String)>,
}
impl Default for TypeInterpreter {
fn default() -> Self {
Self::new()
}
}
impl TypeInterpreter {
pub fn new() -> Self {
Self {
types: HashMap::new(),
ext_map: HashMap::new(),
magic_signatures: Vec::new(),
compiled_patterns: Vec::new(),
}
}
pub fn create_default() -> Self {
let mut interpreter = Self::new();
if let Ok(dict) = serde_json::from_str::<RawDictionary>(CANONICAL_DICTIONARY_JSON) {
for (mime, raw) in dict.types {
let def = TypeDefinition {
mime: mime.clone(),
extensions: raw.extensions,
universe: raw.universe,
is_binary: raw.is_binary,
category: raw.category,
description: raw.description,
magic_bytes: raw.magic_bytes,
text_patterns: raw.text_patterns,
validator: None,
priority: 0,
metadata: HashMap::new(),
};
interpreter.register_type(def);
}
}
interpreter
}
pub fn register_type(&mut self, mut type_def: TypeDefinition) {
let mut norm_exts = Vec::new();
for ext in &type_def.extensions {
let clean = ext.trim().to_lowercase();
let standardized = if clean.starts_with('.') {
clean
} else {
format!(".{}", clean)
};
if !norm_exts.contains(&standardized) {
norm_exts.push(standardized);
}
}
type_def.extensions = norm_exts;
for ext in &type_def.extensions {
self.ext_map.insert(ext.clone(), type_def.mime.clone());
}
for sig in &type_def.magic_bytes {
self.magic_signatures.push((sig.clone(), type_def.mime.clone()));
}
self.magic_signatures.sort_by_key(|a| std::cmp::Reverse(a.0.byte_length()));
for pat in &type_def.text_patterns {
if let Ok(re) = regex::Regex::new(pat) {
self.compiled_patterns.push((re, type_def.mime.clone()));
}
}
self.types.insert(type_def.mime.clone(), type_def);
}
pub fn get_type(&self, mime: &str) -> Option<&TypeDefinition> {
self.types.get(mime)
}
pub fn types_count(&self) -> usize {
self.types.len()
}
pub fn judge(
&self,
data: Option<&[u8]>,
filename: Option<&str>,
ext_hint: Option<&str>,
) -> TypeJudgment {
let raw_bytes = data.unwrap_or(&[]);
let resolved_ext = self.resolve_extension(filename, ext_hint);
for def in self.types.values() {
if def.priority > 0
&& let Some(ref val) = def.validator
&& val(raw_bytes) {
return TypeJudgment {
mime: def.mime.clone(),
universe: def.universe.clone(),
is_binary: def.is_binary,
confidence: 1.0,
matched_rule: "programmatic_predicate_high_priority".to_string(),
extension: def.extensions.first().cloned().or(resolved_ext),
category: def.category.clone(),
metadata: def.metadata.clone(),
};
}
}
if !raw_bytes.is_empty() {
let mut matching: Vec<(&MagicSignature, &TypeDefinition)> = Vec::new();
for (sig, mime) in &self.magic_signatures {
if sig.matches(raw_bytes)
&& let Some(def) = self.types.get(mime) {
matching.push((sig, def));
}
}
if !matching.is_empty() {
let mut best = matching[0];
if let Some(ref ext) = resolved_ext {
if let Some(ext_match) = matching.iter().find(|(_, def)| def.extensions.contains(ext)) {
best = *ext_match;
}
} else if let Some(base_match) = matching.iter().find(|(_, def)| !def.mime.contains('+')) {
best = *base_match;
}
let (best_sig, best_def) = best;
let mut meta = HashMap::new();
meta.insert("signature_hex".to_string(), best_sig.hex.clone());
return TypeJudgment {
mime: best_def.mime.clone(),
universe: best_def.universe.clone(),
is_binary: best_def.is_binary,
confidence: 0.99,
matched_rule: "magic_bytes".to_string(),
extension: best_def.extensions.first().cloned().or(resolved_ext),
category: best_def.category.clone(),
metadata: meta,
};
}
}
let is_bin = self.is_binary_payload(raw_bytes);
if let Some(ref ext) = resolved_ext
&& let Some(mime) = self.ext_map.get(ext)
&& let Some(def) = self.types.get(mime) {
return TypeJudgment {
mime: def.mime.clone(),
universe: def.universe.clone(),
is_binary: def.is_binary,
confidence: 0.95,
matched_rule: "extension_hint".to_string(),
extension: Some(ext.clone()),
category: def.category.clone(),
metadata: HashMap::new(),
};
}
if !is_bin && !raw_bytes.is_empty()
&& let Ok(text) = std::str::from_utf8(raw_bytes) {
let trimmed = text.trim();
if let Some((mime, rule, conf)) = self.probe_structured_text(trimmed) {
let def_opt = self.types.get(&mime);
return TypeJudgment {
mime: mime.clone(),
universe: def_opt.map(|d| d.universe.clone()).unwrap_or_else(|| "U0".to_string()),
is_binary: false,
confidence: conf,
matched_rule: rule,
extension: def_opt.and_then(|d| d.extensions.first().cloned()).or(resolved_ext),
category: def_opt.map(|d| d.category.clone()).unwrap_or_else(|| "text".to_string()),
metadata: HashMap::new(),
};
}
for (re, mime) in &self.compiled_patterns {
if re.is_match(trimmed)
&& let Some(def) = self.types.get(mime) {
return TypeJudgment {
mime: def.mime.clone(),
universe: def.universe.clone(),
is_binary: false,
confidence: 0.95,
matched_rule: "text_pattern".to_string(),
extension: def.extensions.first().cloned().or(resolved_ext),
category: def.category.clone(),
metadata: HashMap::new(),
};
}
}
}
for def in self.types.values() {
if def.priority == 0
&& let Some(ref val) = def.validator
&& val(raw_bytes) {
return TypeJudgment {
mime: def.mime.clone(),
universe: def.universe.clone(),
is_binary: def.is_binary,
confidence: 0.92,
matched_rule: "programmatic_predicate".to_string(),
extension: def.extensions.first().cloned().or(resolved_ext),
category: def.category.clone(),
metadata: def.metadata.clone(),
};
}
}
if is_bin {
TypeJudgment {
mime: "application/octet-stream".to_string(),
universe: "U0".to_string(),
is_binary: true,
confidence: 0.50,
matched_rule: "binary_fallback".to_string(),
extension: resolved_ext.or_else(|| Some(".bin".to_string())),
category: "binary".to_string(),
metadata: HashMap::new(),
}
} else {
TypeJudgment {
mime: "text/plain".to_string(),
universe: "U0".to_string(),
is_binary: false,
confidence: 0.50,
matched_rule: "text_fallback".to_string(),
extension: resolved_ext.or_else(|| Some(".txt".to_string())),
category: "text".to_string(),
metadata: HashMap::new(),
}
}
}
pub fn detect_content_type(&self, data: &[u8], ext_hint: Option<&str>) -> (String, String) {
let judgment = self.judge(Some(data), None, ext_hint);
let clean_ext = judgment.extension.map(|e| e.trim_start_matches('.').to_string()).unwrap_or_default();
(judgment.mime, clean_ext)
}
pub fn is_binary_content(&self, data: &[u8]) -> bool {
self.is_binary_payload(data)
}
fn resolve_extension(&self, filename: Option<&str>, ext_hint: Option<&str>) -> Option<String> {
let mut target = ext_hint.unwrap_or("").trim().to_lowercase();
if target.is_empty()
&& let Some(name) = filename
&& let Some(idx) = name.rfind('.') {
target = name[idx..].trim().to_lowercase();
}
if target.is_empty() {
None
} else if target.starts_with('.') {
Some(target)
} else {
Some(format!(".{}", target))
}
}
fn is_binary_payload(&self, data: &[u8]) -> bool {
if data.is_empty() {
return false;
}
let limit = std::cmp::min(data.len(), 4096);
let sample = &data[..limit];
if sample.contains(&0) {
return true;
}
let non_text = sample
.iter()
.filter(|&&b| b < 9 || (b > 13 && b < 32) || b > 127)
.count();
(non_text as f32 / limit as f32) > 0.30
}
fn probe_structured_text(&self, text: &str) -> Option<(String, String, f32)> {
if ((text.starts_with('{') && text.ends_with('}')) || (text.starts_with('[') && text.ends_with(']')))
&& serde_json::from_str::<serde_json::Value>(text).is_ok() {
if text.contains("\"mcard_version\"") {
return Some(("application/vnd.mcard+json".to_string(), "structured_mcard_probe".to_string(), 0.99));
}
if text.contains("\"pcard_version\"") {
return Some(("application/vnd.pcard+json".to_string(), "structured_pcard_probe".to_string(), 0.99));
}
if text.contains("\"vcard_version\"") {
return Some(("application/vnd.vcard+json".to_string(), "structured_vcard_probe".to_string(), 0.99));
}
return Some(("application/json".to_string(), "structured_json_probe".to_string(), 0.98));
}
if text.starts_with("<?xml") || text.to_lowercase().contains("<svg") {
if text.to_lowercase().contains("<svg") {
return Some(("image/svg+xml".to_string(), "structured_svg_probe".to_string(), 0.96));
}
return Some(("application/xml".to_string(), "structured_xml_probe".to_string(), 0.92));
}
let lower = text.to_lowercase();
if lower.contains("<!doctype html") || lower.contains("<html") {
return Some(("text/html".to_string(), "structured_html_probe".to_string(), 0.95));
}
if text.starts_with("---") && text[3..].contains("\n---") {
return Some(("text/markdown".to_string(), "markdown_frontmatter_probe".to_string(), 0.90));
}
if text.contains("\\documentclass") || text.contains("\\begin{tikzpicture}") || text.contains("\\begin{document}") {
if text.contains("\\begin{tikzpicture}") {
return Some(("text/x-tikz".to_string(), "structured_tikz_probe".to_string(), 0.95));
}
return Some(("text/x-tex".to_string(), "structured_latex_probe".to_string(), 0.92));
}
None
}
}