use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
pub static CANONICAL_DICTIONARY_JSON: &str = include_str!("../../schema/type_dictionary.json");
#[path = "type_interpreter_magic.rs"]
mod magic;
pub use magic::*;
#[derive(Clone)]
pub struct TypeDefinition {
pub mime: String,
pub extensions: Vec<String>,
pub universe: String,
pub is_binary: bool,
pub category: String,
pub description: String,
pub magic_bytes: Vec<MagicSignature>,
pub text_patterns: Vec<String>,
pub validator: Option<Arc<dyn Fn(&[u8]) -> bool + Send + Sync>>,
pub priority: i32,
pub metadata: HashMap<String, String>,
}
impl Default for TypeDefinition {
fn default() -> Self {
Self {
mime: String::new(),
extensions: Vec::new(),
universe: "U0".to_string(),
is_binary: false,
category: "text".to_string(),
description: String::new(),
magic_bytes: Vec::new(),
text_patterns: Vec::new(),
validator: None,
priority: 0,
metadata: HashMap::new(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TypeJudgment {
pub mime: String,
pub universe: String,
pub is_binary: bool,
pub confidence: f32,
pub matched_rule: String,
pub extension: Option<String>,
pub category: String,
pub metadata: HashMap<String, String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sub_dialect: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub truncated_signature: Option<bool>,
}
#[derive(Debug, Deserialize)]
struct RawDictionary {
#[serde(default)]
types: std::collections::BTreeMap<String, RawTypeEntry>,
}
#[derive(Debug, Deserialize)]
struct RawTypeEntry {
#[serde(default)]
extensions: Vec<String>,
#[serde(default = "default_universe")]
universe: String,
#[serde(default)]
is_binary: bool,
#[serde(default = "default_category")]
category: String,
#[serde(default)]
description: String,
#[serde(default)]
magic_bytes: Vec<MagicSignature>,
#[serde(default)]
text_patterns: Vec<String>,
}
fn default_universe() -> String {
"U0".to_string()
}
fn default_category() -> String {
"text".to_string()
}
pub struct TypeInterpreter {
types: HashMap<String, TypeDefinition>,
ext_map: HashMap<String, String>,
magic_signatures: Vec<(MagicSignature, String)>,
compiled_patterns: Vec<(regex::Regex, String)>,
}
impl Default for TypeInterpreter {
fn default() -> Self {
Self::new()
}
}
impl TypeInterpreter {
pub fn new() -> Self {
Self {
types: HashMap::new(),
ext_map: HashMap::new(),
magic_signatures: Vec::new(),
compiled_patterns: Vec::new(),
}
}
pub fn create_default() -> Self {
let mut interpreter = Self::new();
if let Ok(dict) = serde_json::from_str::<RawDictionary>(CANONICAL_DICTIONARY_JSON) {
for (mime, raw) in dict.types {
let def = TypeDefinition {
mime: mime.clone(),
extensions: raw.extensions,
universe: raw.universe,
is_binary: raw.is_binary,
category: raw.category,
description: raw.description,
magic_bytes: raw.magic_bytes,
text_patterns: raw.text_patterns,
validator: None,
priority: 0,
metadata: HashMap::new(),
};
interpreter.register_type(def);
}
}
interpreter
}
pub fn register_type(&mut self, mut type_def: TypeDefinition) {
let mut norm_exts = Vec::new();
for ext in &type_def.extensions {
let clean = ext.trim().to_lowercase();
let standardized = if clean.starts_with('.') {
clean
} else {
format!(".{}", clean)
};
if !norm_exts.contains(&standardized) {
norm_exts.push(standardized);
}
}
type_def.extensions = norm_exts;
for ext in &type_def.extensions {
self.ext_map.insert(ext.clone(), type_def.mime.clone());
}
for sig in &type_def.magic_bytes {
self.magic_signatures.push((sig.clone(), type_def.mime.clone()));
}
self.magic_signatures.sort_by_key(|a| std::cmp::Reverse(a.0.byte_length()));
for pat in &type_def.text_patterns {
if let Ok(re) = regex::Regex::new(pat) {
self.compiled_patterns.push((re, type_def.mime.clone()));
}
}
self.types.insert(type_def.mime.clone(), type_def);
}
pub fn get_type(&self, mime: &str) -> Option<&TypeDefinition> {
self.types.get(mime)
}
pub fn types_count(&self) -> usize {
self.types.len()
}
pub fn judge(
&self,
data: Option<&[u8]>,
filename: Option<&str>,
ext_hint: Option<&str>,
) -> TypeJudgment {
let raw_bytes = data.unwrap_or(&[]);
let resolved_ext = resolve_extension(filename, ext_hint);
for def in self.types.values() {
if def.priority > 0
&& let Some(ref val) = def.validator
&& val(raw_bytes) {
return TypeJudgment {
mime: def.mime.clone(),
universe: def.universe.clone(),
is_binary: def.is_binary,
confidence: 1.0,
matched_rule: "programmatic_predicate_high_priority".to_string(),
extension: def.extensions.first().cloned().or(resolved_ext),
category: def.category.clone(),
metadata: def.metadata.clone(),
sub_dialect: None,
truncated_signature: None,
};
}
}
if let Some(brand) = inspect_container_brand(raw_bytes) {
let def = self.types.get(&brand.mime);
let mut metadata = HashMap::new();
if let Some(major) = &brand.major_brand {
metadata.insert("majorBrand".to_string(), major.clone());
}
if !brand.compatible_brands.is_empty() {
metadata.insert("compatibleBrands".to_string(), brand.compatible_brands.join(","));
}
if let (Some(ext), Some(expected)) = (&resolved_ext, resolved_ext.as_ref().and_then(|value| self.ext_map.get(value))) {
if expected != &brand.mime {
metadata.insert("extensionConflict".to_string(), format!("{ext}|{expected}|{}", brand.mime));
}
}
return TypeJudgment {
mime: brand.mime,
universe: def.map(|entry| entry.universe.clone()).unwrap_or_else(|| "U0".to_string()),
is_binary: def.map(|entry| entry.is_binary).unwrap_or(true),
confidence: 0.99,
matched_rule: "magic_bytes".to_string(),
extension: def.and_then(|entry| entry.extensions.first().cloned()).or(resolved_ext),
category: def.map(|entry| entry.category.clone()).unwrap_or_else(|| "media".to_string()),
metadata,
sub_dialect: brand.sub_dialect,
truncated_signature: None,
};
}
if !raw_bytes.is_empty() {
let mut matching: Vec<(&MagicSignature, &TypeDefinition)> = Vec::new();
for (sig, mime) in &self.magic_signatures {
if sig.matches(raw_bytes)
&& let Some(def) = self.types.get(mime) {
matching.push((sig, def));
}
}
if !matching.is_empty() {
let mut best = matching[0];
if let Some(ref ext) = resolved_ext {
if let Some(ext_match) = matching.iter().find(|(_, def)| def.extensions.contains(ext)) {
best = *ext_match;
}
} else if let Some(base_match) = matching.iter().find(|(_, def)| !def.mime.contains('+')) {
best = *base_match;
}
let (best_sig, best_def) = best;
let mut meta = HashMap::new();
meta.insert("signature_hex".to_string(), best_sig.hex.clone());
return TypeJudgment {
mime: best_def.mime.clone(),
universe: best_def.universe.clone(),
is_binary: best_def.is_binary,
confidence: 0.99,
matched_rule: "magic_bytes".to_string(),
extension: best_def.extensions.first().cloned().or(resolved_ext),
category: best_def.category.clone(),
metadata: meta,
sub_dialect: None,
truncated_signature: None,
};
}
}
let is_bin = is_binary_payload(raw_bytes);
if let Some(ref ext) = resolved_ext
&& let Some(mime) = self.ext_map.get(ext)
&& let Some(def) = self.types.get(mime) {
return TypeJudgment {
mime: def.mime.clone(),
universe: def.universe.clone(),
is_binary: def.is_binary,
confidence: 0.95,
matched_rule: "extension_hint".to_string(),
extension: Some(ext.clone()),
category: def.category.clone(),
metadata: HashMap::new(),
sub_dialect: None,
truncated_signature: None,
};
}
if let Some(truncated) = detect_truncated_signature(
raw_bytes,
&self.magic_signatures,
|mime| self.types.get(mime).is_some_and(|definition| definition.is_binary),
) {
let mut metadata = HashMap::new();
metadata.insert("truncatedSignature".to_string(), "true".to_string());
metadata.insert("candidateMimes".to_string(), truncated.candidate_mimes.join(","));
metadata.insert("matchedBytes".to_string(), truncated.matched_bytes.to_string());
return TypeJudgment {
mime: "application/octet-stream".to_string(),
universe: "U0".to_string(),
is_binary: true,
confidence: 0.50,
matched_rule: "binary_fallback".to_string(),
extension: resolved_ext.or_else(|| Some(".bin".to_string())),
category: "binary".to_string(),
metadata,
sub_dialect: None,
truncated_signature: Some(true),
};
}
if !is_bin && !raw_bytes.is_empty()
&& let Ok(text) = std::str::from_utf8(raw_bytes) {
let trimmed = text.trim();
if let Some((mime, rule, conf)) = probe_structured_text(trimmed) {
let def_opt = self.types.get(&mime);
return TypeJudgment {
mime: mime.clone(),
universe: def_opt.map(|d| d.universe.clone()).unwrap_or_else(|| "U0".to_string()),
is_binary: false,
confidence: conf,
matched_rule: rule,
extension: def_opt.and_then(|d| d.extensions.first().cloned()).or(resolved_ext),
category: def_opt.map(|d| d.category.clone()).unwrap_or_else(|| "text".to_string()),
metadata: HashMap::new(),
sub_dialect: None,
truncated_signature: None,
};
}
for (re, mime) in &self.compiled_patterns {
if re.is_match(trimmed)
&& let Some(def) = self.types.get(mime) {
return TypeJudgment {
mime: def.mime.clone(),
universe: def.universe.clone(),
is_binary: false,
confidence: 0.95,
matched_rule: "text_pattern".to_string(),
extension: def.extensions.first().cloned().or(resolved_ext),
category: def.category.clone(),
metadata: HashMap::new(),
sub_dialect: None,
truncated_signature: None,
};
}
}
}
for def in self.types.values() {
if def.priority == 0
&& let Some(ref val) = def.validator
&& val(raw_bytes) {
return TypeJudgment {
mime: def.mime.clone(),
universe: def.universe.clone(),
is_binary: def.is_binary,
confidence: 0.92,
matched_rule: "programmatic_predicate".to_string(),
extension: def.extensions.first().cloned().or(resolved_ext),
category: def.category.clone(),
metadata: def.metadata.clone(),
sub_dialect: None,
truncated_signature: None,
};
}
}
if is_bin {
TypeJudgment {
mime: "application/octet-stream".to_string(),
universe: "U0".to_string(),
is_binary: true,
confidence: 0.50,
matched_rule: "binary_fallback".to_string(),
extension: resolved_ext.or_else(|| Some(".bin".to_string())),
category: "binary".to_string(),
metadata: HashMap::new(),
sub_dialect: None,
truncated_signature: None,
}
} else {
TypeJudgment {
mime: "text/plain".to_string(),
universe: "U0".to_string(),
is_binary: false,
confidence: 0.50,
matched_rule: "text_fallback".to_string(),
extension: resolved_ext.or_else(|| Some(".txt".to_string())),
category: "text".to_string(),
metadata: HashMap::new(),
sub_dialect: None,
truncated_signature: None,
}
}
}
pub fn detect_content_type(&self, data: &[u8], ext_hint: Option<&str>) -> (String, String) {
let judgment = self.judge(Some(data), None, ext_hint);
let clean_ext = judgment.extension.map(|e| e.trim_start_matches('.').to_string()).unwrap_or_default();
(judgment.mime, clean_ext)
}
pub fn is_binary_content(&self, data: &[u8]) -> bool {
is_binary_payload(data)
}
}