use crate::layer0::functional::first_match;
use crate::layer0::type_phases as tp;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
pub static CANONICAL_DICTIONARY_JSON: &str = include_str!("../../schema/type_dictionary.json");
#[path = "type_interpreter_magic.rs"]
mod magic;
pub use magic::*;
#[derive(Clone)]
pub struct TypeDefinition {
pub mime: String,
pub extensions: Vec<String>,
pub universe: String,
pub is_binary: bool,
pub category: String,
pub description: String,
pub magic_bytes: Vec<MagicSignature>,
pub text_patterns: Vec<String>,
pub validator: Option<Arc<dyn Fn(&[u8]) -> bool + Send + Sync>>,
pub priority: i32,
pub metadata: HashMap<String, String>,
}
impl Default for TypeDefinition {
fn default() -> Self {
Self {
mime: String::new(),
extensions: Vec::new(),
universe: tp::TYPE_DEFAULT_UNIVERSE.to_string(),
is_binary: false,
category: tp::TYPE_FALLBACK_TEXT_CATEGORY.to_string(),
description: String::new(),
magic_bytes: Vec::new(),
text_patterns: Vec::new(),
validator: None,
priority: 0,
metadata: HashMap::new(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TypeJudgment {
pub mime: String,
pub universe: String,
pub is_binary: bool,
pub confidence: f32,
pub matched_rule: String,
pub extension: Option<String>,
pub category: String,
pub metadata: HashMap<String, String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sub_dialect: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub truncated_signature: Option<bool>,
}
#[derive(Debug, Deserialize)]
struct RawDictionary {
#[serde(default)]
types: std::collections::BTreeMap<String, RawTypeEntry>,
}
#[derive(Debug, Deserialize)]
struct RawTypeEntry {
#[serde(default)]
extensions: Vec<String>,
#[serde(default = "default_universe")]
universe: String,
#[serde(default)]
is_binary: bool,
#[serde(default = "default_category")]
category: String,
#[serde(default)]
description: String,
#[serde(default)]
magic_bytes: Vec<MagicSignature>,
#[serde(default)]
text_patterns: Vec<String>,
}
fn default_universe() -> String { tp::TYPE_DEFAULT_UNIVERSE.to_string() }
fn default_category() -> String { tp::TYPE_FALLBACK_TEXT_CATEGORY.to_string() }
type JudgmentInput<'a> = (&'a [u8], Option<String>, bool, &'a TypeInterpreter);
#[allow(clippy::too_many_arguments)]
fn build_judgment(
mime: String, universe: String, is_binary: bool, confidence: f32, matched_rule: String,
extension: Option<String>, category: String, metadata: HashMap<String, String>,
sub_dialect: Option<String>, truncated_signature: Option<bool>,
) -> TypeJudgment {
TypeJudgment { mime, universe, is_binary, confidence, matched_rule, extension, category, metadata, sub_dialect, truncated_signature }
}
fn phase_high_priority(inp: &JudgmentInput<'_>) -> Option<TypeJudgment> {
let (raw_bytes, resolved_ext, _is_bin, dictionary) = inp;
for def in dictionary.types.values() {
if def.priority > tp::TYPE_MIN_HIGH_PRIORITY_EXCLUSIVE
&& let Some(ref val) = def.validator
&& val(raw_bytes)
{
return Some(build_judgment(
def.mime.clone(), def.universe.clone(), def.is_binary,
tp::TYPE_CONF_HIGH_PRIORITY, tp::TYPE_RULE_HIGH_PRIORITY.to_string(),
def.extensions.first().cloned().or_else(|| resolved_ext.clone()),
def.category.clone(), def.metadata.clone(), None, None,
));
}
}
None
}
fn phase_magic_and_brand(inp: &JudgmentInput<'_>) -> Option<TypeJudgment> {
let (raw_bytes, resolved_ext, _is_bin, dictionary) = inp;
if let Some(brand) = inspect_container_brand(raw_bytes) {
let def = dictionary.types.get(&brand.mime);
let mut metadata = HashMap::new();
if let Some(major) = &brand.major_brand {
metadata.insert("majorBrand".to_string(), major.clone());
}
if !brand.compatible_brands.is_empty() {
metadata.insert("compatibleBrands".to_string(), brand.compatible_brands.join(","));
}
if let (Some(ext), Some(expected)) = (
resolved_ext.as_ref(),
resolved_ext.as_ref().and_then(|value| dictionary.ext_map.get(value)),
) && expected != &brand.mime
{
metadata.insert("extensionConflict".to_string(), format!("{ext}|{expected}|{}", brand.mime));
}
return Some(build_judgment(
brand.mime,
def.map(|e| e.universe.clone()).unwrap_or_else(|| tp::TYPE_DEFAULT_UNIVERSE.to_string()),
def.map(|e| e.is_binary).unwrap_or(true),
tp::TYPE_CONF_MAGIC_BRAND, tp::TYPE_RULE_MAGIC_BYTES.to_string(),
def.and_then(|e| e.extensions.first().cloned()).or_else(|| resolved_ext.clone()),
def.map(|e| e.category.clone()).unwrap_or_else(|| tp::TYPE_BRAND_DEFAULT_CATEGORY.to_string()),
metadata, brand.sub_dialect, None,
));
}
if raw_bytes.is_empty() {
return None;
}
let mut matching: Vec<(&MagicSignature, &TypeDefinition)> = Vec::new();
for (sig, mime) in &dictionary.magic_signatures {
if sig.matches(raw_bytes) && let Some(def) = dictionary.types.get(mime) {
matching.push((sig, def));
}
}
if matching.is_empty() {
return None;
}
let mut best = matching[0];
if let Some(ext) = resolved_ext {
if let Some(ext_match) = matching.iter().find(|(_, def)| def.extensions.contains(ext)) {
best = *ext_match;
}
} else if let Some(base_match) = matching.iter().find(|(_, def)| !def.mime.contains('+')) {
best = *base_match;
}
let (best_sig, best_def) = best;
let mut meta = HashMap::new();
meta.insert("signature_hex".to_string(), best_sig.hex.clone());
Some(build_judgment(
best_def.mime.clone(), best_def.universe.clone(), best_def.is_binary,
tp::TYPE_CONF_MAGIC_SIGNATURE, tp::TYPE_RULE_MAGIC_BYTES.to_string(),
best_def.extensions.first().cloned().or_else(|| resolved_ext.clone()),
best_def.category.clone(), meta, None, None,
))
}
fn phase_extension(inp: &JudgmentInput<'_>) -> Option<TypeJudgment> {
let (_raw_bytes, resolved_ext, _is_bin, dictionary) = inp;
let ext = resolved_ext.as_ref()?;
let mime = dictionary.ext_map.get(ext)?;
let def = dictionary.types.get(mime)?;
Some(build_judgment(
def.mime.clone(), def.universe.clone(), def.is_binary, tp::TYPE_CONF_EXTENSION_HINT,
tp::TYPE_RULE_EXTENSION_HINT.to_string(), Some(ext.clone()), def.category.clone(),
HashMap::new(), None, None,
))
}
fn phase_truncated_magic(inp: &JudgmentInput<'_>) -> Option<TypeJudgment> {
let (raw_bytes, resolved_ext, _is_bin, dictionary) = inp;
let truncated = detect_truncated_signature(raw_bytes, &dictionary.magic_signatures, |mime| {
dictionary.types.get(mime).is_some_and(|definition| definition.is_binary)
})?;
let mut metadata = HashMap::new();
metadata.insert("truncatedSignature".to_string(), "true".to_string());
metadata.insert("candidateMimes".to_string(), truncated.candidate_mimes.join(","));
metadata.insert("matchedBytes".to_string(), truncated.matched_bytes.to_string());
Some(build_judgment(
crate::layer0::vocabulary_types::MIME_APPLICATION_OCTET_STREAM.to_string(),
tp::TYPE_DEFAULT_UNIVERSE.to_string(), true, tp::TYPE_CONF_TRUNCATED_MAGIC,
tp::TYPE_RULE_TRUNCATED_MAGIC.to_string(),
resolved_ext.clone().or_else(|| Some(tp::TYPE_FALLBACK_BINARY_EXT.to_string())),
tp::TYPE_FALLBACK_BINARY_CATEGORY.to_string(), metadata, None, Some(true),
))
}
fn phase_structured_and_text_pattern(inp: &JudgmentInput<'_>) -> Option<TypeJudgment> {
let (raw_bytes, resolved_ext, is_bin, dictionary) = inp;
if *is_bin || raw_bytes.is_empty() {
return None;
}
let trimmed = std::str::from_utf8(raw_bytes).ok()?.trim();
if let Some((mime, rule, conf)) = probe_structured_text(trimmed) {
let def_opt = dictionary.types.get(&mime);
return Some(build_judgment(
mime, def_opt.map(|d| d.universe.clone()).unwrap_or_else(|| tp::TYPE_DEFAULT_UNIVERSE.to_string()),
false, conf, rule,
def_opt.and_then(|d| d.extensions.first().cloned()).or_else(|| resolved_ext.clone()),
def_opt.map(|d| d.category.clone()).unwrap_or_else(|| tp::TYPE_FALLBACK_TEXT_CATEGORY.to_string()),
HashMap::new(), None, None,
));
}
for (re, mime) in &dictionary.compiled_patterns {
if re.is_match(trimmed) && let Some(def) = dictionary.types.get(mime) {
return Some(build_judgment(
def.mime.clone(), def.universe.clone(), false, tp::TYPE_CONF_TEXT_PATTERN,
tp::TYPE_RULE_TEXT_PATTERN.to_string(),
def.extensions.first().cloned().or_else(|| resolved_ext.clone()),
def.category.clone(), HashMap::new(), None, None,
));
}
}
None
}
fn phase_standard_predicate_and_fallback(inp: &JudgmentInput<'_>) -> Option<TypeJudgment> {
let (raw_bytes, resolved_ext, is_bin, dictionary) = inp;
for def in dictionary.types.values() {
if def.priority == tp::TYPE_MIN_HIGH_PRIORITY_EXCLUSIVE
&& let Some(ref val) = def.validator
&& val(raw_bytes)
{
return Some(build_judgment(
def.mime.clone(), def.universe.clone(), def.is_binary,
tp::TYPE_CONF_STANDARD_PREDICATE, tp::TYPE_RULE_STANDARD_PREDICATE.to_string(),
def.extensions.first().cloned().or_else(|| resolved_ext.clone()),
def.category.clone(), def.metadata.clone(), None, None,
));
}
}
if *is_bin {
return Some(build_judgment(
crate::layer0::vocabulary_types::MIME_APPLICATION_OCTET_STREAM.to_string(),
tp::TYPE_DEFAULT_UNIVERSE.to_string(), true, tp::TYPE_CONF_FALLBACK,
tp::TYPE_RULE_BINARY_FALLBACK.to_string(),
resolved_ext.clone().or_else(|| Some(tp::TYPE_FALLBACK_BINARY_EXT.to_string())),
tp::TYPE_FALLBACK_BINARY_CATEGORY.to_string(), HashMap::new(), None, None,
));
}
Some(build_judgment(
crate::layer0::vocabulary_types::MIME_TEXT_PLAIN.to_string(),
tp::TYPE_DEFAULT_UNIVERSE.to_string(), false, tp::TYPE_CONF_FALLBACK,
tp::TYPE_RULE_TEXT_FALLBACK.to_string(),
resolved_ext.clone().or_else(|| Some(tp::TYPE_FALLBACK_TEXT_EXT.to_string())),
tp::TYPE_FALLBACK_TEXT_CATEGORY.to_string(), HashMap::new(), None, None,
))
}
pub struct TypeInterpreter {
types: HashMap<String, TypeDefinition>,
ext_map: HashMap<String, String>,
magic_signatures: Vec<(MagicSignature, String)>,
compiled_patterns: Vec<(regex::Regex, String)>,
}
impl Default for TypeInterpreter {
fn default() -> Self {
Self::new()
}
}
impl TypeInterpreter {
pub fn new() -> Self {
Self {
types: HashMap::new(),
ext_map: HashMap::new(),
magic_signatures: Vec::new(),
compiled_patterns: Vec::new(),
}
}
pub fn create_default() -> Self {
let mut interpreter = Self::new();
if let Ok(dict) = serde_json::from_str::<RawDictionary>(CANONICAL_DICTIONARY_JSON) {
for (mime, raw) in dict.types {
let def = TypeDefinition {
mime: mime.clone(),
extensions: raw.extensions,
universe: raw.universe,
is_binary: raw.is_binary,
category: raw.category,
description: raw.description,
magic_bytes: raw.magic_bytes,
text_patterns: raw.text_patterns,
validator: None,
priority: 0,
metadata: HashMap::new(),
};
interpreter.register_type(def);
}
}
interpreter
}
pub fn register_type(&mut self, mut type_def: TypeDefinition) {
let mut norm_exts = Vec::new();
for ext in &type_def.extensions {
let clean = ext.trim().to_lowercase();
let standardized = if clean.starts_with('.') {
clean
} else {
format!(".{}", clean)
};
if !norm_exts.contains(&standardized) {
norm_exts.push(standardized);
}
}
type_def.extensions = norm_exts;
for ext in &type_def.extensions {
self.ext_map.insert(ext.clone(), type_def.mime.clone());
}
for sig in &type_def.magic_bytes {
self.magic_signatures.push((sig.clone(), type_def.mime.clone()));
}
self.magic_signatures.sort_by_key(|a| std::cmp::Reverse(a.0.byte_length()));
for pat in &type_def.text_patterns {
if let Ok(re) = regex::RegexBuilder::new(pat).multi_line(true).build() {
self.compiled_patterns.push((re, type_def.mime.clone()));
}
}
self.types.insert(type_def.mime.clone(), type_def);
}
pub fn get_type(&self, mime: &str) -> Option<&TypeDefinition> {
self.types.get(mime)
}
pub fn types_count(&self) -> usize {
self.types.len()
}
pub fn judge(
&self,
data: Option<&[u8]>,
filename: Option<&str>,
ext_hint: Option<&str>,
) -> TypeJudgment {
let raw_bytes = data.unwrap_or(&[]);
let resolved_ext = resolve_extension(filename, ext_hint, Some(&self.ext_map));
let is_bin = is_binary_payload(raw_bytes);
let phases = tp::build_type_phase_pipeline::<JudgmentInput<'_>, TypeJudgment>(
phase_high_priority,
phase_magic_and_brand,
phase_extension,
phase_truncated_magic,
phase_structured_and_text_pattern,
phase_standard_predicate_and_fallback,
);
let input: JudgmentInput<'_> = (raw_bytes, resolved_ext, is_bin, self);
first_match(&phases, &input).expect("standard_predicate_and_fallback guarantees terminal judgment")
}
pub fn detect_content_type(&self, data: &[u8], ext_hint: Option<&str>) -> (String, String) {
let judgment = self.judge(Some(data), None, ext_hint);
let clean_ext = judgment
.extension
.map(|e| e.trim_start_matches('.').to_string())
.unwrap_or_default();
(judgment.mime, clean_ext)
}
pub fn is_binary_content(&self, data: &[u8]) -> bool {
let binary_signature = self.magic_signatures.iter().any(|(sig, mime)| {
sig.matches(data) && self.types.get(mime).is_some_and(|def| def.is_binary)
});
binary_signature || is_binary_payload(data)
}
}