clm-kernel 0.0.24

Cubical Logic Model (CLM) sovereign execution engine, CPN-VM, and Cordis spatiotemporal substrate
Documentation
//! # Universal TypeInterpreter & Programmatic Judgment Engine (INV-447, INV-628)

use crate::layer0::functional::first_match;
use crate::layer0::type_phases as tp;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;

/// Raw canonical dictionary embedded at compile time for zero-IO, self-contained crates.io distribution.
pub static CANONICAL_DICTIONARY_JSON: &str = include_str!("../../schema/type_dictionary.json");

#[path = "type_interpreter_magic.rs"]
mod magic;
pub use magic::*;

/// Declarative and programmatic type specification in Rust.
#[derive(Clone)]
pub struct TypeDefinition {
    pub mime: String,
    pub extensions: Vec<String>,
    pub universe: String,
    pub is_binary: bool,
    pub category: String,
    pub description: String,
    pub magic_bytes: Vec<MagicSignature>,
    pub text_patterns: Vec<String>,
    pub validator: Option<Arc<dyn Fn(&[u8]) -> bool + Send + Sync>>,
    pub priority: i32,
    pub metadata: HashMap<String, String>,
}

impl Default for TypeDefinition {
    fn default() -> Self {
        Self {
            mime: String::new(),
            extensions: Vec::new(),
            universe: tp::TYPE_DEFAULT_UNIVERSE.to_string(),
            is_binary: false,
            category: tp::TYPE_FALLBACK_TEXT_CATEGORY.to_string(),
            description: String::new(),
            magic_bytes: Vec::new(),
            text_patterns: Vec::new(),
            validator: None,
            priority: 0,
            metadata: HashMap::new(),
        }
    }
}

/// Deterministic result of evaluating content and metadata against the TypeInterpreter.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TypeJudgment {
    pub mime: String,
    pub universe: String,
    pub is_binary: bool,
    pub confidence: f32,
    pub matched_rule: String,
    pub extension: Option<String>,
    pub category: String,
    pub metadata: HashMap<String, String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub sub_dialect: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub truncated_signature: Option<bool>,
}

#[derive(Debug, Deserialize)]
struct RawDictionary {
    #[serde(default)]
    types: std::collections::BTreeMap<String, RawTypeEntry>,
}

#[derive(Debug, Deserialize)]
struct RawTypeEntry {
    #[serde(default)]
    extensions: Vec<String>,
    #[serde(default = "default_universe")]
    universe: String,
    #[serde(default)]
    is_binary: bool,
    #[serde(default = "default_category")]
    category: String,
    #[serde(default)]
    description: String,
    #[serde(default)]
    magic_bytes: Vec<MagicSignature>,
    #[serde(default)]
    text_patterns: Vec<String>,
}

fn default_universe() -> String { tp::TYPE_DEFAULT_UNIVERSE.to_string() }
fn default_category() -> String { tp::TYPE_FALLBACK_TEXT_CATEGORY.to_string() }

type JudgmentInput<'a> = (&'a [u8], Option<String>, bool, &'a TypeInterpreter);

#[allow(clippy::too_many_arguments)]
fn build_judgment(
    mime: String, universe: String, is_binary: bool, confidence: f32, matched_rule: String,
    extension: Option<String>, category: String, metadata: HashMap<String, String>,
    sub_dialect: Option<String>, truncated_signature: Option<bool>,
) -> TypeJudgment {
    TypeJudgment { mime, universe, is_binary, confidence, matched_rule, extension, category, metadata, sub_dialect, truncated_signature }
}

fn phase_high_priority(inp: &JudgmentInput<'_>) -> Option<TypeJudgment> {
    let (raw_bytes, resolved_ext, _is_bin, dictionary) = inp;
    for def in dictionary.types.values() {
        if def.priority > tp::TYPE_MIN_HIGH_PRIORITY_EXCLUSIVE
            && let Some(ref val) = def.validator
            && val(raw_bytes)
        {
            return Some(build_judgment(
                def.mime.clone(), def.universe.clone(), def.is_binary,
                tp::TYPE_CONF_HIGH_PRIORITY, tp::TYPE_RULE_HIGH_PRIORITY.to_string(),
                def.extensions.first().cloned().or_else(|| resolved_ext.clone()),
                def.category.clone(), def.metadata.clone(), None, None,
            ));
        }
    }
    None
}

fn phase_magic_and_brand(inp: &JudgmentInput<'_>) -> Option<TypeJudgment> {
    let (raw_bytes, resolved_ext, _is_bin, dictionary) = inp;
    if let Some(brand) = inspect_container_brand(raw_bytes) {
        let def = dictionary.types.get(&brand.mime);
        let mut metadata = HashMap::new();
        if let Some(major) = &brand.major_brand {
            metadata.insert("majorBrand".to_string(), major.clone());
        }
        if !brand.compatible_brands.is_empty() {
            metadata.insert("compatibleBrands".to_string(), brand.compatible_brands.join(","));
        }
        if let (Some(ext), Some(expected)) = (
            resolved_ext.as_ref(),
            resolved_ext.as_ref().and_then(|value| dictionary.ext_map.get(value)),
        ) && expected != &brand.mime
        {
            metadata.insert("extensionConflict".to_string(), format!("{ext}|{expected}|{}", brand.mime));
        }
        return Some(build_judgment(
            brand.mime,
            def.map(|e| e.universe.clone()).unwrap_or_else(|| tp::TYPE_DEFAULT_UNIVERSE.to_string()),
            def.map(|e| e.is_binary).unwrap_or(true),
            tp::TYPE_CONF_MAGIC_BRAND, tp::TYPE_RULE_MAGIC_BYTES.to_string(),
            def.and_then(|e| e.extensions.first().cloned()).or_else(|| resolved_ext.clone()),
            def.map(|e| e.category.clone()).unwrap_or_else(|| tp::TYPE_BRAND_DEFAULT_CATEGORY.to_string()),
            metadata, brand.sub_dialect, None,
        ));
    }
    if raw_bytes.is_empty() {
        return None;
    }
    let mut matching: Vec<(&MagicSignature, &TypeDefinition)> = Vec::new();
    for (sig, mime) in &dictionary.magic_signatures {
        if sig.matches(raw_bytes) && let Some(def) = dictionary.types.get(mime) {
            matching.push((sig, def));
        }
    }
    if matching.is_empty() {
        return None;
    }
    let mut best = matching[0];
    if let Some(ext) = resolved_ext {
        if let Some(ext_match) = matching.iter().find(|(_, def)| def.extensions.contains(ext)) {
            best = *ext_match;
        }
    } else if let Some(base_match) = matching.iter().find(|(_, def)| !def.mime.contains('+')) {
        best = *base_match;
    }
    let (best_sig, best_def) = best;
    let mut meta = HashMap::new();
    meta.insert("signature_hex".to_string(), best_sig.hex.clone());
    Some(build_judgment(
        best_def.mime.clone(), best_def.universe.clone(), best_def.is_binary,
        tp::TYPE_CONF_MAGIC_SIGNATURE, tp::TYPE_RULE_MAGIC_BYTES.to_string(),
        best_def.extensions.first().cloned().or_else(|| resolved_ext.clone()),
        best_def.category.clone(), meta, None, None,
    ))
}

fn phase_extension(inp: &JudgmentInput<'_>) -> Option<TypeJudgment> {
    let (_raw_bytes, resolved_ext, _is_bin, dictionary) = inp;
    let ext = resolved_ext.as_ref()?;
    let mime = dictionary.ext_map.get(ext)?;
    let def = dictionary.types.get(mime)?;
    Some(build_judgment(
        def.mime.clone(), def.universe.clone(), def.is_binary, tp::TYPE_CONF_EXTENSION_HINT,
        tp::TYPE_RULE_EXTENSION_HINT.to_string(), Some(ext.clone()), def.category.clone(),
        HashMap::new(), None, None,
    ))
}

fn phase_truncated_magic(inp: &JudgmentInput<'_>) -> Option<TypeJudgment> {
    let (raw_bytes, resolved_ext, _is_bin, dictionary) = inp;
    let truncated = detect_truncated_signature(raw_bytes, &dictionary.magic_signatures, |mime| {
        dictionary.types.get(mime).is_some_and(|definition| definition.is_binary)
    })?;
    let mut metadata = HashMap::new();
    metadata.insert("truncatedSignature".to_string(), "true".to_string());
    metadata.insert("candidateMimes".to_string(), truncated.candidate_mimes.join(","));
    metadata.insert("matchedBytes".to_string(), truncated.matched_bytes.to_string());
    Some(build_judgment(
        crate::layer0::vocabulary_types::MIME_APPLICATION_OCTET_STREAM.to_string(),
        tp::TYPE_DEFAULT_UNIVERSE.to_string(), true, tp::TYPE_CONF_TRUNCATED_MAGIC,
        tp::TYPE_RULE_TRUNCATED_MAGIC.to_string(),
        resolved_ext.clone().or_else(|| Some(tp::TYPE_FALLBACK_BINARY_EXT.to_string())),
        tp::TYPE_FALLBACK_BINARY_CATEGORY.to_string(), metadata, None, Some(true),
    ))
}

fn phase_structured_and_text_pattern(inp: &JudgmentInput<'_>) -> Option<TypeJudgment> {
    let (raw_bytes, resolved_ext, is_bin, dictionary) = inp;
    if *is_bin || raw_bytes.is_empty() {
        return None;
    }
    let trimmed = std::str::from_utf8(raw_bytes).ok()?.trim();
    if let Some((mime, rule, conf)) = probe_structured_text(trimmed) {
        let def_opt = dictionary.types.get(&mime);
        return Some(build_judgment(
            mime, def_opt.map(|d| d.universe.clone()).unwrap_or_else(|| tp::TYPE_DEFAULT_UNIVERSE.to_string()),
            false, conf, rule,
            def_opt.and_then(|d| d.extensions.first().cloned()).or_else(|| resolved_ext.clone()),
            def_opt.map(|d| d.category.clone()).unwrap_or_else(|| tp::TYPE_FALLBACK_TEXT_CATEGORY.to_string()),
            HashMap::new(), None, None,
        ));
    }
    for (re, mime) in &dictionary.compiled_patterns {
        if re.is_match(trimmed) && let Some(def) = dictionary.types.get(mime) {
            return Some(build_judgment(
                def.mime.clone(), def.universe.clone(), false, tp::TYPE_CONF_TEXT_PATTERN,
                tp::TYPE_RULE_TEXT_PATTERN.to_string(),
                def.extensions.first().cloned().or_else(|| resolved_ext.clone()),
                def.category.clone(), HashMap::new(), None, None,
            ));
        }
    }
    None
}

fn phase_standard_predicate_and_fallback(inp: &JudgmentInput<'_>) -> Option<TypeJudgment> {
    let (raw_bytes, resolved_ext, is_bin, dictionary) = inp;
    for def in dictionary.types.values() {
        if def.priority == tp::TYPE_MIN_HIGH_PRIORITY_EXCLUSIVE
            && let Some(ref val) = def.validator
            && val(raw_bytes)
        {
            return Some(build_judgment(
                def.mime.clone(), def.universe.clone(), def.is_binary,
                tp::TYPE_CONF_STANDARD_PREDICATE, tp::TYPE_RULE_STANDARD_PREDICATE.to_string(),
                def.extensions.first().cloned().or_else(|| resolved_ext.clone()),
                def.category.clone(), def.metadata.clone(), None, None,
            ));
        }
    }
    if *is_bin {
        return Some(build_judgment(
            crate::layer0::vocabulary_types::MIME_APPLICATION_OCTET_STREAM.to_string(),
            tp::TYPE_DEFAULT_UNIVERSE.to_string(), true, tp::TYPE_CONF_FALLBACK,
            tp::TYPE_RULE_BINARY_FALLBACK.to_string(),
            resolved_ext.clone().or_else(|| Some(tp::TYPE_FALLBACK_BINARY_EXT.to_string())),
            tp::TYPE_FALLBACK_BINARY_CATEGORY.to_string(), HashMap::new(), None, None,
        ));
    }
    Some(build_judgment(
        crate::layer0::vocabulary_types::MIME_TEXT_PLAIN.to_string(),
        tp::TYPE_DEFAULT_UNIVERSE.to_string(), false, tp::TYPE_CONF_FALLBACK,
        tp::TYPE_RULE_TEXT_FALLBACK.to_string(),
        resolved_ext.clone().or_else(|| Some(tp::TYPE_FALLBACK_TEXT_EXT.to_string())),
        tp::TYPE_FALLBACK_TEXT_CATEGORY.to_string(), HashMap::new(), None, None,
    ))
}

/// Universal Native TypeInterpreter.
pub struct TypeInterpreter {
    types: HashMap<String, TypeDefinition>,
    ext_map: HashMap<String, String>,
    magic_signatures: Vec<(MagicSignature, String)>,
    compiled_patterns: Vec<(regex::Regex, String)>,
}

impl Default for TypeInterpreter {
    fn default() -> Self {
        Self::new()
    }
}

impl TypeInterpreter {
    pub fn new() -> Self {
        Self {
            types: HashMap::new(),
            ext_map: HashMap::new(),
            magic_signatures: Vec::new(),
            compiled_patterns: Vec::new(),
        }
    }

    /// Instantiate a TypeInterpreter pre-loaded with the canonical SSOT type dictionary.
    pub fn create_default() -> Self {
        let mut interpreter = Self::new();
        if let Ok(dict) = serde_json::from_str::<RawDictionary>(CANONICAL_DICTIONARY_JSON) {
            for (mime, raw) in dict.types {
                let def = TypeDefinition {
                    mime: mime.clone(),
                    extensions: raw.extensions,
                    universe: raw.universe,
                    is_binary: raw.is_binary,
                    category: raw.category,
                    description: raw.description,
                    magic_bytes: raw.magic_bytes,
                    text_patterns: raw.text_patterns,
                    validator: None,
                    priority: 0,
                    metadata: HashMap::new(),
                };
                interpreter.register_type(def);
            }
        }
        interpreter
    }

    /// Programmatically register or override a TypeDefinition.
    pub fn register_type(&mut self, mut type_def: TypeDefinition) {
        let mut norm_exts = Vec::new();
        for ext in &type_def.extensions {
            let clean = ext.trim().to_lowercase();
            let standardized = if clean.starts_with('.') {
                clean
            } else {
                format!(".{}", clean)
            };
            if !norm_exts.contains(&standardized) {
                norm_exts.push(standardized);
            }
        }
        type_def.extensions = norm_exts;

        for ext in &type_def.extensions {
            self.ext_map.insert(ext.clone(), type_def.mime.clone());
        }

        for sig in &type_def.magic_bytes {
            self.magic_signatures.push((sig.clone(), type_def.mime.clone()));
        }

        self.magic_signatures.sort_by_key(|a| std::cmp::Reverse(a.0.byte_length()));

        for pat in &type_def.text_patterns {
            if let Ok(re) = regex::RegexBuilder::new(pat).multi_line(true).build() {
                self.compiled_patterns.push((re, type_def.mime.clone()));
            }
        }

        self.types.insert(type_def.mime.clone(), type_def);
    }

    pub fn get_type(&self, mime: &str) -> Option<&TypeDefinition> {
        self.types.get(mime)
    }

    pub fn types_count(&self) -> usize {
        self.types.len()
    }

    /// Execute the generated 7-step deterministic type judgment over `(bytes, resolved_ext, is_bin, dictionary)`.
    pub fn judge(
        &self,
        data: Option<&[u8]>,
        filename: Option<&str>,
        ext_hint: Option<&str>,
    ) -> TypeJudgment {
        let raw_bytes = data.unwrap_or(&[]);
        let resolved_ext = resolve_extension(filename, ext_hint, Some(&self.ext_map));
        let is_bin = is_binary_payload(raw_bytes);
        let phases = tp::build_type_phase_pipeline::<JudgmentInput<'_>, TypeJudgment>(
            phase_high_priority,
            phase_magic_and_brand,
            phase_extension,
            phase_truncated_magic,
            phase_structured_and_text_pattern,
            phase_standard_predicate_and_fallback,
        );
        let input: JudgmentInput<'_> = (raw_bytes, resolved_ext, is_bin, self);
        first_match(&phases, &input).expect("standard_predicate_and_fallback guarantees terminal judgment")
    }

    pub fn detect_content_type(&self, data: &[u8], ext_hint: Option<&str>) -> (String, String) {
        let judgment = self.judge(Some(data), None, ext_hint);
        let clean_ext = judgment
            .extension
            .map(|e| e.trim_start_matches('.').to_string())
            .unwrap_or_default();
        (judgment.mime, clean_ext)
    }

    pub fn is_binary_content(&self, data: &[u8]) -> bool {
        let binary_signature = self.magic_signatures.iter().any(|(sig, mime)| {
            sig.matches(data) && self.types.get(mime).is_some_and(|def| def.is_binary)
        });
        binary_signature || is_binary_payload(data)
    }
}