zahirscan 0.3.2

Token-efficient content compression for AI analysis using probabilistic template mining
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

//! Unit tests for code/script metadata: detect_file_type fallback and extract_code_metadata

use crate::get_test_config;
use std::fs::File;
use std::time::Duration;
use tempfile::TempDir;
use zahirscan::parsers::code::extract_code_metadata;
use zahirscan::parsers::{FileType, ParseResult};
use zahirscan::utils::filetypes::detect_file_type;

fn get_test_stats(file_path: &str, byte_count: usize, line_count: usize) -> ParseResult {
    ParseResult {
        file_path: file_path.to_string(),
        file_type: FileType::Code,
        line_count,
        byte_count,
        token_count: byte_count / 4,
        duration: Duration::ZERO,
        is_binary: false,
        ..Default::default()
    }
}

#[test]
fn detect_file_type_known_types_stay_known() {
    // Our FILE_EXTENSION_MAP first — never hit linguist
    assert_eq!(detect_file_type("foo.txt"), FileType::Text);
    assert_eq!(detect_file_type("foo.zip"), FileType::Zip);
    assert_eq!(detect_file_type("foo.sqlite3"), FileType::Sqlite);
    assert_eq!(detect_file_type("data.json"), FileType::Json);
    assert_eq!(detect_file_type("readme.md"), FileType::Markdown);
    assert_eq!(detect_file_type("track.aif"), FileType::Audio);
    assert_eq!(detect_file_type("track.aiff"), FileType::Audio);
    assert_eq!(detect_file_type("track.aifc"), FileType::Audio);
}

#[test]
fn detect_file_type_unknown_then_linguist_code() {
    // Unknown to our map → linguist fallback (extension/filename; file must exist) → Code if recognized.
    let dir = TempDir::new().unwrap();
    for (name, content) in [
        ("script.py", b"print(1)\n" as &[u8]),
        ("main.rs", b"fn main() {}\n"),
        ("app.js", b"console.log(1)\n"),
        ("run.sh", b"echo hi\n"),
    ] {
        let path = dir.path().join(name);
        std::fs::write(&path, content).unwrap();
        let path_str = path.to_str().unwrap();
        assert_eq!(detect_file_type(path_str), FileType::Code, "{}", name);
    }
}

#[test]
fn detect_file_type_unknown_stays_unknown() {
    // Unknown to our map and linguist returns empty or path doesn't exist
    assert_eq!(detect_file_type("file.xyz"), FileType::Unknown);
    assert_eq!(detect_file_type("data.unknown"), FileType::Unknown);
}

#[test]
fn extract_code_metadata_script_type_from_path() {
    let dir = TempDir::new().unwrap();
    let path = dir.path().join("main.rs");
    let content = b"fn main() {}\n";
    std::fs::write(&path, content).unwrap();

    let file = File::open(&path).unwrap();
    let mmap = unsafe { memmap2::Mmap::map(&file).unwrap() };
    let path_str = path.to_str().unwrap();
    let stats = get_test_stats(path_str, content.len(), 1);
    let config = get_test_config();

    let meta = extract_code_metadata(&mmap, &stats, &config).unwrap();
    assert_eq!(meta.script_type, "rust");
    assert_eq!(meta.byte_count, content.len());
    assert_eq!(meta.line_count, 1);
}

#[test]
fn extract_code_metadata_shebang_override() {
    let dir = TempDir::new().unwrap();
    let path = dir.path().join("script.py");
    let content = b"#!/usr/bin/env python3\nprint(42)\n";
    std::fs::write(&path, content).unwrap();

    let file = File::open(&path).unwrap();
    let mmap = unsafe { memmap2::Mmap::map(&file).unwrap() };
    let path_str = path.to_str().unwrap();
    let stats = get_test_stats(path_str, content.len(), 2);
    let config = get_test_config();

    let meta = extract_code_metadata(&mmap, &stats, &config).unwrap();
    assert_eq!(meta.script_type, "python");
    assert_eq!(meta.byte_count, content.len());
    assert_eq!(meta.line_count, 2);
}