zahirscan 0.2.16

Token-efficient content compression for AI analysis using probabilistic template mining
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

//! Structured formats: CSV, HTML, JSON, EPUB, PDF.

use crate::config::RuntimeConfig;
use crate::parsers::{FileType, ParseResult};
use crate::results::MiningResult;
use anyhow::Result;
use memmap2::Mmap;

mod csv;
mod epub;
mod html;
mod json;
mod pdf;

pub use csv::{
    delimiter_byte_for_reader, detect_delimiter_byte, extract_csv_metadata, extract_csv_templates,
    infer_value_type,
};
pub use epub::{extract_epub_metadata, extract_epub_templates};
pub use html::{extract_html_metadata, extract_html_templates};
pub use json::{extract_json_metadata, extract_json_templates};
pub use pdf::{extract_pdf_metadata, extract_pdf_templates};

/// Dispatch by file type; fills `csv_metadata` or `html_metadata` and returns templates.
/// For text-based formats we pass content (&str) so UTF-8 is validated once at the boundary.
///
/// # Errors
///
/// Returns an error if the mmap is not valid UTF-8 where required, or if a structured parser fails.
pub fn process(
    stats: &mut ParseResult,
    mmap: &Mmap,
    config: &RuntimeConfig,
) -> Result<MiningResult> {
    match stats.file_type {
        FileType::Csv => crate::process_with_metadata!(
            stats,
            mmap,
            config,
            csv_metadata,
            extract_csv_metadata(mmap, stats, config),
            crate::results::CsvMetadata,
            FileType::Csv,
            extract_csv_templates(mmap, stats, config)
        ),
        FileType::Json => {
            let content = std::str::from_utf8(mmap)?;
            stats.json_metadata = Some(extract_json_metadata(content, stats));
            extract_json_templates(content, stats, config)
        }
        FileType::Epub => crate::process_with_metadata!(
            stats,
            mmap,
            config,
            epub_metadata,
            extract_epub_metadata(mmap, stats, config),
            crate::results::EpubMetadata,
            FileType::Epub,
            extract_epub_templates(mmap, stats, config)
        ),
        FileType::Html => crate::process_with_metadata!(
            stats,
            mmap,
            config,
            html_metadata,
            extract_html_metadata(mmap, stats, config),
            crate::results::HtmlMetadata,
            FileType::Html,
            extract_html_templates(mmap, stats, config)
        ),
        FileType::Pdf => crate::process_with_metadata!(
            stats,
            mmap,
            config,
            pdf_metadata,
            extract_pdf_metadata(mmap, stats, config),
            crate::results::PdfMetadata,
            FileType::Pdf,
            extract_pdf_templates(mmap, stats, config)
        ),
        _ => unreachable!("structured::process called with {:?}", stats.file_type),
    }
}