zahirscan 0.3.3

Token-efficient content compression for AI analysis using probabilistic template mining
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

//! Common utilities for parsers

use rayon::prelude::*;

use crate::config::RuntimeConfig;
use crate::parsers::{ParseResult, estimate_compressed_tokens_with_footprint};
use crate::results::{MiningResult, Template};

/// Create an empty `MiningResult` (no templates found)
/// This is shared across all parsers for empty content cases
#[must_use]
pub fn empty_mining_result(stats: &ParseResult) -> MiningResult {
    MiningResult {
        templates: vec![],
        original_tokens: stats.token_count,
        compressed_tokens: 0,
        token_reduction_percent: 0.0,
        writing_footprint: None,
    }
}

/// Build `MiningResult` from templates (sorts and calculates compression)
/// This is shared across all parsers
#[must_use]
pub fn build_mining_result(
    templates: Vec<Template>,
    total_items: usize,
    stats: &ParseResult,
    config: &RuntimeConfig,
) -> MiningResult {
    build_mining_result_with_footprint(templates, total_items, stats, config, None)
}

/// Build `MiningResult` from templates including writing footprint in compression calculation
#[must_use]
pub fn build_mining_result_with_footprint(
    templates: Vec<Template>,
    total_items: usize,
    stats: &ParseResult,
    config: &RuntimeConfig,
    writing_footprint: Option<&crate::results::WritingFootprint>,
) -> MiningResult {
    // Sort by count (most common first)
    let mut sorted_templates = templates;
    sort_templates_by_count(&mut sorted_templates);

    // Calculate compression including writing footprint tokens
    let (original_tokens, compressed_tokens, token_reduction_percent) =
        calculate_compression_with_footprint(
            &sorted_templates,
            total_items,
            stats,
            config,
            writing_footprint,
        );

    MiningResult {
        templates: sorted_templates,
        original_tokens,
        compressed_tokens,
        token_reduction_percent,
        writing_footprint: writing_footprint.cloned(),
    }
}

/// Calculate compression metrics from templates
/// Returns (`original_tokens`, `compressed_tokens`, `token_reduction_percent`)
#[must_use]
pub fn calculate_compression(
    templates: &[Template],
    total_items: usize,
    stats: &ParseResult,
    config: &RuntimeConfig,
) -> (usize, usize, f64) {
    calculate_compression_with_footprint(templates, total_items, stats, config, None)
}

/// Calculate compression metrics from templates including writing footprint
/// Returns (`original_tokens`, `compressed_tokens`, `token_reduction_percent`)
#[must_use]
pub fn calculate_compression_with_footprint(
    templates: &[Template],
    total_items: usize,
    stats: &ParseResult,
    config: &RuntimeConfig,
    writing_footprint: Option<&crate::results::WritingFootprint>,
) -> (usize, usize, f64) {
    let original_tokens = stats.token_count;
    let compressed_tokens = estimate_compressed_tokens_with_footprint(
        templates,
        total_items,
        config,
        writing_footprint,
    );
    let token_reduction_percent = if original_tokens > 0 {
        let reduction = original_tokens.saturating_sub(compressed_tokens);
        (reduction as f64 / original_tokens as f64) * 100.0
    } else {
        0.0
    };
    (original_tokens, compressed_tokens, token_reduction_percent)
}

/// Sort templates by count (most common first)
/// This is shared across all parsers
pub fn sort_templates_by_count(templates: &mut [Template]) {
    templates.sort_by_key(|item| std::cmp::Reverse(item.count));
}

// ============================================================================
// Parallel Processing Utilities
// ============================================================================

// Re-export optimal_chunk_size from chunking module for convenience
pub use crate::engine::chunking::optimal_chunk_size;

/// Extension trait for collections to enable adaptive parallel iteration
/// Combines chunk size calculation with parallel iteration setup
pub trait AdaptiveParallel:
    IntoParallelIterator<Iter: rayon::iter::IndexedParallelIterator>
{
    /// Returns a parallel iterator configured with adaptive chunking based on config
    fn par_iter_adaptive(self, config: &RuntimeConfig) -> rayon::iter::MinLen<Self::Iter>
    where
        Self: Sized,
    {
        let iter = self.into_par_iter();
        let collection_size = iter.len();
        let chunk_size = optimal_chunk_size(
            collection_size,
            config.target_chunks_per_file,
            config.min_collection_size_for_chunking,
        );
        iter.with_min_len(chunk_size)
    }
}

// Implement for common collection types
impl<T> AdaptiveParallel for &[T] where T: Send + Sync {}
impl<T> AdaptiveParallel for Vec<T> where T: Send + Sync {}