zahirscan 0.3.6

Token-efficient content compression for AI analysis using probabilistic template mining
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

//! Apache Parquet metadata and column statistics.

use anyhow::{Context, Result};
use bytes::Bytes;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;

use crate::config::RuntimeConfig;
use crate::parsers::{ParseResult, structured::constants::StructuredEncoding};
use crate::results::{ColumnarCommonFields, ParquetMetadata};

use super::utils as columnar_utils;

/// Extract Parquet schema, row counts, row groups, and CSV-like column statistics from a sample.
///
/// # Errors
///
/// Returns an error if the bytes are not a valid Parquet file or Arrow decoding fails.
pub fn extract_parquet_metadata(
    mmap: &[u8],
    _stats: &ParseResult,
    config: &RuntimeConfig,
) -> Result<ParquetMetadata> {
    let bytes = Bytes::copy_from_slice(mmap);
    let builder = ParquetRecordBatchReaderBuilder::try_new(bytes).context("open Parquet")?;
    let num_row_groups = Some(builder.metadata().num_row_groups());
    let row_count_total = builder
        .metadata()
        .file_metadata()
        .num_rows()
        .try_into()
        .unwrap_or(usize::MAX);
    let schema_ref = builder.schema().clone();
    let column_names = columnar_utils::schema_column_names(schema_ref.as_ref());
    let arrow_field_types = columnar_utils::schema_arrow_dtype_strings(schema_ref.as_ref());
    let column_count = column_names.len();
    let file_bytes = mmap.len() as u64;

    let max_sample = columnar_utils::tabular_effective_sample_rows(
        config.max_tabular_sample_rows,
        file_bytes,
        column_count.max(1),
        Some(row_count_total),
    );
    let mut reader = builder.build().context("build Parquet reader")?;
    let sample_data = columnar_utils::record_batches_to_string_sample(
        reader.by_ref(),
        max_sample,
        config,
        "read Parquet batch",
    )?;

    let ts = columnar_utils::tabular_stats_from_sample(&sample_data, column_count, config);

    let columns = columnar_utils::columns_from_tabular_sample(
        column_count,
        Some(column_names),
        ts,
        Some(arrow_field_types),
    );

    Ok(ParquetMetadata {
        common: ColumnarCommonFields {
            row_count: Some(row_count_total),
            column_count: Some(column_count),
            stats_rows_sampled: Some(sample_data.len()),
            encoding: Some(StructuredEncoding::TABULAR_BINARY.to_string()),
            columns,
        },
        num_row_groups,
    })
}

crate::no_template_mining!(
    extract_parquet_templates,
    "Parquet is columnar binary; no text template mining."
);