use serde::ser::SerializeStruct;
use serde::{Deserialize, Serialize, Serializer};
use std::collections::BTreeMap;
use crate::engine::chunking::ProcessingTask;
#[allow(clippy::wildcard_imports)]
use crate::metadata::*;
use super::writing::{CompressionStats, WritingFootprint};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OutputMode {
Templates,
Full,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Template {
pub pattern: String,
pub count: usize,
pub examples: BTreeMap<String, Vec<String>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MiningResult {
pub templates: Vec<Template>,
pub original_tokens: usize,
pub compressed_tokens: usize,
pub token_reduction_percent: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub writing_footprint: Option<WritingFootprint>,
}
#[derive(Debug, Clone, Default, Deserialize)]
pub struct Output {
pub templates: Vec<Template>,
pub source: Option<String>,
pub file_type: Option<String>,
pub line_count: Option<usize>,
pub byte_count: Option<usize>,
pub token_count: Option<usize>,
pub processing_time_ms: Option<f64>,
pub is_binary: Option<bool>,
pub compression: Option<CompressionStats>,
pub writing_footprint: Option<WritingFootprint>,
pub image_metadata: Option<ImageMetadata>,
pub video_metadata: Option<VideoMetadata>,
pub audio_metadata: Option<AudioMetadata>,
pub csv_metadata: Option<super::metadata::CsvMetadata>,
pub pdf_metadata: Option<super::metadata::PdfMetadata>,
pub docx_metadata: Option<DocumentMetadata>,
pub sqlite_metadata: Option<SqliteMetadata>,
pub toml_metadata: Option<TomlMetadata>,
pub zip_metadata: Option<ZipMetadata>,
pub xml_metadata: Option<XmlMetadata>,
pub html_metadata: Option<HtmlMetadata>,
pub yaml_metadata: Option<YamlMetadata>,
pub ini_metadata: Option<IniMetadata>,
pub pptx_metadata: Option<PptxMetadata>,
pub epub_metadata: Option<EpubMetadata>,
pub archive_metadata: Option<ArchiveMetadata>,
pub code_metadata: Option<CodeMetadata>,
pub log_metadata: Option<LogMetadata>,
pub json_metadata: Option<JsonMetadata>,
pub parquet_metadata: Option<super::metadata::ParquetMetadata>,
pub arrow_ipc_metadata: Option<super::metadata::ArrowIpcMetadata>,
pub avro_metadata: Option<super::metadata::AvroMetadata>,
pub orc_metadata: Option<super::metadata::OrcMetadata>,
pub npy_metadata: Option<super::metadata::NpyMetadata>,
pub npz_metadata: Option<super::metadata::NpzMetadata>,
pub hdf5_metadata: Option<super::metadata::Hdf5Metadata>,
pub netcdf_metadata: Option<super::metadata::NetCdfMetadata>,
pub mtx_metadata: Option<super::metadata::MtxMetadata>,
pub mat_metadata: Option<super::metadata::MatMetadata>,
pub onnx_metadata: Option<super::metadata::OnnxMetadata>,
pub gguf_metadata: Option<super::metadata::GgufMetadata>,
pub tflite_metadata: Option<super::metadata::TfliteMetadata>,
pub safetensors_metadata: Option<super::metadata::SafetensorsMetadata>,
pub zarr_metadata: Option<super::metadata::ZarrMetadata>,
pub tetration_metadata: Option<super::metadata::TetrationMetadata>,
pub pickle_metadata: Option<super::metadata::PickleMetadata>,
}
#[derive(Debug, Clone)]
pub struct FileMetadata {
pub source: String,
pub file_type: String,
pub line_count: usize,
pub byte_count: usize,
pub token_count: usize,
pub processing_time_ms: f64,
pub is_binary: bool,
}
impl Serialize for Output {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = serializer.serialize_struct("Output", 46)?;
state.serialize_field("templates", &self.templates)?;
crate::serialize_optional!(state, self.source, "source");
crate::serialize_optional!(state, self.file_type, "file_type");
crate::serialize_optional!(state, self.line_count, "line_count");
crate::serialize_optional!(state, self.byte_count, "byte_count");
crate::serialize_optional!(state, self.token_count, "token_count");
crate::serialize_optional!(state, self.processing_time_ms, "processing_time_ms");
crate::serialize_optional!(state, self.is_binary, "is_binary");
crate::serialize_optional!(state, self.compression, "compression");
crate::serialize_optional!(state, self.writing_footprint, "writing_footprint");
macro_rules! serialize_metadata_field {
($state:expr, $output:expr, $field:ident, $parse_ty:path, $output_ty:path, $serialized_name:literal) => {
crate::serialize_optional!($state, $output.$field, $serialized_name);
};
}
crate::for_each_metadata_field!(serialize_metadata_field, state, self);
state.end()
}
}
impl Output {
#[must_use]
pub fn templates_only(
templates: Vec<Template>,
source: Option<String>,
file_type: Option<String>,
) -> Self {
Self {
templates,
source,
file_type,
..Default::default()
}
}
#[must_use]
pub fn full(
templates: Vec<Template>,
metadata: FileMetadata,
compression: CompressionStats,
) -> Self {
Self {
templates,
source: Some(metadata.source),
file_type: Some(metadata.file_type),
line_count: Some(metadata.line_count),
byte_count: Some(metadata.byte_count),
token_count: Some(metadata.token_count),
processing_time_ms: Some(metadata.processing_time_ms),
is_binary: Some(metadata.is_binary),
compression: Some(compression),
..Default::default()
}
}
}
#[derive(Debug, Default)]
pub struct Phase1Result {
pub tasks: Vec<ProcessingTask>,
pub failed: Vec<(String, String)>,
}
#[derive(Debug, Clone, Default)]
pub struct Phase2Result {
pub outputs: Vec<Output>,
pub failed: Vec<(String, String)>,
}
#[derive(Debug, Clone, Default)]
pub struct ZahirScanResult {
pub outputs: Vec<Output>,
pub phase1_failed: Vec<(String, String)>,
pub phase2_failed: Vec<(String, String)>,
}