tauq 0.2.0

//! Tauq Binary Format (TBF) - State-of-the-Art Binary Serialization
//!
//! TBF is a custom binary format designed specifically for Tauq's schema-based
//! architecture. It achieves best-in-class performance through:
//!
//! - **Direct serde integration**: No intermediate representations
//! - **Schema-aware encoding**: No type tags needed - schema defines structure
//! - **Varint encoding**: Compact integers using LEB128
//! - **String dictionary**: Deduplicate repeated strings
//! - **Zero-copy decoding**: Borrowed references where possible
//!
//! # Performance
//!
//! TBF achieves ~17% of JSON size with competitive serialization speed.
//!
//! # Format Specification
//!
//! ```text
//! TBF File Structure:
//! ┌─────────────────────────────────────┐
//! │ Header (8 bytes)                    │
//! │   Magic: "TBF\x01" (4 bytes)        │
//! │   Version: u8                       │
//! │   Flags: u8                         │
//! │   Reserved: u16                     │
//! ├─────────────────────────────────────┤
//! │ String Dictionary                   │
//! │   Count: varint                     │
//! │   Strings: [len:varint, utf8...]    │
//! ├─────────────────────────────────────┤
//! │ Schemas (optional)                  │
//! │   [schema definitions...]           │
//! ├─────────────────────────────────────┤
//! │ Codec Metadata (optional)           │
//! │   Count: varint                     │
//! │   For each codec:                   │
//! │     Type: u8                        │
//! │     Metadata: [varint/specific]     │
//! ├─────────────────────────────────────┤
//! │ Data Section                        │
//! │   Encoded values (type-tagged)      │
//! ├─────────────────────────────────────┤
//! │ Statistics Footer (optional)        │
//! │   Footer offset: u64                │
//! └─────────────────────────────────────┘
//! ```

#[allow(dead_code)]
mod adaptive_encode;
mod batch_encode;
mod bitmap;
#[allow(dead_code)]
mod bloom;
#[allow(dead_code)]
mod codec_decode;
#[allow(dead_code)]
mod codec_encode;
#[allow(dead_code)]
mod columnar;
mod decoder;
mod dictionary;
#[allow(dead_code)]
mod encoder;
#[allow(dead_code)]
mod fast_decode;
#[allow(dead_code)]
mod fast_encode;
#[allow(dead_code)]
mod parallel_encode;
mod predicate_pushdown;
mod schema;
#[allow(dead_code)]
mod schema_encode;
mod serde_impl;
#[allow(dead_code)]
mod simd_decode;
mod stats;
mod stats_collector;
mod traits;
#[allow(dead_code)]
mod ultra_encode;
#[allow(dead_code)]
mod varint;

// ==========================================================================
// Public API: Only export types needed by library consumers
// ==========================================================================
pub use decoder::{EnumAccess, MapAccess, SeqAccess, TbfDeserializer};
pub use dictionary::{BorrowedDictionary, StringDictionary};
pub use encoder::TbfSerializer;
pub use schema::{Schema, SchemaField, SchemaRegistry, SchemaType, infer_schema_from_json};
pub use traits::{TbfDecode, TbfEncode};
pub use varint::{decode_signed_varint, decode_varint, encode_signed_varint, encode_varint};

// Schema-aware encoding API
pub use schema_encode::{
    ColumnSchema, FieldEncoding, TableEncode, TableSchema, TableSchemaBuilder,
};

// Columnar encoding API
pub use columnar::{
    ColumnarDecode, ColumnarDecoder, ColumnarEncode, ColumnarEncoder, TBC_MAGIC, TBC_VERSION,
};

// Statistics
pub use stats::ColumnStats;
pub use stats_collector::StatisticsCollector;

// Fast encode/decode for advanced users
pub use fast_decode::{
    FastBorrowedDictionary, FastDecode, fast_decode_signed_varint, fast_decode_varint,
};
pub use fast_encode::{
    FastBuffer, FastEncode, FastStringDictionary, fast_encode_signed_varint, fast_encode_slice,
    fast_encode_varint,
};

// Predicate pushdown
pub use predicate_pushdown::{Predicate, QueryFilter};

// Bloom filter
pub use bloom::BloomFilter;

// Null bitmap
pub use bitmap::NullBitmap;

// Adaptive codecs (public for benchmarks and advanced use)
pub use adaptive_encode::{CodecAnalysis, CodecAnalyzer, CompressionCodec};

// Batch encoding
pub use batch_encode::{BatchEncoder, BatchEncodingStats};

// Ultra encoding (public for benchmarks)
pub use ultra_encode::{
    ColumnCollectors, DirectStringEncoder, DirectU32Encoder, ULTRA_MAGIC, ULTRA_VERSION,
    UltraBuffer, UltraEncode, UltraEncodeDirect, encode_varint_to_ultra,
};
// Re-export ultra_encode::ColumnType under the name benchmarks expect
pub use ultra_encode::ColumnType as UltraColumnType;

// Columnar encoding types (public for benchmarks)
pub use columnar::{ColumnReader, ColumnType};

// Codec encode/decode (public for benchmarks)
pub use codec_decode::CodecDecodingContext;
pub use codec_encode::{CodecEncodingContext, CodecMetadata};

// Schema encoding (public for benchmarks)
pub use schema_encode::{
    AdaptiveIntEncoder, AdaptiveStringEncoder, SCHEMA_MAGIC, encode_varint_fast,
};

#[cfg(feature = "performance")]
pub use parallel_encode::{ParallelBatchEncoder, ParallelEncodingStats};

// serde_impl exports are not needed at module level

use crate::error::TauqError;

// ============================================================================
// Constants
// ============================================================================

/// TBF magic bytes: "TBF\x01"
pub const TBF_MAGIC: [u8; 4] = [0x54, 0x42, 0x46, 0x01];

/// Current TBF version
pub const TBF_VERSION: u8 = 1;

/// Flag: String dictionary enabled
pub const FLAG_DICTIONARY: u8 = 0x02;

/// Flag: Codec metadata section enabled
pub const FLAG_CODEC_METADATA: u8 = 0x04;

// ============================================================================
// Type Tags
// ============================================================================

/// Type tags for TBF encoding
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum TypeTag {
    /// Null value (0x00)
    Null = 0,
    /// Boolean value (0x01)
    Bool = 1,
    /// Integer value (0x02)
    Int = 2,
    /// Floating point value (0x03)
    Float = 3,
    /// UTF-8 string (0x04)
    String = 4,
    /// Byte array (0x05)
    Bytes = 5,
    /// Sequence/Array (0x06)
    Seq = 6,
    /// Map/Object (0x07)
    Map = 7,
    // Extended tags
    /// Unit type (0x08)
    Unit = 8,
    /// Option::None (0x09)
    None = 9,
    /// Option::Some (0x0A)
    Some = 10,
    /// Signed 8-bit integer (0x0B)
    I8 = 11,
    /// Signed 16-bit integer (0x0C)
    I16 = 12,
    /// Signed 32-bit integer (0x0D)
    I32 = 13,
    /// Signed 64-bit integer (0x0E)
    I64 = 14,
    /// Signed 128-bit integer (0x0F)
    I128 = 15,
    /// Unsigned 8-bit integer (0x10)
    U8 = 16,
    /// Unsigned 16-bit integer (0x11)
    U16 = 17,
    /// Unsigned 32-bit integer (0x12)
    U32 = 18,
    /// Unsigned 64-bit integer (0x13)
    U64 = 19,
    /// Unsigned 128-bit integer (0x14)
    U128 = 20,
    /// 32-bit Float (0x15)
    F32 = 21,
    /// 64-bit Float (0x16)
    F64 = 22,
    /// Character (0x17)
    Char = 23,
}

impl TypeTag {
    /// Convert byte to TypeTag variant
    pub fn from_u8(v: u8) -> Option<Self> {
        match v {
            0 => Some(TypeTag::Null),
            1 => Some(TypeTag::Bool),
            2 => Some(TypeTag::Int),
            3 => Some(TypeTag::Float),
            4 => Some(TypeTag::String),
            5 => Some(TypeTag::Bytes),
            6 => Some(TypeTag::Seq),
            7 => Some(TypeTag::Map),
            8 => Some(TypeTag::Unit),
            9 => Some(TypeTag::None),
            10 => Some(TypeTag::Some),
            11 => Some(TypeTag::I8),
            12 => Some(TypeTag::I16),
            13 => Some(TypeTag::I32),
            14 => Some(TypeTag::I64),
            15 => Some(TypeTag::I128),
            16 => Some(TypeTag::U8),
            17 => Some(TypeTag::U16),
            18 => Some(TypeTag::U32),
            19 => Some(TypeTag::U64),
            20 => Some(TypeTag::U128),
            21 => Some(TypeTag::F32),
            22 => Some(TypeTag::F64),
            23 => Some(TypeTag::Char),
            _ => None,
        }
    }
}

// ============================================================================
// Convenience Functions
// ============================================================================

/// Serialize a value directly to TBF bytes (fast path)
///
/// This uses direct serde integration, bypassing any intermediate representation.
///
/// # Example
/// ```
/// use serde::Serialize;
///
/// #[derive(Serialize)]
/// struct User {
///     id: u32,
///     name: String,
/// }
///
/// let user = User { id: 1, name: "Alice".into() };
/// let bytes = tauq::tbf::to_bytes(&user).unwrap();
/// ```
pub fn to_bytes<T: serde::Serialize>(value: &T) -> Result<Vec<u8>, TauqError> {
    let mut serializer = TbfSerializer::new();
    value.serialize(&mut serializer)?;
    Ok(serializer.into_bytes())
}

/// Serialize a value to TBF bytes with pre-allocated capacity
pub fn to_bytes_with_capacity<T: serde::Serialize>(
    value: &T,
    capacity: usize,
) -> Result<Vec<u8>, TauqError> {
    let mut serializer = TbfSerializer::with_capacity(capacity);
    value.serialize(&mut serializer)?;
    Ok(serializer.into_bytes())
}

/// Deserialize TBF bytes directly to a value (fast path)
///
/// # Example
/// ```
/// use serde::Deserialize;
///
/// #[derive(Deserialize, Debug)]
/// struct User {
///     id: u32,
///     name: String,
/// }
///
/// // let bytes = ...;
/// // let user: User = tauq::tbf::from_bytes(&bytes).unwrap();
/// ```
pub fn from_bytes<'de, T: serde::Deserialize<'de>>(bytes: &'de [u8]) -> Result<T, TauqError> {
    let mut deserializer = TbfDeserializer::new(bytes)?;
    T::deserialize(&mut deserializer)
}

/// Encode Tauq source to TBF binary format (via JSON - slower path)
pub fn encode(source: &str) -> Result<Vec<u8>, TauqError> {
    let json = crate::compile_tauq(source)?;
    encode_json(&json)
}

/// Encode JSON value to TBF binary format
pub fn encode_json(json: &serde_json::Value) -> Result<Vec<u8>, TauqError> {
    to_bytes(json)
}

/// Decode TBF binary to JSON
pub fn decode(data: &[u8]) -> Result<serde_json::Value, TauqError> {
    from_bytes(data)
}

/// Decode TBF binary to Tauq string
pub fn decode_to_tauq(data: &[u8]) -> Result<String, TauqError> {
    let json: serde_json::Value = from_bytes(data)?;
    Ok(crate::format_to_tauq(&json))
}

// ============================================================================
// Tests
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;
    use serde::{Deserialize, Serialize};

    #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
    struct TestUser {
        id: u32,
        name: String,
        age: u32,
        active: bool,
    }

    #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
    struct Employee {
        id: u32,
        name: String,
        age: u32,
        city: String,
        department: String,
        salary: u32,
    }

    #[test]
    fn test_direct_serde_roundtrip() {
        let user = TestUser {
            id: 1,
            name: "Alice".into(),
            age: 30,
            active: true,
        };

        let bytes = to_bytes(&user).unwrap();
        let decoded: TestUser = from_bytes(&bytes).unwrap();

        assert_eq!(user, decoded);
    }

    #[test]
    fn test_vec_roundtrip() {
        let users = vec![
            TestUser {
                id: 1,
                name: "Alice".into(),
                age: 30,
                active: true,
            },
            TestUser {
                id: 2,
                name: "Bob".into(),
                age: 25,
                active: false,
            },
            TestUser {
                id: 3,
                name: "Carol".into(),
                age: 35,
                active: true,
            },
        ];

        let bytes = to_bytes(&users).unwrap();
        let decoded: Vec<TestUser> = from_bytes(&bytes).unwrap();

        assert_eq!(users, decoded);
    }

    #[test]
    fn test_primitives() {
        // Integers
        let v: i32 = -42;
        assert_eq!(v, from_bytes::<i32>(&to_bytes(&v).unwrap()).unwrap());

        let v: u64 = 12345678901234;
        assert_eq!(v, from_bytes::<u64>(&to_bytes(&v).unwrap()).unwrap());

        // Floats
        let v: f64 = 1.234567890123;
        assert_eq!(v, from_bytes::<f64>(&to_bytes(&v).unwrap()).unwrap());

        // Bool
        let v: bool = true;
        assert_eq!(v, from_bytes::<bool>(&to_bytes(&v).unwrap()).unwrap());

        // String
        let v: String = "Hello, World!".into();
        assert_eq!(v, from_bytes::<String>(&to_bytes(&v).unwrap()).unwrap());
    }

    #[test]
    fn test_option() {
        let some: Option<i32> = Some(42);
        let none: Option<i32> = None;

        assert_eq!(some, from_bytes(&to_bytes(&some).unwrap()).unwrap());
        assert_eq!(
            none,
            from_bytes::<Option<i32>>(&to_bytes(&none).unwrap()).unwrap()
        );
    }

    #[test]
    fn test_nested_struct() {
        #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
        struct Outer {
            name: String,
            inner: Inner,
        }

        #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
        struct Inner {
            value: i32,
            data: Vec<u8>,
        }

        let outer = Outer {
            name: "test".into(),
            inner: Inner {
                value: 42,
                data: vec![1, 2, 3, 4, 5],
            },
        };

        let bytes = to_bytes(&outer).unwrap();
        let decoded: Outer = from_bytes(&bytes).unwrap();

        assert_eq!(outer, decoded);
    }

    #[test]
    fn test_json_value_roundtrip() {
        let json = serde_json::json!({
            "users": [
                {"id": 1, "name": "Alice", "age": 30},
                {"id": 2, "name": "Bob", "age": 25},
            ],
            "count": 2
        });

        let bytes = encode_json(&json).unwrap();
        let decoded = decode(&bytes).unwrap();

        assert_eq!(json, decoded);
    }

    #[test]
    fn test_size_comparison() {
        let employees: Vec<Employee> = (0..100)
            .map(|i| Employee {
                id: i,
                name: format!("Employee{}", i),
                age: 25 + (i % 40),
                city: ["NYC", "LA", "Chicago", "Houston", "Phoenix"][i as usize % 5].into(),
                department: ["Engineering", "Sales", "Marketing", "HR", "Finance"][i as usize % 5]
                    .into(),
                salary: 50000 + (i * 1000),
            })
            .collect();

        let json_str = serde_json::to_string(&employees).unwrap();
        let tbf_bytes = to_bytes(&employees).unwrap();

        println!("JSON size: {} bytes", json_str.len());
        println!("TBF size: {} bytes", tbf_bytes.len());
        println!(
            "Compression ratio: {:.1}%",
            (tbf_bytes.len() as f64 / json_str.len() as f64) * 100.0
        );

        // TBF should be smaller than JSON
        assert!(tbf_bytes.len() < json_str.len());
    }
}