json-extractor 0.1.0

//! Integration tests for large file streaming
//!
//! These tests verify that the scanner can handle large volumes of data
//! efficiently, processing megabytes of JSON fragments.

use json_extractor::JsonFragmentScanner;

#[test]
fn test_large_file_single_fragment() {
    // Create a 1MB+ JSON object
    let mut large_json = String::from("{");
    for i in 0..50_000 {
        if i > 0 {
            large_json.push(',');
        }
        large_json.push_str(&format!(r#""key_{}": "value_{}""#, i, i));
    }
    large_json.push('}');

    let fragments = JsonFragmentScanner::scan_fragments(large_json.as_bytes());

    assert_eq!(fragments.len(), 1);
    assert!(fragments[0].is_complete());
    assert_eq!(fragments[0].start, 0);
    assert_eq!(fragments[0].length, large_json.len());
}

#[test]
fn test_large_file_many_fragments() {
    // Create many small fragments totaling >1 MB
    // Reduced from 100k to 10k for faster test execution
    let mut data = Vec::new();
    for i in 0..10_000 {
        data.extend_from_slice(format!(r#"{{"id":{}}}"#, i).as_bytes());
        data.push(b' ');
    }

    let fragments = JsonFragmentScanner::scan_fragments(&data);

    // All 10,000 fragments should be detected
    assert_eq!(fragments.len(), 10_000);
    for frag in fragments {
        assert!(frag.is_complete());
    }
}

#[test]
fn test_large_file_chunked_processing() {
    // Test that large documents crossing multiple SIMD chunk boundaries work correctly
    // Create a large JSON array (>10KB to cross many 64-byte AVX2 chunks)
    let mut large_array = String::from("[");
    for i in 0..10_000 {
        if i > 0 {
            large_array.push(',');
        }
        large_array.push_str(&i.to_string());
    }
    large_array.push(']');

    // Verify document is large enough to cross multiple SIMD chunks
    assert!(
        large_array.len() > 1024,
        "Document should be >1KB to cross multiple SIMD chunks"
    );

    // Process entire document at once (new full-document architecture)
    let fragments = JsonFragmentScanner::scan_fragments(large_array.as_bytes());

    // Should get exactly one complete fragment
    assert_eq!(fragments.len(), 1);
    assert!(fragments[0].is_complete());
    assert_eq!(fragments[0].start, 0);
    assert_eq!(fragments[0].length, large_array.len());
}

#[test]
fn test_large_nested_array() {
    // Create a large nested array
    let mut nested = String::from("[");
    for i in 0..1_000 {
        if i > 0 {
            nested.push(',');
        }
        nested.push_str(&format!("[{}, {}, {}]", i * 3, i * 3 + 1, i * 3 + 2));
    }
    nested.push(']');

    let fragments = JsonFragmentScanner::scan_fragments(nested.as_bytes());

    assert_eq!(fragments.len(), 1);
    assert!(fragments[0].is_complete());
}

#[test]
fn test_large_file_with_errors() {
    // Create a large file with some structurally invalid fragments mixed in
    let mut data = Vec::new();
    for i in 0..10_000 {
        if i % 100 == 0 {
            // Every 100th fragment has missing closing brace (100 total: 0, 100, 200, ..., 9900)
            data.extend_from_slice(format!(r#"{{"id":{} "#, i).as_bytes());
        } else {
            data.extend_from_slice(format!(r#"{{"id":{}}} "#, i).as_bytes());
        }
    }

    let fragments = JsonFragmentScanner::scan_fragments(&data);

    // Should have 10,000 fragments total
    assert_eq!(
        fragments.len(),
        10_000,
        "Expected 10,000 fragments but got {}",
        fragments.len()
    );

    // Count complete vs incomplete
    let complete_count = fragments.iter().filter(|f| f.is_complete()).count();
    let incomplete_count = fragments.iter().filter(|f| !f.is_complete()).count();

    // Note: The scanner does minimal validation (bracket matching only).
    // Fragments with matched brackets are "complete" even if content is invalid JSON.
    // Detailed JSON validation is left to downstream parsers.
    assert_eq!(complete_count, 9_900);
    assert_eq!(incomplete_count, 100);
}

#[test]
fn test_memory_efficiency_no_buffering() {
    // Verify that scanner doesn't buffer fragment content
    // by processing a very large fragment without causing memory issues
    // Reduced from 10MB to 1MB for faster test execution
    let mut huge_string_value = String::from(r#"{"data": ""#);
    // Add 1MB of 'a' characters
    for _ in 0..1_000_000 {
        huge_string_value.push('a');
    }
    huge_string_value.push_str("\"}");

    // Process entire document (new full-document architecture)
    let fragments = JsonFragmentScanner::scan_fragments(huge_string_value.as_bytes());

    // Should complete successfully without buffering the entire string content
    // (only indices and metadata are stored)
    assert_eq!(fragments.len(), 1);
    assert!(fragments[0].is_complete());
    assert_eq!(fragments[0].start, 0);
    assert_eq!(fragments[0].length, huge_string_value.len());
}