Skip to main content

rypipe_core/
lib.rs

1//! `rypipe-core`: format-agnostic ingestion framework.
2//!
3//! The crate parses byte streams into typed Arrow record batches through a
4//! small decoder API (`Splitter` + `RecordParser` + `ColumnarSink`).  It has
5//! no format-specific logic and no Python/FFI dependencies.
6
7#![allow(unexpected_cfgs)]
8#![allow(unsafe_code)]
9
10#[cfg(feature = "mimalloc")]
11#[global_allocator]
12static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
13
14#[cfg(feature = "alloc-stats")]
15pub mod alloc_stats;
16pub mod arrow_export;
17pub mod auto;
18#[cfg(feature = "bench")]
19pub mod bench;
20pub mod block_masks;
21pub mod bounded;
22pub mod budget;
23pub mod columnar;
24pub mod consumer;
25pub mod decoder;
26#[cfg(feature = "streaming-parser")]
27pub mod diagnostics;
28pub mod dict;
29pub mod engine;
30pub mod error;
31pub mod input;
32pub mod merge;
33pub mod observer;
34pub mod parallel;
35pub mod parallel_stream;
36pub mod pipeline;
37pub mod plan;
38pub mod profiling;
39pub mod scan;
40pub mod schema;
41pub mod streaming;
42#[cfg(feature = "streaming-parser")]
43pub mod streaming_parser;
44pub mod value;
45
46pub use arrow_export::apply_compare_filter;
47pub use auto::{resolve_engine, AutoConfig, EngineMode};
48pub use bounded::{MemoryBudget, MAX_SPLIT_CHUNKS};
49pub use budget::{BudgetLedger, BudgetPartition, StreamStats};
50pub use consumer::{BatchConsumer, CollectingConsumer, DiscardingConsumer};
51pub use decoder::{
52    find_next_record_boundary, is_continued_newline, split_points_to_ranges, ColumnarSink,
53    RecordBoundary, RecordParser, Splitter,
54};
55#[cfg(feature = "streaming-parser")]
56pub use diagnostics::{CollectingDiagnostics, NoopDiagnostics, ParseDiagnostics};
57pub use engine::{LocateOnly, TableBuilder};
58#[cfg(feature = "profile")]
59pub use engine::{
60    IS_PRED_FALSE, IS_PRED_TRUE, PREDICATE_EVALUATIONS, PREDICATE_FAILS, PREDICATE_UNDECIDED,
61    RESOLVE_AND_PUT_COUNT,
62};
63pub use error::{Error, Result};
64pub use input::InputBuffer;
65pub use merge::engines_to_record_batches;
66pub use observer::RowObserver;
67pub use parallel_stream::{
68    discover_schema_for_bytes, discovery_profile, reset_discovery_profile, ParallelStreamOpts,
69    ParallelStreamingBatchIterator, ParallelStreamingExecutor,
70};
71pub use pipeline::Pipeline;
72pub use plan::{CompareOp, ExecutionPlan, FieldType, FilterPredicate, RegexSpec, TrimMode};
73pub use schema::{DiscoveryOpts, FrozenSchema, UnknownFieldPolicy};
74pub use streaming::StreamingBatchIterator;
75#[cfg(feature = "streaming-parser")]
76pub use streaming_parser::{RecordStream, StreamState, StreamingRecordParser};
77pub use value::Value;