Skip to main content

rypipe_core/
lib.rs

1//! `rypipe-core`: format-agnostic ingestion framework.
2//!
3//! The crate parses byte streams into typed Arrow record batches through a
4//! small decoder API (`Splitter` + `RecordParser` + `ColumnarSink`).  It has
5//! no format-specific logic and no Python/FFI dependencies.
6
7#![allow(unexpected_cfgs)]
8
9#[cfg(feature = "mimalloc")]
10#[global_allocator]
11static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
12
13#[cfg(feature = "alloc-stats")]
14pub mod alloc_stats;
15pub mod arrow_export;
16#[cfg(feature = "bench")]
17pub mod bench;
18pub mod block_masks;
19pub mod bounded;
20pub mod columnar;
21pub mod consumer;
22pub mod decoder;
23pub mod dict;
24pub mod engine;
25pub mod error;
26pub mod input;
27pub mod merge;
28pub mod parallel;
29pub mod parallel_stream;
30pub mod pipeline;
31pub mod plan;
32pub mod scan;
33pub mod schema;
34pub mod streaming;
35pub mod value;
36
37pub use arrow_export::apply_compare_filter;
38pub use bounded::MemoryBudget;
39pub use consumer::{BatchConsumer, CollectingConsumer, DiscardingConsumer};
40pub use decoder::{split_points_to_ranges, ColumnarSink, RecordParser, Splitter};
41pub use engine::{LocateOnly, TableBuilder};
42#[cfg(feature = "profile")]
43pub use engine::{
44    IS_PRED_FALSE, IS_PRED_TRUE, PREDICATE_EVALUATIONS, PREDICATE_FAILS, PREDICATE_UNDECIDED,
45    RESOLVE_AND_PUT_COUNT,
46};
47pub use error::{Error, Result};
48pub use input::InputBuffer;
49pub use merge::engines_to_record_batches;
50pub use parallel_stream::{
51    discover_schema_for_bytes, discovery_profile, reset_discovery_profile, ParallelStreamOpts,
52    ParallelStreamingBatchIterator, ParallelStreamingExecutor,
53};
54pub use pipeline::Pipeline;
55pub use plan::{CompareOp, ExecutionPlan, FieldType, FilterPredicate};
56pub use schema::{DiscoveryOpts, FrozenSchema, UnknownFieldPolicy};
57pub use streaming::StreamingBatchIterator;
58pub use value::Value;