polars_io/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2#![cfg_attr(feature = "simd", feature(portable_simd))]
3#![cfg_attr(
4    feature = "allow_unused",
5    allow(unused, dead_code, irrefutable_let_patterns)
6)] // Maybe be caused by some feature
7#![allow(ambiguous_glob_reexports)]
8extern crate core;
9
10#[cfg(feature = "avro")]
11pub mod avro;
12#[cfg(feature = "catalog")]
13pub mod catalog;
14pub mod cloud;
15#[cfg(any(feature = "csv", feature = "json"))]
16pub mod csv;
17#[cfg(feature = "file_cache")]
18pub mod file_cache;
19#[cfg(any(feature = "ipc", feature = "ipc_streaming"))]
20pub mod ipc;
21#[cfg(feature = "json")]
22pub mod json;
23pub mod mmap;
24#[cfg(feature = "json")]
25pub mod ndjson;
26mod options;
27#[cfg(feature = "parquet")]
28pub mod parquet;
29pub mod path_utils;
30#[cfg(feature = "async")]
31pub mod pl_async;
32pub mod predicates;
33pub mod prelude;
34#[cfg(feature = "scan_lines")]
35pub mod scan_lines;
36mod shared;
37pub mod utils;
38
39#[cfg(feature = "cloud")]
40pub use cloud::glob as async_glob;
41pub use options::*;
42pub use path_utils::*;
43pub use shared::*;
44pub mod metrics;
45
46pub mod hive;
47
48pub fn get_upload_chunk_size() -> usize {
49    use std::sync::LazyLock;
50
51    return *UPLOAD_CHUNK_SIZE;
52
53    static UPLOAD_CHUNK_SIZE: LazyLock<usize> = LazyLock::new(|| {
54        let v = std::env::var("POLARS_UPLOAD_CHUNK_SIZE")
55            .map(|x| {
56                x.parse::<usize>()
57                    .ok()
58                    .filter(|x| *x > 0)
59                    .unwrap_or_else(|| panic!("invalid value for POLARS_UPLOAD_CHUNK_SIZE: {x}"))
60            })
61            .unwrap_or(64 * 1024 * 1024);
62
63        if polars_core::config::verbose() {
64            eprintln!("async upload_chunk_size: {v}")
65        }
66
67        v
68    });
69}
70
71pub fn get_upload_concurrency() -> usize {
72    use std::sync::LazyLock;
73
74    return *UPLOAD_CONCURRENCY;
75
76    static UPLOAD_CONCURRENCY: LazyLock<usize> = LazyLock::new(|| {
77        // Max number of parts concurrently uploaded per Writer.
78        // @NOTE. The object_store::BufWriter uses 8 as default.
79        let v = std::env::var("POLARS_UPLOAD_CONCURRENCY")
80            .map(|x| {
81                x.parse::<usize>()
82                    .ok()
83                    .filter(|x| *x > 0)
84                    .unwrap_or_else(|| panic!("invalid value for POLARS_UPLOAD_CONCURRENCY: {x}"))
85            })
86            .unwrap_or(8);
87
88        if polars_core::config::verbose() {
89            eprintln!("async upload_concurrency: {v}")
90        }
91
92        v
93    });
94}
polars_io/lib.rs

polars_io/
lib.rs
