polars-lazy 0.55.2

Lazy query engine for the Polars DataFrame library
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

use polars_core::prelude::*;
use polars_io::{HiveOptions, RowIndex};
use polars_utils::slice_enum::Slice;

use crate::prelude::*;

#[derive(Clone)]
pub struct ScanArgsAnonymous {
    pub infer_schema_length: Option<usize>,
    pub schema: Option<SchemaRef>,
    pub skip_rows: Option<usize>,
    pub n_rows: Option<usize>,
    pub row_index: Option<RowIndex>,
    pub name: &'static str,
}

impl Default for ScanArgsAnonymous {
    fn default() -> Self {
        Self {
            infer_schema_length: None,
            skip_rows: None,
            n_rows: None,
            schema: None,
            row_index: None,
            name: "ANONYMOUS SCAN",
        }
    }
}
impl LazyFrame {
    pub fn anonymous_scan(
        function: Arc<dyn AnonymousScan>,
        args: ScanArgsAnonymous,
    ) -> PolarsResult<Self> {
        let schema = match args.schema {
            Some(s) => s,
            None => function.schema(args.infer_schema_length)?,
        };

        let mut lf: LazyFrame = DslBuilder::anonymous_scan(
            function,
            AnonymousScanOptions {
                skip_rows: args.skip_rows,
                fmt_str: args.name,
            },
            UnifiedScanArgs {
                schema: Some(schema),
                cloud_options: None,
                hive_options: HiveOptions::new_disabled(),
                rechunk: false,
                cache: false,
                glob: false,
                hidden_file_prefix: None,
                projection: None,
                column_mapping: None,
                default_values: None,
                row_index: None,
                pre_slice: args.n_rows.map(|len| Slice::Positive { offset: 0, len }),
                cast_columns_policy: CastColumnsPolicy::ERROR_ON_MISMATCH,
                missing_columns_policy: MissingColumnsPolicy::Raise,
                extra_columns_policy: ExtraColumnsPolicy::Raise,
                include_file_paths: None,
                deletion_files: None,
                table_statistics: None,
                row_count: None,
            },
        )?
        .build()
        .into();

        if let Some(rc) = args.row_index {
            lf = lf.with_row_index(rc.name.clone(), Some(rc.offset))
        };

        Ok(lf)
    }
}