tetration 0.1.8

Chunked tensor .tet format, tet CLI, and optional C ABI (tetration-ffi)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

//! Resolve transform `write` hints to an in-memory buffer or spill file path.

use std::path::{Path, PathBuf};

use crate::query::engine::{budget::ExecutionBudget, spill_policy::SpillPathAllowlist};
use crate::query::types::{ReadPlan, TetError, WriteHints, WriteTarget};
use crate::utils::dtype::ElementDtype;

/// Where pass 2 should materialize the transformed logical selection.
pub(crate) enum ResolvedTransformOutput {
    Ram,
    Spill(PathBuf),
}

/// Map [`WriteHints`] and the memory budget to RAM or a validated spill path.
///
/// # Errors
///
/// Returns [`TetError::Validation`] for unimplemented sidecar output, budget
/// overflow with `write: ram`, or spill path allowlist failures.
pub(crate) fn resolve_transform_output(
    write: Option<&WriteHints>,
    budget: &ExecutionBudget,
    plan: &ReadPlan,
    dtype: ElementDtype,
    allowlist: &SpillPathAllowlist,
) -> Result<(ResolvedTransformOutput, WriteTarget), TetError> {
    let target = write.map_or(WriteTarget::Switch, |w| w.target);
    let path_hint = write.and_then(|w| w.path.as_deref());

    if target == WriteTarget::Sidecar {
        return Err(TetError::Validation(
            "sidecar transform output is not implemented yet; use `write`: `spill` or `switch`"
                .into(),
        ));
    }

    let exceeds = budget.full_tensor_exceeds_budget(plan, dtype)?;

    match target {
        WriteTarget::Ram => {
            if exceeds {
                return Err(TetError::Validation(format!(
                    "logical selection ({} elements, {} bytes) exceeds memory_budget_bytes ({}); \
                     use `write`: `switch` or `spill`, or raise execution.memory_budget_bytes",
                    plan.logical_f32_element_count,
                    budget.logical_element_bytes(dtype, plan.logical_f32_element_count)?,
                    budget.memory_budget_bytes
                )));
            }
            Ok((ResolvedTransformOutput::Ram, WriteTarget::Ram))
        }
        WriteTarget::Spill => {
            let path = resolve_spill_path(path_hint, allowlist)?;
            Ok((ResolvedTransformOutput::Spill(path), WriteTarget::Spill))
        }
        WriteTarget::Switch => {
            if exceeds {
                let path = resolve_spill_path(path_hint, allowlist)?;
                Ok((ResolvedTransformOutput::Spill(path), WriteTarget::Switch))
            } else {
                Ok((ResolvedTransformOutput::Ram, WriteTarget::Switch))
            }
        }
        WriteTarget::Sidecar => unreachable!("handled above"),
    }
}

fn resolve_spill_path(
    path_hint: Option<&str>,
    allowlist: &SpillPathAllowlist,
) -> Result<PathBuf, TetError> {
    match path_hint {
        Some(handle) => allowlist.validate(Path::new(handle)),
        None => allowlist.allocate_temp_spill_path(),
    }
}