liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

//! Leaf error type for the low-level LMDB layer, bridged into `OxenError` via the
//! `#[from]`-derived `OxenError::LmdbLayer` variant (the same module-local-enum pattern as
//! `MerkleDbError` and `RepoConfigError`).
//!
//! A dedicated enum, rather than variants on `OxenError`, for two reasons. It keeps `heed::Error`
//! (carried by `Open`/`Txn`/`Read`/`Write`/`Copy`) out of the top-level `OxenError` surface, so
//! the heed dependency stays contained to this layer. And it keeps this cohesive cluster of
//! low-level infra errors — none of which a public-API caller branches on — together and out of
//! the already-large `OxenError`, while still letting in-layer code branch on a specific heed
//! cause (`is_map_full`, `is_env_already_opened`) against a narrow type before the `?`-conversion.
//!
//! Values are opaque bytes at this layer, so per-record parse errors belong to the caller, not
//! here.

use std::path::PathBuf;

use bytesize::ByteSize;
use heed::MdbError;
use thiserror::Error;

#[derive(Debug, Error)]
pub enum LmdbLayerError {
    #[error("Cannot create LMDB map of size {0}: larger than this system's addressable memory.")]
    MapSizeUnrepresentable(ByteSize),

    #[error("Could not create LMDB environment directory {path}: {source}")]
    CreateDir {
        path: PathBuf,
        #[source]
        source: std::io::Error,
    },

    /// `EnvOpenOptions::open` failed. Notably wraps heed `EnvAlreadyOpened`; the env registry
    /// exists to avoid triggering it, but eviction-while-borrowed can still surface it (see
    /// `env_registry.rs`), where the registry retries by re-sharing the live handle.
    #[error("Could not open LMDB environment at {path}: {source}")]
    Open {
        path: PathBuf,
        #[source]
        source: heed::Error,
    },

    #[error("LMDB transaction error: {0}")]
    Txn(#[source] heed::Error),

    #[error("LMDB read error: {0}")]
    Read(#[source] heed::Error),

    #[error("LMDB write error: {0}")]
    Write(#[source] heed::Error),

    #[error("Could not open LMDB database {name:?}: {source}")]
    OpenDb {
        name: String,
        #[source]
        source: heed::Error,
    },

    #[error(
        "LMDB map is full (capacity {capacity}). The map size is a compile-time constant per \
         store; increase it and rebuild."
    )]
    MapFull { capacity: ByteSize },

    #[error("Could not copy LMDB environment to {dst}: {source}")]
    Copy {
        dst: PathBuf,
        #[source]
        source: heed::Error,
    },
}

impl LmdbLayerError {
    /// True if this is heed's `MDB_MAP_FULL` underneath, regardless of whether it surfaced as
    /// `Write` or `Txn`. The write path uses this to translate a raw heed map-full into the
    /// `MapFull { capacity }` variant (which carries the compile-time-constant hint) instead of
    /// an opaque heed error.
    pub fn is_map_full(&self) -> bool {
        matches!(
            self,
            LmdbLayerError::Write(heed::Error::Mdb(MdbError::MapFull))
                | LmdbLayerError::Txn(heed::Error::Mdb(MdbError::MapFull))
        )
    }

    /// True if this is heed's `EnvAlreadyOpened` underneath. The env registry uses this to
    /// detect an open that raced a concurrent close and re-share the live handle instead of
    /// surfacing the error.
    pub fn is_env_already_opened(&self) -> bool {
        matches!(
            self,
            LmdbLayerError::Open {
                source: heed::Error::EnvAlreadyOpened,
                ..
            }
        )
    }
}