lance_core/utils/
blob.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4use object_store::path::Path;
5
6use crate::{Error, Result};
7
8/// Validate a Managed descriptor's object-relative path and known byte range.
9pub fn validate_managed_reference(uri: &str, position: u64, size: u64) -> Result<Path> {
10    if uri.is_empty()
11        || uri.starts_with('/')
12        || uri.ends_with('/')
13        || uri.contains("://")
14        || uri.contains('\\')
15        || uri
16            .split('/')
17            .any(|part| part.is_empty() || part == "." || part == "..")
18    {
19        return Err(Error::invalid_input(format!(
20            "Managed blob_uri must be a non-empty canonical relative object path, got {uri:?}"
21        )));
22    }
23    position.checked_add(size).ok_or_else(|| {
24        Error::invalid_input(format!(
25            "Managed blob range overflows u64: position={position}, size={size}"
26        ))
27    })?;
28    Path::parse(uri)
29        .map_err(|error| Error::invalid_input(format!("Invalid Managed blob_uri {uri:?}: {error}")))
30}
31
32/// Format a blob sidecar path for a data file.
33///
34/// Layout: `<base>/<data_file_key>/<obfuscated_blob_id>.blob`
35/// - `base` is typically the dataset's data directory.
36/// - `data_file_key` is the stem of the data file (without extension).
37/// - `blob_id` is transformed via `reverse_bits()` before binary formatting.
38pub fn blob_path(base: &Path, data_file_key: &str, blob_id: u32) -> Path {
39    let file_name = format!("{:032b}.blob", blob_id.reverse_bits());
40    base.clone().join(data_file_key).join(file_name.as_str())
41}
42
43#[cfg(test)]
44mod tests {
45    use super::*;
46    use rstest::rstest;
47
48    #[rstest]
49    #[case("")]
50    #[case("/data/a.blob")]
51    #[case("data/../a.blob")]
52    #[case("data/./a.blob")]
53    #[case("data//a.blob")]
54    #[case("data/a.blob/")]
55    #[case("s3://bucket/a.blob")]
56    #[case("data\\a.blob")]
57    fn managed_paths_reject_noncanonical_references(#[case] uri: &str) {
58        let error = validate_managed_reference(uri, 0, 1).unwrap_err();
59        assert!(matches!(error, Error::InvalidInput { .. }));
60        assert!(error.to_string().contains("Managed blob_uri"));
61    }
62
63    #[test]
64    fn managed_ranges_preserve_empty_values_and_reject_overflow() {
65        assert!(validate_managed_reference("data/a.blob", u64::MAX, 0).is_ok());
66        let error = validate_managed_reference("data/a.blob", u64::MAX, 1).unwrap_err();
67        assert!(matches!(error, Error::InvalidInput { .. }));
68        assert!(
69            error
70                .to_string()
71                .contains("position=18446744073709551615, size=1")
72        );
73    }
74
75    #[test]
76    fn test_blob_path_formatting() {
77        let base = Path::from("base");
78        let path = blob_path(&base, "deadbeef", 2);
79        assert_eq!(
80            path.to_string(),
81            "base/deadbeef/01000000000000000000000000000000.blob"
82        );
83    }
84
85    #[test]
86    fn test_blob_path_scattered_prefixes_for_sequential_ids() {
87        let base = Path::from("base");
88        let p1 = blob_path(&base, "deadbeef", 1);
89        let p2 = blob_path(&base, "deadbeef", 2);
90        assert_ne!(p1.to_string(), p2.to_string());
91        assert_eq!(
92            p1.to_string(),
93            "base/deadbeef/10000000000000000000000000000000.blob"
94        );
95        assert_eq!(
96            p2.to_string(),
97            "base/deadbeef/01000000000000000000000000000000.blob"
98        );
99    }
100}
lance_core/utils/blob.rs

lance_core/utils/
blob.rs
